harper 5.3.2 → 5.3.4

package/dist/resources/Table.js

@@ -118,6 +118,8 @@ const MAX_CONCURRENT_LMDB_HISTORY_REMOVALS = 1000;
 // Each evict otherwise pays a full transaction commit, so batching amortizes that cost. LMDB already
 // coalesces async writes per event turn (eventTurnBatching), so it keeps the per-record path.
 const EVICTION_BATCH_SIZE = 100;
+const COMMITTED_READ = { uncachedRead: true };
+const UNRESOLVED = Symbol('unresolved');
 // Cap on eviction-batch commits in flight at once, so commit I/O overlaps scan/staging without
 // letting an unbounded number of open transactions (and their snapshots) accumulate.
 const MAX_INFLIGHT_EVICTION_BATCHES = 4;
@@ -2969,10 +2971,22 @@ function makeTable(options) {
                 before: this.constructor.source?.relocate && !context?.source
                     ? this.constructor.source.relocate.bind(this.constructor.source, id, undefined, context)
                     : undefined,
-                commit: (txnTime, existingEntry, _retry, transaction) => {
+                commit: (txnTime, existingEntry, _retry, transaction, committedBy) => {
                     const txnLogKey = isRocksDB && options?.version != null ? (transaction?.getTimestamp?.() ?? txnTime) : txnTime;
-                    if (precedesExistingVersion(txnTime, existingEntry, options?.nodeId) < 0)
+                    write.skipped = false;
+                    const precedesExisting = precedesExistingVersion(txnTime, existingEntry, options?.nodeId);
+                    if (precedesExisting < 0)
+                        return;
+                    if (
+                    // a re-delivered relocate would restamp the committed head, which can be another write of its transaction
+                    precedesExisting === 0 &&
+                        committedBy?.sourceApply &&
+                        isRocksDB &&
+                        txnLogKey === txnTime &&
+                        isHeadIdentity(identityBase(write, existingEntry, committedBy), txnTime, options?.nodeId)) {
+                        write.skipped = true;
                         return;
+                    }
                     const residency = _a.getResidencyRecord(options.residencyId);
                     let metadata = 0;
                     let newRecord = null;
@@ -3477,7 +3491,9 @@ function makeTable(options) {
             return this;
         }
         /**
-         * Release the lock this instance holds. Resolves true when this call cleared the native key lock.
+         * Release the lock this instance holds. Resolves true when this call cleared the native key lock;
+         * false when it was already released, or when a commit of this lock's writes is still in flight
+         * and the native key is returned once that commit settles.
          * Works for both held (`{ hold: true }`) and transaction-scoped locks. After unlock() the
          * instance is no longer lock-writable; writes through it require a fresh lock.
          */
@@ -3776,16 +3792,14 @@ function makeTable(options) {
                     }
                 },
                 before: writeToSource(),
-                commit: (txnTime, existingEntry, retry, transaction) => {
-                    // Whether a prior attempt of THIS write appended its own audit entry (sticky, set in
-                    // save(); log entries are not part of the aborted rocks transaction, so they survive).
-                    // Only such a write can find its own orphaned entry in the dedup lookups below and must
-                    // not treat it as "already applied". Per-write on purpose: the transaction-wide retry
-                    // flag would also suppress dedup for a genuine re-delivered duplicate co-batched with
-                    // the conflicting write (double-applying it) and for fresh writes staged through a
-                    // reused transaction whose retries counter is stale. Sticky on purpose: a proxy read
-                    // from the last attempt's skipped state launders when a recommit round self-skips
-                    // (walk identity tie against its own staged record) before a fresh-transaction replay.
+                commit: (txnTime, existingEntry, retry, transaction, committedBy) => {
+                    // Whether a prior attempt of THIS write appended its audit entry (sticky, set in
+                    // save()). A retry cannot infer that its record mutation committed from an exact log
+                    // match alone, so keyed probes stay shielded and walk skips require committed provenance.
+                    // Per-write: a transaction-wide retry flag would also suppress dedup for unrelated
+                    // deliveries co-batched with the conflict or fresh writes in a reused transaction.
+                    // Sticky: a proxy read from the last attempt's skipped state launders when a recommit
+                    // round self-skips (walk identity tie against its own staged record) before a replay.
                     const stagedOwnAuditEntry = retry && write.appendedAuditEntry === true;
                     write.skipped = false; // reset on each retry; cleanup happens after commit if still true
                     write.stagedEntry = undefined; // likewise: only set once this round actually stores a record
@@ -3867,6 +3881,16 @@ function makeTable(options) {
                         let belowAuditFloor = false;
                         let dedupVersionCouldBeRetained;
                         if (audit) {
+                            let identityRecord = UNRESOLVED;
+                            if (committedBy?.sourceApply &&
+                                isRocksDB &&
+                                !replaying &&
+                                precedesExisting === 0 &&
+                                txnLogKey === txnTime &&
+                                isHeadIdentity((identityRecord = identityBase(write, existingEntry, committedBy)), txnTime, options?.nodeId)) {
+                                write.skipped = true;
+                                return;
+                            }
                             // A re-delivered out-of-order write (full-copy audit-replay re-delivers writes) must not have
                             // its commutative ops re-folded. additionalAuditRefs is the record's own list of folded
                             // out-of-order versions, read with read-your-writes consistency, so this skips the duplicate up
@@ -3879,7 +3903,9 @@ function makeTable(options) {
                             // best-effort keyed lookup in the capped block below — see #1148. precedesExistingVersion(...)
                             // === 0 is the identity tie: same version AND same node (the local node is id 0, so an undefined
                             // options?.nodeId resolves to the same 0 the ref stored).
-                            if (existingEntry.additionalAuditRefs?.some((ref) => ref.version === txnLogKey &&
+                            if ((identityRecord === UNRESOLVED
+                                ? identityBase(write, existingEntry, committedBy)
+                                : identityRecord)?.additionalAuditRefs?.some((ref) => ref.version === txnLogKey &&
                                 precedesExistingVersion(txnTime, { version: txnTime, localTime: txnLogKey, key: id, nodeId: ref.nodeId }, options?.nodeId) === 0)) {
                                 write.skipped = true;
                                 return; // out-of-order write already folded into this record
@@ -3930,11 +3956,7 @@ function makeTable(options) {
                             // RocksDB-only: LMDB audit entries are keyed by local audit time, so this lookup doesn't apply
                             // there (LMDB keeps the exact unbounded walk). A miss (the keyed lookup can lag a back-to-back re-delivery — #1137)
                             // simply falls through to the walk, so this never changes correctness; the additionalAuditRefs
-                            // check above remains the read-your-writes guard. Never when this write staged in a prior
-                            // failed attempt: that attempt already appended this write's own audit entry, so the lookup
-                            // would find it and skip the write as "already applied" when the record was never committed.
-                            // A recommit of the same transaction survived that skip only because the old write batch
-                            // still carried the put; a fresh-transaction replay (ERR_TRY_AGAIN) would drop the write.
+                            // check above remains the read-your-writes guard. Not on a same-write retry (see stagedOwnAuditEntry).
                             if (isRocksDB && !replaying && !stagedOwnAuditEntry && dedupVersionCouldBeRetained(txnLogKey)) {
                                 const priorAudit = auditStore.get(txnLogKey, tableId, id, options?.nodeId);
                                 if (priorAudit &&
@@ -3970,9 +3992,18 @@ function makeTable(options) {
                             // `previousNodeId` is never encoded, so absent there means the log is unknown and only the
                             // aggregate lookup can resolve a cross-origin predecessor.
                             let nodeId = initialAuditHead.nodeId ?? 0;
+                            // A retry can load a base restamped by an earlier saved publish/relocate from this
+                            // transaction. Those writes do not set stagedEntry, so priorStaged alone cannot
+                            // establish that the base is committed.
+                            const retryBaseIsCommitted = !priorStaged && !(stagedOwnAuditEntry && hasSavedSameKeySibling(write, committedBy));
+                            let walkOriginKnown = retryBaseIsCommitted;
                             const succeedingUpdates = []; // record the "future" updates, as we need to apply the updates in reverse order
                             const auditRefsToVisit = existingEntry.additionalAuditRefs
-                                ? existingEntry.additionalAuditRefs.map((ref) => ({ localTime: ref.version, nodeId: ref.nodeId }))
+                                ? existingEntry.additionalAuditRefs.map((ref) => ({
+                                    localTime: ref.version,
+                                    nodeId: ref.nodeId,
+                                    originKnown: retryBaseIsCommitted,
+                                }))
                                 : [];
                             // Out-of-order merges retain every existing branch head; per-origin log keys are not globally ordered.
                             if (existingEntry.additionalAuditRefs) {
@@ -3983,11 +4014,11 @@ function makeTable(options) {
                             let addedAuditRef = false;
                             let nextRef;
                             const visitedAuditRefs = new Set();
-                            const queuePreviousAuditRefs = (auditRecord) => {
+                            const queuePreviousAuditRefs = (auditRecord, originKnown) => {
                                 const previousRefs = auditRecord.previousAdditionalAuditRefs;
                                 if (previousRefs) {
                                     for (const ref of previousRefs) {
-                                        auditRefsToVisit.push({ localTime: ref.version, nodeId: ref.nodeId });
+                                        auditRefsToVisit.push({ localTime: ref.version, nodeId: ref.nodeId, originKnown });
                                         logger_ts_1.logger.debug?.('Adding audit ref from audit record to visit queue', {
                                             version: ref.version,
                                             nodeId: ref.nodeId,
@@ -4002,6 +4033,7 @@ function makeTable(options) {
                                     : { txnLogKey: auditRecord.previousVersion, nodeId: auditRecord.previousNodeId };
                                 localTime = previousHead.txnLogKey;
                                 nodeId = previousHead.nodeId;
+                                walkOriginKnown = walkOriginKnown && previousHead.nodeId != null;
                             };
                             let walkSteps = 0;
                             let auditWalkCapped = false;
@@ -4017,9 +4049,7 @@ function makeTable(options) {
                             // A re-delivered write whose exact (version, nodeId) is already in the audit log was already
                             // applied; drop it rather than re-applying it (double-applying commutative ops) or writing a
                             // duplicate audit-only record. Used by the early-out and the depth-cap block below.
-                            // Never a duplicate for a write that staged in a prior failed attempt: that attempt already
-                            // appended this write's own audit entry, so the lookup would match it while the record was
-                            // never committed (see the up-front keyed dedup above).
+                            // Not for a same-write retry (see stagedOwnAuditEntry).
                             const isReDeliveredDuplicate = () => {
                                 if (replaying || stagedOwnAuditEntry)
                                     return false;
@@ -4047,12 +4077,14 @@ function makeTable(options) {
                                     const auditRecord = auditStore.get(localTime, tableId, id, nodeId);
                                     if (!auditRecord)
                                         break;
-                                    queuePreviousAuditRefs(auditRecord);
+                                    queuePreviousAuditRefs(auditRecord, walkOriginKnown);
                                     if (isRocksDB &&
                                         !replaying &&
-                                        !stagedOwnAuditEntry &&
+                                        // on a retry only a link from committed state proves the write committed: an
+                                        // orphaned log entry can match the identity but is never linked from a committed record
+                                        (!stagedOwnAuditEntry || walkOriginKnown) &&
                                         localTime === txnLogKey &&
-                                        precedesExistingVersion(txnTime, { version: txnTime, localTime: txnLogKey, key: id, nodeId: auditRecord.nodeId }, options?.nodeId) === 0) {
+                                        (auditRecord.nodeId ?? 0) === (options?.nodeId ?? (0, nodeIdMapping_ts_1.getThisNodeId)(auditStore))) {
                                         write.skipped = true;
                                         return;
                                     }
@@ -4066,9 +4098,8 @@ function makeTable(options) {
                                                     // orders the otherwise non-unique record clock without comparing keys across origins.
                                                     precedesExisting = txnLogKey > localTime ? 1 : -1;
                                                 }
-                                                else if (replaying || stagedOwnAuditEntry) {
-                                                    // The log entry being replayed (or staged by this write's failed attempt) is
-                                                    // the write itself, not proof that its primary-store mutation committed.
+                                                else if (replaying || (stagedOwnAuditEntry && !walkOriginKnown)) {
+                                                    // replayed, or reached on a retry without committed provenance: not proof it committed
                                                     precedesExisting = 1;
                                                 }
                                                 else {
@@ -4150,6 +4181,7 @@ function makeTable(options) {
                                 if (nextRef) {
                                     localTime = auditedVersion = nextRef.localTime;
                                     nodeId = nextRef.nodeId;
+                                    walkOriginKnown = nextRef.originKnown;
                                     logger_ts_1.logger.debug?.('Following additional audit ref to continue scanning', { localTime, nodeId });
                                 }
                             } while (nextRef);
@@ -4975,6 +5007,7 @@ function makeTable(options) {
             // scans), the read transaction reads against the latest committed data without pinning a
             // consistent snapshot, so the scan doesn't hold a snapshot that blocks compaction.
             const readTxn = txn.useReadTxn(target.snapshot === false);
+            const readTxnGeneration = txn.readTxnGeneration;
             // The explicit row filter participates in query execution: it is pushed into HNSW
             // traversal for vector sorts, applied after conditions otherwise, and checked again after
             // cache/source materialization below. A policy error aborts the query instead of silently
@@ -5093,7 +5126,8 @@ function makeTable(options) {
                         finally {
                             // We own the iteration here (no results.onDone consumer), so release the read
                             // transaction unconditionally — including when the drain throws — or the snapshot leaks.
-                            txn.doneReadTxn();
+                            if (readTxn)
+                                txn.doneReadTxn(readTxnGeneration);
                         }
                         let total;
                         if (wantExact) {
@@ -5137,7 +5171,8 @@ function makeTable(options) {
                 results.onDone = () => {
                     results.onDone = null; // ensure that it isn't called twice
                     txn.unregisterReadIterator(results);
-                    txn.doneReadTxn();
+                    if (readTxn)
+                        txn.doneReadTxn(readTxnGeneration);
                 };
                 // Recorded ownership: if the request dies before anything consumes these results, the
                 // transaction closes them itself rather than leaving its read snapshot pinned.
@@ -5147,7 +5182,8 @@ function makeTable(options) {
                 return results;
             }
             catch (error) {
-                txn.doneReadTxn();
+                if (readTxn)
+                    txn.doneReadTxn(readTxnGeneration);
                 throw error;
             }
         }
@@ -6966,6 +7002,7 @@ function makeTable(options) {
                 },
             };
             propertyResolvers.$highlights.directReturn = true;
+            primaryKeyAttribute = undefined;
             for (const attribute of this.attributes) {
                 if (attribute.isPrimaryKey)
                     primaryKeyAttribute = attribute;
@@ -8137,6 +8174,32 @@ function makeTable(options) {
         if (!(existingEntry?.metadataFlags & exports.INVALIDATED))
             return existingEntry;
     }
+    // The commit base is the committed record unless another write to the key already saved without being
+    // skipped this round: a publish or relocate saves when it is added and restamps the base without staging it.
+    function identityBase(write, existingEntry, transaction) {
+        if (hasSavedSameKeySibling(write, transaction))
+            return committedEntry(write.key);
+        return existingEntry;
+    }
+    function hasSavedSameKeySibling(write, transaction) {
+        for (let other = transaction?.writesByKey?.get(write.store)?.get(write.keyId); other; other = other.priorWrite) {
+            if (other !== write &&
+                other.saved &&
+                !other.skipped &&
+                other.committedOn === write.committedOn &&
+                other.committedRound === write.committedRound)
+                return true;
+        }
+        return false;
+    }
+    function committedEntry(id) {
+        return primaryStore.getEntry(id, COMMITTED_READ);
+    }
+    function isHeadIdentity(entry, version, nodeId) {
+        return (entry?.version === version &&
+            (entry.nodeId ?? 0) === (nodeId ?? (0, nodeIdMapping_ts_1.getThisNodeId)(auditStore)) &&
+            !entry.additionalAuditRefs?.length);
+    }
     function precedesExistingVersion(txnTime, existingEntry, nodeId) {
         if (nodeId === undefined) {
             nodeId = (0, nodeIdMapping_ts_1.getThisNodeId)(auditStore);