@harperfast/harper 5.2.10 → 5.3.0-alpha.1

package/dist/resources/DatabaseTransaction.js

@@ -33,24 +33,27 @@ var __importStar = (this && this.__importStar) || (function () {
     };
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.RELEASED_TRANSACTION = exports.ImmediateTransaction = exports.DatabaseTransaction = exports.TRANSACTION_STATE = void 0;
+exports.RELEASED_TRANSACTION = exports.ImmediateTransaction = exports.DatabaseTransaction = exports.COMMIT_PHASE_GRACE = exports.TRANSACTION_STATE = void 0;
 exports.trackOutstandingCommit = trackOutstandingCommit;
 exports.getOutstandingCommits = getOutstandingCommits;
 exports.setCommitLatencyRecorder = setCommitLatencyRecorder;
 exports.getTransactionQueueDepths = getTransactionQueueDepths;
 exports.replicationConfirmation = replicationConfirmation;
 exports.transactionOpenTooLongError = transactionOpenTooLongError;
-exports.trackReadRange = trackReadRange;
+exports.getAppliedWriteVersion = getAppliedWriteVersion;
 exports.priorStagedWrite = priorStagedWrite;
 exports.writeKeyId = writeKeyId;
+exports.shouldSpareCommitPhase = shouldSpareCommitPhase;
 exports.isReleasedTransaction = isReleasedTransaction;
 exports.isJoinableScope = isJoinableScope;
 exports.resetReplayedWritesWarning = resetReplayedWritesWarning;
+exports.setMaxOutstandingTxnDuration = setMaxOutstandingTxnDuration;
 exports.isWriteSupervised = isWriteSupervised;
 exports.setTxnExpiration = setTxnExpiration;
 const blob_ts_1 = require("./blob.js");
 const commonUtility_ts_1 = require("../utility/lmdb/commonUtility.js");
 const hdbError_ts_1 = require("../utility/errors/hdbError.js");
+const recordLock_ts_1 = require("./recordLock.js");
 const harperLogger = __importStar(require("../utility/logging/harper_logger.js"));
 const envMngr = __importStar(require("../utility/environment/environmentManager.js"));
 const hdbTerms_ts_1 = require("../utility/hdbTerms.js");
@@ -60,8 +63,8 @@ const promises_1 = require("node:timers/promises");
 const rocksdb_js_1 = require("@harperfast/rocksdb-js");
 const RETRY_NOW_VALUE = rocksdb_js_1.constants.RETRY_NOW_VALUE;
 const ordered_binary_1 = require("ordered-binary");
+const longLivedTransactions_ts_1 = require("./longLivedTransactions.js");
 const trackedTxns = new Set();
-const readTransactionOwners = new WeakMap();
 // Read options for a rotated generation's native transactions; shared because they never vary.
 const SNAPSHOT_FREE = Object.freeze({ disableSnapshot: true });
 // Logical transactions the monitor supervises for their WRITES, kept apart from trackedTxns because the
@@ -69,7 +72,7 @@ const SNAPSHOT_FREE = Object.freeze({ disableSnapshot: true });
 // is what the read-queue-depth metric counts, while this holds one entry per logical transaction — the
 // chain root — so a chain child can never become its own timeout root (issue #2231).
 const supervisedWriteRoots = new Set();
-const MAX_OUTSTANDING_TXN_DURATION = (0, common_utils_ts_1.convertToMS)(envMngr.get(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_MAXTRANSACTIONQUEUETIME)) || 45000; // Allow write transactions to be queued for up to 45 seconds before we start rejecting them
+let MAX_OUTSTANDING_TXN_DURATION = (0, common_utils_ts_1.convertToMS)(envMngr.get(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_MAXTRANSACTIONQUEUETIME)) || 45000; // Allow write transactions to be queued for up to 45 seconds before we start rejecting them
 const DEBUG_LONG_TXNS = envMngr.get(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_DEBUGLONGTRANSACTIONS);
 exports.TRANSACTION_STATE = {
     CLOSED: 0, // the transaction has been committed or aborted and can no longer be used for writes (if read txn is active, it can be used for reads)
@@ -81,6 +84,12 @@ exports.TRANSACTION_STATE = {
     LINGERING: 2,
 };
 const MAX_RETRIES = 40;
+// Over-limit monitor ticks a transaction parked in its commit phase is spared before it is aborted
+// like any other over-time transaction. Sparing re-arms `timeout`, so each of these costs two
+// monitor intervals — roughly 10 minutes at the 30s default. Generous, so a legitimately large blob
+// write finishes, but bounded, so a pre-commit source that stalls instead of finishing can't pin its
+// read snapshot forever (issue #2062).
+exports.COMMIT_PHASE_GRACE = 10;
 // Cap the per-retry backoff so replication-applied transactions, which retry conflicts without a
 // cap (see the commit rejection handler), don't grow the delay unbounded.
 const MAX_RETRY_DELAY_MS = 1000;
@@ -91,7 +100,27 @@ let outstandingCommitCount = 0;
 // the whole thread, regardless of how many distinct commits individually cross the threshold — see
 // the comment at the log site for why a per-commit-only dedup isn't enough under sustained overload.
 const OVERLOAD_LOG_MIN_INTERVAL_MS = 1000;
-let lastOverloadLogAt = -Infinity;
+// One cooldown per reporting site, not one shared: `shed` fires on every bystander write during a
+// wedge and would starve `abandon`, which names a different transaction and is the only server-side
+// record of why a request was failed. Mutates only when it grants, so a caller with its own
+// suppression as well (checkOverloaded's per-node `logged`) must test that first.
+const lastStuckCommitLogAt = { shed: -Infinity, abandon: -Infinity };
+function allowStuckCommitLog(site, now) {
+    if (now - lastStuckCommitLogAt[site] <= OVERLOAD_LOG_MIN_INTERVAL_MS)
+        return false;
+    lastStuckCommitLogAt[site] = now;
+    return true;
+}
+// Which database/table, which native transaction, and which request a stuck commit belongs to —
+// without it a wedge gives no indication of what to investigate (harper#2001).
+function describeCommitIdentity(store, startedFrom, nativeTransaction) {
+    const nativeTransactionId = nativeTransaction?.id;
+    return (`from table: ${store?.rootStore?.databaseName ?? '?'}.${store?.name ?? '?'}` +
+        (nativeTransactionId !== undefined ? ` (transaction ${nativeTransactionId})` : '') +
+        (startedFrom?.resourceName
+            ? `, started from ${startedFrom.resourceName}${startedFrom.method ? '.' + startedFrom.method : ''}`
+            : ''));
+}
 // Track a submitted commit until it settles. Every attempt is tracked unconditionally: a
 // coordinated retry round and a chained second-store commit are both issued from inside the
 // preceding commit's own resolve handler, which runs before any reaction that could release a
@@ -169,6 +198,21 @@ function getOutstandingCommits() {
 // Once per process: committing under open read iterators forces a write replay, so the warning is
 // about the caller's pattern, not the individual commit.
 let replayedWritesWarned = false;
+/**
+ * Abort a detached native handle. RocksTransaction.abort() throws on one that was already
+ * committed or aborted, and every caller is a cleanup path whose own callers have no handler — a
+ * throw there would abandon the rest of the cleanup.
+ */
+function abortNativeTransaction(transaction, context) {
+    if (transaction == null)
+        return;
+    try {
+        transaction.abort();
+    }
+    catch (error) {
+        harperLogger.debug?.(context, error);
+    }
+}
 // The analytics module registers a recorder here at load (dependency inversion, mirroring
 // `replicationConfirmation` below) so the storage layer doesn't statically import the analytics/server
 // modules. Unset until analytics loads, and when analytics is disabled the recorder call is cheap.
@@ -253,78 +297,8 @@ function transactionOpenTooLongError() {
     // 422 signals the request itself must change (split the work), which is the actionable response.
     return new hdbError_ts_1.ServerError('Transaction was aborted after exceeding the maximum open-transaction time; split long-running work into smaller transactions', 422);
 }
-class ReadSnapshotExpiredError extends hdbError_ts_1.ServerError {
-    constructor() {
-        super('Read scan snapshot expired; retry the read without replaying previously committed writes', 503);
-        this.name = 'ReadSnapshotExpiredError';
-    }
-}
-function trackReadRange(transaction, createRange) {
-    const owner = readTransactionOwners.get(transaction);
-    if (!owner)
-        return createRange();
-    function checkActive() {
-        if (owner.timedOut)
-            throw transactionOpenTooLongError();
-        if (owner.transaction !== transaction) {
-            throw new ReadSnapshotExpiredError();
-        }
-    }
-    checkActive();
-    const range = createRange();
-    const iterate = range.iterate;
-    range.iterate = function (options) {
-        const iterator = iterate.call(this, options);
-        let done = false;
-        // Closing the underlying iterator is the one step here that can throw for a reason the caller
-        // must not see: `next()` only reaches it once the snapshot is already gone, which is the
-        // likeliest moment for the native layer to object, and an error from cleanup would replace the
-        // named 503 with exactly the raw iterator error this wrapper exists to stop surfacing. The
-        // closure reference rather than `this` so a destructured `next` still cleans up.
-        const wrapper = {
-            [Symbol.iterator]() {
-                return this;
-            },
-            next() {
-                if (done)
-                    return { done: true, value: undefined };
-                try {
-                    checkActive();
-                    owner.rangeReadActive = true;
-                    const result = iterator.next();
-                    done = result.done === true;
-                    return result;
-                }
-                catch (error) {
-                    closeQuietly();
-                    throw error;
-                }
-            },
-            return(value) {
-                if (!done) {
-                    done = true;
-                    iterator.return?.(value);
-                }
-                return { done: true, value };
-            },
-            throw(error) {
-                // Not delegated to `iterator.throw`: it closes and rethrows the same error anyway, and
-                // delegating after the close below would run it against an iterator already closed.
-                closeQuietly();
-                throw error;
-            },
-        };
-        function closeQuietly() {
-            try {
-                wrapper.return();
-            }
-            catch {
-                // the error being propagated is the actionable one
-            }
-        }
-        return wrapper;
-    };
-    return range;
+function getAppliedWriteVersion(recordVersion, txnLogKey) {
+    return recordVersion == null ? txnLogKey : Math.min(recordVersion, txnLogKey);
 }
 /**
  * The state a preceding write in this transaction left for `operation`'s key, or undefined if this
@@ -375,7 +349,11 @@ class DatabaseTransaction {
     readTxnRefCount;
     readTxnsUsed;
     timeout;
+    writeTick = -1;
     timeoutBudget = 0;
+    // Initialized rather than `declare`d so the class shape stays monomorphic on the write path. One clock
+    // read per handle is what lets the monitor age a handle without calling into the registry every tick.
+    handleOpenedAt = 0;
     // save() only stages here; ImmediateTransaction overrides it to commit, which addWrite must not defer
     saveCommits = false;
     // True where save() puts the write into this transaction's native handle, which is what lets a scope
@@ -394,8 +372,24 @@ class DatabaseTransaction {
     // asks to stop reading a pinned snapshot, so re-pinning one for the rest of the scope would take
     // back what it asked for.
     snapshotFree = false;
-    rangeReadActive = false;
-    renewReadTimeout() {
+    // Set while commit() is parked in its pre-commit await (`before`/`beforeIntermediate` completions —
+    // in practice a blob's durable file write). The write set is sealed and the caller is awaiting the
+    // commit, so this is core's own I/O rather than an application holding a transaction open, which is
+    // what the open-transaction limit polices: the monitor spares it for COMMIT_PHASE_GRACE ticks
+    // instead of poisoning it (issue #2062).
+    committing = false;
+    commitPhaseTicks = 0;
+    setCommitPhase(committing) {
+        // A commit phase covers the sealed write set across the whole multi-store chain.
+        for (let txn = this; txn; txn = txn.next) {
+            txn.committing = committing;
+            txn.commitChainHead = committing ? this : undefined;
+            if (committing)
+                txn.commitPhaseTicks = 0;
+        }
+    }
+    getReadTxn(disableSnapshot) {
+        this.readTxnRefCount = (this.readTxnRefCount || 0) + 1;
         // The limit is an IDLE limit. Writes always re-arm it (see addWrite), but reads only do so
         // while no uncommitted writes are held: staged writes hold write intents that other writers'
         // coordinated-retry commits park on, so a handler that wrote once and then only reads — an
@@ -408,10 +402,6 @@ class DatabaseTransaction {
         if ((this.writes.length === 0 && !this.next) || this.open !== exports.TRANSACTION_STATE.OPEN || !this.hasPendingWrites()) {
             this.timeout = Math.max(txnExpiration, this.timeoutBudget);
         }
-    }
-    getReadTxn(disableSnapshot) {
-        this.readTxnRefCount = (this.readTxnRefCount || 0) + 1;
-        this.renewReadTimeout();
         if (this.transaction) {
             if (this.transaction.openTimer)
                 this.transaction.openTimer = 0;
@@ -444,10 +434,9 @@ class DatabaseTransaction {
     // Monitor state is not ownership state: it stays with `trackedTxns.add` in getReadTxn().
     attachOwnedTransaction(transaction) {
         this.transaction = transaction;
-        readTransactionOwners.set(transaction, this);
-        this.rangeReadActive = false;
         this.readTxnsUsed = 1;
         this.baseReadRefConsumed = false;
+        this.handleOpenedAt = performance.now();
     }
     /**
      * Drop this link's supervision claim, and the root's with it once no link in the chain still holds
@@ -486,9 +475,9 @@ class DatabaseTransaction {
         trackedTxns.delete(this);
         this.endWriteSupervision();
         this.transaction = null;
-        this.rangeReadActive = false;
         this.readTxnsUsed = 0;
         this.readTxnRefCount = 0;
+        this.handleOpenedAt = 0;
         return transaction;
     }
     useReadTxn(disableSnapshot) {
@@ -535,12 +524,7 @@ class DatabaseTransaction {
      */
     releaseReadTxn() {
         const transaction = this.detachOwnedTransaction();
-        try {
-            transaction?.abort();
-        }
-        catch (error) {
-            harperLogger.debug?.('releasing timed-out read transaction', error);
-        }
+        abortNativeTransaction(transaction, 'releasing timed-out read transaction');
         this.completeDeferredContextRelease();
     }
     /**
@@ -620,6 +604,77 @@ class DatabaseTransaction {
             }
         }
     }
+    registerRecordLock(handle) {
+        if (!this.recordLocks)
+            this.recordLocks = new Map();
+        let storeMap = this.recordLocks.get(handle.store);
+        if (!storeMap)
+            this.recordLocks.set(handle.store, (storeMap = new Map()));
+        storeMap.set(handle.keyId, handle);
+    }
+    recordLockFor(store, keyId) {
+        const storeMap = this.recordLocks?.get(store);
+        if (!storeMap)
+            return undefined;
+        const h = storeMap.get(keyId);
+        if (!h)
+            return undefined;
+        if (!h.released)
+            return h;
+        // Prune released handles so they don't accumulate; an expired handle checking re-entrancy
+        // would otherwise be seen as the holder and incorrectly granted access.
+        storeMap.delete(keyId);
+        return undefined;
+    }
+    unregisterRecordLock(handle) {
+        const storeMap = this.recordLocks?.get(handle.store);
+        if (storeMap?.get(handle.keyId) === handle)
+            storeMap.delete(handle.keyId);
+    }
+    registerPendingLock(store, keyId, pending) {
+        if (!this.pendingLocks)
+            this.pendingLocks = new Map();
+        let storeMap = this.pendingLocks.get(store);
+        if (!storeMap)
+            this.pendingLocks.set(store, (storeMap = new Map()));
+        storeMap.set(keyId, pending);
+    }
+    pendingLockFor(store, keyId) {
+        return this.pendingLocks?.get(store)?.get(keyId);
+    }
+    unregisterPendingLock(store, keyId) {
+        this.pendingLocks?.get(store)?.delete(keyId);
+    }
+    /** Release every transaction-scoped lock() handle this link owns. */
+    releaseRecordLocks() {
+        const recordLocks = this.recordLocks;
+        if (!recordLocks)
+            return;
+        for (const [store, storeMap] of recordLocks) {
+            for (const [keyId, handle] of storeMap) {
+                if (handle.hold)
+                    continue; // hold handles outlive the transaction; released by unlock()
+                handle.release();
+                storeMap.delete(keyId);
+            }
+            if (storeMap.size === 0)
+                recordLocks.delete(store);
+        }
+        if (recordLocks.size === 0)
+            this.recordLocks = undefined;
+    }
+    noteCommittedLockVersions() {
+        const recordLocks = this.recordLocks;
+        if (!recordLocks)
+            return;
+        for (const write of this.writes) {
+            if (write?.appliedRecordVersion == null)
+                continue;
+            const handle = write.lockHandle ?? recordLocks.get(write.store)?.get(writeKeyId(write.key));
+            if (handle && !handle.released)
+                handle.noteHolderVersion(write.appliedRecordVersion);
+        }
+    }
     /**
      * Discard the staged write set (committed or aborted); the per-key chain must go with it so a
      * reused transaction never bases a write on a previous batch's staged state.
@@ -679,25 +734,20 @@ class DatabaseTransaction {
             // original per-request log was fixed to avoid, just shifted from per-request to per-commit.
             // A commit skipped by the cooldown is NOT marked `logged`, so it still gets a log later if
             // it's still the oldest once the cooldown clears, rather than going silent forever.
-            if (!oldestOutstandingCommit.logged && now - lastOverloadLogAt > OVERLOAD_LOG_MIN_INTERVAL_MS) {
+            if (!oldestOutstandingCommit.logged && allowStuckCommitLog('shed', now)) {
                 // Log once per stuck commit (not once per rejected request, harper#2001): a wedged
                 // thread otherwise logs nothing at all server-side while rejecting every write with a
                 // 503, which was the single biggest obstacle to root-causing a recurrence. The flag lives
                 // on the node itself, so if THIS commit settles while still over the limit and a
                 // different one is now oldest, that one logs too instead of staying silent forever.
                 oldestOutstandingCommit.logged = true;
-                lastOverloadLogAt = now;
-                const nativeTransactionId = oldestOutstandingCommit.nativeTransaction?.id;
-                const store = oldestOutstandingCommit.store;
-                const startedFrom = oldestOutstandingCommit.startedFrom;
                 harperLogger.error(`Rejecting writes on this thread: a commit has been outstanding for ` +
                     `${Math.round(now - oldestOutstandingCommit.start)}ms (exceeds the ` +
-                    `${MAX_OUTSTANDING_TXN_DURATION}ms limit), from table: ${store?.rootStore?.databaseName ?? '?'}.${store?.name ?? '?'}` +
-                    (nativeTransactionId !== undefined ? ` (transaction ${nativeTransactionId})` : '') +
-                    (startedFrom?.resourceName
-                        ? `, started from ${startedFrom.resourceName}${startedFrom.method ? '.' + startedFrom.method : ''}`
-                        : '') +
-                    `. Further record updates and publishes from new application requests on this thread ` +
+                    `${MAX_OUTSTANDING_TXN_DURATION}ms limit), ` +
+                    describeCommitIdentity(oldestOutstandingCommit.store, oldestOutstandingCommit.startedFrom, oldestOutstandingCommit.nativeTransaction) +
+                    `.` +
+                    (0, longLivedTransactions_ts_1.describeHolderCandidates)(oldestOutstandingCommit.store?.rootStore?.path, oldestOutstandingCommit.nativeTransaction?.id) +
+                    ` Further record updates and publishes from new application requests on this thread ` +
                     `will be rejected with 503 until the commit settles or the process is restarted (deletes, ` +
                     `and writes applied from a canonical source, e.g. replication or a caching source, bypass this check).`);
             }
@@ -758,6 +808,7 @@ class DatabaseTransaction {
         // Independent write-recency signal for chainStillActive (see the field comment) — reads never
         // touch this, only writes do.
         this.writeTimeout = this.timeout;
+        this.writeTick = monitorTick;
         this.linkWrite(operation);
         this.writes.push(operation);
         operation.stagedIn = this;
@@ -787,7 +838,34 @@ class DatabaseTransaction {
         return operation;
     }
     save(operation, transaction, reloadEntry = false, options) {
-        let txnTime = this.timestamp;
+        const lockHandle = operation.lockHandle;
+        // Guard: a write staged through an expired or released lock handle must not land.
+        // The handle's lease timer already unlocked the native key; another holder may have taken it.
+        if (lockHandle && (lockHandle.expired || lockHandle.released)) {
+            // Remove the operation from the staged set so subsequent writes on this context do not
+            // re-throw 409 due to a stale null-saved entry sitting in this.writes.
+            const failedIdx = this.writes.indexOf(operation);
+            if (failedIdx > -1)
+                this.writes[failedIdx] = null;
+            throw (0, recordLock_ts_1.lockNotHeldError)(lockHandle);
+        }
+        // Lock-write timestamp rules.
+        if (lockHandle) {
+            if (this.open === exports.TRANSACTION_STATE.CLOSED || this.saveCommits) {
+                // CLOSED path (second+ write per ImmediateTransaction cycle) OR the first write in
+                // an ImmediateTransaction (open=OPEN until commit sets it CLOSED, but saveCommits
+                // signals the per-write-commit semantics):  stamp with nextHolderVersion() so that
+                // each sequential save() gets its own monotonically increasing stamp — scoped or hold.
+                // The stamp stays on the operation and is consumed below rather than assigned to
+                // this.timestamp: pinning the link's clock would stamp every OTHER write staged on
+                // the same context before the commit resets it — a concurrent write in the caller's
+                // own Promise.all, or the next operation in a retry/replay save loop — with the
+                // lock's version, which LWW then silently drops against a newer record version.
+                if (!operation.lockStamp)
+                    operation.lockStamp = lockHandle.holderVersionCandidate();
+            }
+        }
+        let txnTime = operation.lockStamp ?? this.timestamp;
         // Only an OPEN transaction accepts new staged writes. After commit, this.transaction may still
         // be retained for outstanding read iterators; staging into it would silently discard the write
         // when doneReadTxn() aborts the handle, so such writes commit immediately on a fresh
@@ -832,13 +910,18 @@ class DatabaseTransaction {
         }
         if (!txnTime)
             txnTime = this.timestamp = transaction.getTimestamp();
+        // `txnTime` is this transaction's timestamp — the key its entries take in the per-origin log.
+        // A write applied from elsewhere carries the origin's record version too, and that is what the
+        // record is stored at; the two coincide for every locally-originated write. Gated on the apply
+        // flags so an ordinary write never reads the property (harper#2412).
+        const writeVersion = this.sourceApply || this.isReplay ? getAppliedWriteVersion(operation.recordVersion, txnTime) : txnTime;
         if (reloadEntry || operation.entry === undefined) {
             operation.entry = operation.store.getEntry(operation.key, { transaction });
         }
         if (!operation.saved) {
             operation.saved = true;
             // immediately execute in this transaction
-            if (operation.validate?.(txnTime, this) === false) {
+            if (operation.validate?.(writeVersion, this) === false) {
                 operation.commit = () => { }; // noop if we try again
                 return;
             }
@@ -849,7 +932,13 @@ class DatabaseTransaction {
             if (result?.then)
                 this.stageCompletion(result);
         }
-        const completion = operation.commit(txnTime, operation.entry, this.retries > 0, transaction);
+        if (lockHandle || this.recordLocks)
+            operation.trackRecordVersion = true;
+        if (operation.trackRecordVersion)
+            operation.recordVersionApplied = false;
+        const completion = operation.commit(writeVersion, operation.entry, this.retries > 0, transaction);
+        if (operation.trackRecordVersion)
+            operation.appliedRecordVersion = operation.recordVersionApplied ? writeVersion : undefined;
         if (typeof completion?.then === 'function')
             this.stageCompletion(completion);
         // Sticky record that THIS write staged with its audit entry appended (log entries batch on the
@@ -862,7 +951,13 @@ class DatabaseTransaction {
             operation.appendedAuditEntry = true;
         }
         if (immediateCommit) {
-            return this.commit({ ...options, transaction }); // immediately commit if the harper transaction is closed
+            // immediately commit if the harper transaction is closed
+            const innerCommit = this.commit({ ...options, transaction });
+            // Expose on the operation so the lock-writable Table.save() path can await the real
+            // native commit — without this the ImmediateTransaction outer commit resolves before
+            // the native transaction.commit() settles (fire-and-forget from the if-branch).
+            operation.innerCommit = innerCommit;
+            return innerCommit;
         }
     }
     /**
@@ -874,17 +969,38 @@ class DatabaseTransaction {
         // reused across retries — the native layer resets it in place (fresh snapshot) on IsBusy/TryAgain —
         // but reassigned to a fresh replay transaction when outstanding read iterators retain this.transaction
         let transaction = options.transaction ?? this.transaction;
-        for (let i = 0; i < this.writes.length; i++) {
-            let operation = this.writes[i];
-            if (!operation || (this.retries === 0 && operation.saved))
-                continue;
-            this.save(operation, transaction, i < this.validated, options);
+        try {
+            for (let i = 0; i < this.writes.length; i++) {
+                let operation = this.writes[i];
+                if (!operation || (this.retries === 0 && operation.saved))
+                    continue;
+                this.save(operation, transaction, i < this.validated, options);
+            }
+        }
+        catch (error) {
+            this.abort();
+            throw error;
         }
         this.validated = this.writes.length;
         const completions = this.completions;
         if (completions.length > 0)
             this.completions = []; // reset
+        const stagedWrites = this.writes.length;
+        if (completions.length > 0) {
+            this.setCommitPhase(true);
+        }
         return (0, when_ts_1.when)(completions.length > 0 ? Promise.all(completions) : null, () => {
+            if (completions.length > 0)
+                this.setCommitPhase(false);
+            // The transaction can be aborted underneath us while we are parked in the await above — by the
+            // monitor once the commit phase outlives its grace, or through the multi-store poison chain.
+            // abort() cleared the write set and released the handle, so resuming would commit nothing and
+            // resolve as SUCCESS: the caller is told its write landed when it was dropped, and a write
+            // carrying a blob is left holding an instance whose file was unlinked (issue #2062).
+            if (this.timedOut)
+                throw transactionOpenTooLongError();
+            if (stagedWrites > 0 && this.writes.length === 0 && this.open === exports.TRANSACTION_STATE.CLOSED)
+                throw new hdbError_ts_1.ServerError('Transaction was aborted while its commit was waiting on pre-commit work', 500);
             if (this.writes.length > this.validated) {
                 // check just in case we got any more transactions while we were waiting, if so just recursively continue to finish the additional writes now
                 return this.commit(options);
@@ -980,8 +1096,6 @@ class DatabaseTransaction {
                 if (transaction) {
                     this.writes = this.writes.filter((write) => write); // filter out removed entries
                     if (this.writes.length > 0) {
-                        // Commit retries can construct fresh ranges on this live handle after read ownership ends.
-                        readTransactionOwners.delete(transaction);
                         // The transaction was created with coordinatedRetry:true (see
                         // getReadTxn), so commit() can resolve to RETRY_NOW_VALUE. That
                         // sentinel (a number) is why commitResolution is typed
@@ -1020,14 +1134,22 @@ class DatabaseTransaction {
                 // claimed this per-database transaction in txnForContext and so can name the wrong
                 // table when a transaction spans more than one table in the same database.
                 trackOutstandingCommit(commitResolution, this.writes[0]?.store, this.startedFrom, transaction);
+                // Every retry round and every chained store re-enters here and must inherit the chain
+                // root's clock rather than restart it, so only the first submission stamps.
+                const chainRoot = this.root ?? this;
+                if (chainRoot.commitStartedAt == null)
+                    chainRoot.commitStartedAt = performance.now();
                 const completions = [];
-                return commitResolution.then((commitResult) => {
+                const commitOutcome = commitResolution.then((commitResult) => {
                     if (commitResult === RETRY_NOW_VALUE) {
                         this.retries++;
                         harperLogger.debug?.('coordinated retry', transaction.id, this.retries);
                         // Mark this specific native transaction as a retry so RocksTransactionLogStore
                         // skips re-writing its already-staged txn-log entries (#2).
                         transaction.isRetry = true;
+                        const pastBudget = this.elapsedPastCommitBudget();
+                        if (pastBudget)
+                            this.abandonCommitAfterDeadline(transaction, pastBudget);
                         // Mirror the ERR_BUSY cap/warn policy: non-sourceApply transactions abort
                         // at MAX_RETRIES; sourceApply transactions keep retrying with periodic warn.
                         if (this.retries > MAX_RETRIES) {
@@ -1080,9 +1202,12 @@ class DatabaseTransaction {
                         if (write?.savedBlobs && (write.skipped || (write.superseded && !write.blobsAuditReferenced)))
                             (0, blob_ts_1.cleanupUnusedBlobs)(write.savedBlobs, (0, blob_ts_1.collectRetainedFileIds)(write.store.getEntry(write.key)?.value));
                     }
+                    if (this.recordLocks)
+                        this.noteCommittedLockVersions();
                     // now reset transactions tracking; this transaction be reused and committed again
                     this.retries = 0; // reset per-native-transaction retry counter so a reused DatabaseTransaction's next batch starts fresh
                     this.clearWrites();
+                    this.releaseRecordLocks();
                     if (options.doneWriting)
                         this.endScopeOwnership();
                     this.releaseContext(!!options.doneWriting);
@@ -1134,6 +1259,11 @@ class DatabaseTransaction {
                         // premature publish, and no fresh-transaction replay that would drop the entry.
                         // Mark the native transaction as a retry so RocksTransactionLogStore skips re-staging entries.
                         transaction.isRetry = true;
+                        // Before the backoff gate below: the budget can already be spent on the first
+                        // retry when an earlier store in this chain consumed it.
+                        const pastBudget = this.elapsedPastCommitBudget();
+                        if (pastBudget)
+                            this.abandonCommitAfterDeadline(transaction, pastBudget);
                         if (this.retries > 2) {
                             // Transactions applying data from a canonical source of truth (replication peer or
                             // external caching source) must never drop a write on a transient conflict: there is no
@@ -1178,24 +1308,43 @@ class DatabaseTransaction {
                         // finished and its durability is unknown, so ownership goes with it.
                         this.endScopeOwnership();
                         this.releaseContext(!!options.doneWriting);
+                        this.releaseRecordLocks();
+                        this.timestamp = 0;
                         throw error;
                     }
                 });
+                // `commitOutcome` settles when the LOGICAL commit ends — its success branch awaits the
+                // chained stores' own commits — so releasing here covers every terminal exit (success,
+                // retry exhaustion, abandonment, terminal failure) in one place rather than five.
+                // Released through the RETURNED promise rather than a second subscriber on
+                // `commitOutcome`: a subscriber would mark a dropped commit rejection as handled and
+                // silence the unhandled-rejection that surfaces it.
+                return commitOutcome.then((resolution) => {
+                    chainRoot.commitStartedAt = undefined;
+                    return resolution;
+                }, (error) => {
+                    chainRoot.commitStartedAt = undefined;
+                    throw error;
+                });
             }
             for (const write of this.writes) {
                 if (write?.savedBlobs && (write.skipped || (write.superseded && !write.blobsAuditReferenced)))
                     (0, blob_ts_1.cleanupUnusedBlobs)(write.savedBlobs, (0, blob_ts_1.collectRetainedFileIds)(write.store.getEntry(write.key)?.value));
             }
+            if (this.recordLocks)
+                this.noteCommittedLockVersions();
             this.clearWrites();
+            this.releaseRecordLocks();
             if (options.doneWriting)
                 this.endScopeOwnership();
             this.releaseContext(!!options.doneWriting);
             const txnResolution = {
                 txnTime: this.timestamp,
             };
+            this.timestamp = 0; // reset like the async path (~1279) so stale lock stamps don't persist
             if (this.next) {
                 // now run any other transactions
-                options.timestamp = this.timestamp;
+                options.timestamp = txnResolution.txnTime;
                 // as above: the next store must not inherit this store's explicit native transaction
                 let nextResolution;
                 try {
@@ -1214,7 +1363,7 @@ class DatabaseTransaction {
                         // scope resumable after a partially failed mid-scope commit.
                         this.completeMidScopeCommit(options);
                         return {
-                            txnTime: this.timestamp,
+                            txnTime: txnResolution.txnTime,
                             next: nextResolution,
                         };
                     }, (error) => {
@@ -1230,6 +1379,7 @@ class DatabaseTransaction {
             this.completeMidScopeCommit(options);
             return txnResolution;
         }, (error) => {
+            this.setCommitPhase(false);
             this.abort();
             throw error;
         });
@@ -1277,6 +1427,7 @@ class DatabaseTransaction {
         if (this.transaction)
             this.releaseReadTxn();
         this.open = exports.TRANSACTION_STATE.CLOSED;
+        this.timestamp = 0; // a lock stamp pinned for this write set must not leak into the next
         this.drainCompletions();
         try {
             for (const write of this.writes) {
@@ -1287,6 +1438,7 @@ class DatabaseTransaction {
         finally {
             this.endScopeOwnership(); // the scope is over; nothing may rotate this instance again
             this.clearWrites();
+            this.releaseRecordLocks();
             // A timeout-poisoned abort (abortDueToTimeout()) is the one abort that is NOT "reuse-free":
             // Resource.ts's dispatcher deliberately keeps joining a `timedOut` transaction (instead of
             // starting a fresh one) so the rest of the logical operation fails atomically via the
@@ -1306,6 +1458,56 @@ class DatabaseTransaction {
             }
         }
     }
+    /** How long this logical commit may keep retrying: the thread-wide queue limit, or its own larger explicit budget. */
+    commitConflictBudget() {
+        return Math.max(MAX_OUTSTANDING_TXN_DURATION, (this.root ?? this).timeoutBudget || 0);
+    }
+    /**
+     * How long this logical commit has been retrying once it is past its budget, else 0. rocksdb-js
+     * returns control from a parked commit every `ROCKSDB_JS_PARK_TIMEOUT_MS` even when the intent
+     * holder never releases, so without this bound a request-path commit retries the attempt cap out
+     * — minutes past the queue limit an operator configured (issue #2450).
+     *
+     * Source-applied writes are exempt for the same reason they are exempt from the attempt cap:
+     * there is no resubscribe/sequence-resume path, so dropping one permanently diverges this node.
+     */
+    elapsedPastCommitBudget() {
+        if (this.sourceApply)
+            return 0;
+        const startedAt = (this.root ?? this).commitStartedAt;
+        if (startedAt == null)
+            return 0;
+        const elapsed = performance.now() - startedAt;
+        return elapsed > this.commitConflictBudget() ? elapsed : 0;
+    }
+    /**
+     * Abandon a logical commit that stayed in write-intent conflict past its budget. Cleanup is
+     * retry exhaustion's, so no link leaks a native handle or read snapshot; the error is distinct
+     * because the condition is distinct — every attempt reported transient contention, so a later
+     * request can succeed once the holder releases, which the generic exhaustion 500 does not say.
+     *
+     * Only a chain root may report `retryable`: a link commits solely from its predecessor's success
+     * handler, so anywhere else in the chain an earlier store has already landed durable audit
+     * entries and hooks that a replayed request would run twice. A head whose scope already rotated
+     * through a mid-scope commit is in the same position.
+     */
+    abandonCommitAfterDeadline(headTransaction, elapsedMs) {
+        const elapsed = Math.round(elapsedMs);
+        const budget = this.commitConflictBudget();
+        const retryable = !this.root && !this.snapshotFree;
+        if (allowStuckCommitLog('abandon', performance.now())) {
+            harperLogger.error(`Abandoning a write transaction: its commit has been in write-intent conflict for ${elapsed}ms ` +
+                `(exceeds the ${budget}ms limit) across ${this.retries} retries, ` +
+                describeCommitIdentity(this.writes[0]?.store, this.startedFrom, headTransaction) +
+                `.` +
+                (0, longLivedTransactions_ts_1.describeHolderCandidates)(this.writes[0]?.store?.rootStore?.path, headTransaction?.id) +
+                ` Another transaction holds a conflicting write intent and has not completed; the request is ` +
+                `failed with a 503${retryable ? '' : ' (not retryable — an earlier store in this transaction already committed)'} ` +
+                `rather than waiting further.`);
+        }
+        this.abortChainAfterRetries(headTransaction);
+        throw new hdbError_ts_1.TransactionCommitConflictTimeoutError(`Commit was in conflict with ongoing writes for ${elapsed}ms, exceeding the ${budget}ms limit; transaction abandoned after ${this.retries} retries`, retryable);
+    }
     /**
      * Give up on a chain of linked transactions after exhausting conflict retries: poison every link
      * first, then abort each link's native transaction and release its DatabaseTransaction-level
@@ -1401,12 +1603,7 @@ class DatabaseTransaction {
             // throws. abort() rather than the native abort alone: it reclaims blobs a replayed write
             // staged.
             this.detachOwnedTransaction();
-            try {
-                transaction?.abort();
-            }
-            catch (abortError) {
-                harperLogger.debug?.('aborting a transaction whose synchronous commit failed', abortError);
-            }
+            abortNativeTransaction(transaction, 'aborting a transaction whose synchronous commit failed');
             try {
                 this.abort();
             }
@@ -1425,6 +1622,18 @@ class DatabaseTransaction {
     }
 }
 exports.DatabaseTransaction = DatabaseTransaction;
+function shouldSpareCommitPhase(txn, checkedCommitPhaseChains) {
+    if (!txn.committing)
+        return false;
+    if (txn.sourceApply || txn.isReplay)
+        return true;
+    const commitChainHead = txn.commitChainHead ?? txn;
+    if (!checkedCommitPhaseChains.has(commitChainHead)) {
+        checkedCommitPhaseChains.add(commitChainHead);
+        commitChainHead.commitPhaseTicks++;
+    }
+    return commitChainHead.commitPhaseTicks <= exports.COMMIT_PHASE_GRACE;
+}
 class ImmediateTransaction extends DatabaseTransaction {
     isCommitting = false;
     saveCommits = true;
@@ -1433,18 +1642,50 @@ class ImmediateTransaction extends DatabaseTransaction {
         this.db = db;
     }
     save(...args) {
-        const transaction = args[0];
+        const operation = args[0]; // the staged write, not a transaction — commit() re-enters here with it
         if (this.isCommitting) {
             // if we are in the commit, do the save and force a reload so we get a read within the transaction
-            super.save(transaction, null, true);
+            super.save(operation, null, true);
         }
         else {
             this.isCommitting = true;
-            return (0, when_ts_1.when)(this.commit(), () => {
+            // A synchronous throw from commit() (e.g. a 409 from an expired lock handle) would
+            // otherwise leave isCommitting latched at true, causing every subsequent save() in this
+            // context to take the fire-and-forget if-branch and silently drop writes.
+            let commitResult;
+            try {
+                commitResult = this.commit();
+            }
+            catch (err) {
                 this.isCommitting = false;
+                throw err;
+            }
+            return (0, when_ts_1.when)(commitResult, () => {
+                this.isCommitting = false;
+            }, (err) => {
+                // Async rejection (e.g. a rejected rocksdb commit promise) must also clear the
+                // latch; without this, every subsequent save() silently fire-and-forgets.
+                this.isCommitting = false;
+                throw err;
             });
         }
     }
+    // Without an explicit transaction() a live lock is released only by unlock() or its lease, never
+    // by the per-write commit. Expired handles are pruned so a reused context does not retain every key.
+    releaseRecordLocks() {
+        const recordLocks = this.recordLocks;
+        if (!recordLocks)
+            return;
+        for (const [store, storeMap] of recordLocks) {
+            for (const [keyId, handle] of storeMap)
+                if (handle.released)
+                    storeMap.delete(keyId);
+            if (storeMap.size === 0)
+                recordLocks.delete(store);
+        }
+        if (recordLocks.size === 0)
+            this.recordLocks = undefined;
+    }
     // @ts-expect-error accessor overriding property
     get timestamp() {
         return this._timestamp || (this._timestamp = (0, commonUtility_ts_1.getNextMonotonicTime)());
@@ -1517,6 +1758,7 @@ function isJoinableScope(transaction) {
     return transaction?.open === exports.TRANSACTION_STATE.OPEN && !transaction.saveCommits;
 }
 let timer;
+let monitorTick = 0;
 /**
  * True when a link other than `txn` in the same multi-store chain was written recently enough to
  * still be active — i.e. its `writeTimeout` (set only by addWrite, see the field comment) hasn't
@@ -1536,22 +1778,86 @@ function chainStillActive(txn) {
     }
     return false;
 }
+/**
+ * Name a chain link still holding its native handle past the reporting threshold, and why the
+ * branches below are not reaping it (harper#2471). Attribution only: it reads the recency clocks
+ * those branches maintain rather than calling `chainStillActive`, which would decay `writeTimeout`
+ * as a side effect, and it runs before them so it cannot reorder them.
+ */
+function reportLongLivedLink(link, thresholdMs, now, reportBudget) {
+    if (!link.transaction || link.handleOpenedAt === 0)
+        return;
+    const ageMs = now - link.handleOpenedAt;
+    if (ageMs < thresholdMs)
+        return;
+    const states = [];
+    if (link.sourceApply)
+        states.push('source-apply');
+    if (link.isReplay)
+        states.push('replay');
+    if (link.committing)
+        states.push('commit-phase');
+    if (link.open === exports.TRANSACTION_STATE.CLOSED)
+        states.push('closed-with-open-iterators');
+    if (monitorTick - link.writeTick <= 1)
+        states.push('active');
+    if (states.length === 0)
+        states.push('over-limit');
+    (0, longLivedTransactions_ts_1.reportLongLivedHolder)({
+        databasePath: link.db?.rootStore?.path,
+        nativeId: link.transaction.id,
+        openedAt: link.handleOpenedAt,
+        ageMs,
+        databaseName: link.db?.rootStore?.databaseName,
+        tableName: link.db?.name,
+        countPendingWrites: () => {
+            let pendingWrites = 0;
+            for (const write of link.writes)
+                if (write)
+                    pendingWrites++;
+            return pendingWrites;
+        },
+        states,
+        timeoutBudget: link.timeoutBudget,
+        startedFrom: link.startedFrom,
+    }, reportBudget);
+}
+/**
+ * Report every link of this logical transaction that holds its own native handle. A blind write to a
+ * second database attaches a handle to a `.next` link while only the root is supervised, so reporting
+ * the entry alone names an id the sweep's line can never be joined to when the child is the holder.
+ * Links that read are in `trackedTxns` and get their own visit from the tick.
+ */
+function reportIfLongLived(txn, thresholdMs, now, reportBudget) {
+    if (thresholdMs === 0)
+        return;
+    for (let link = txn; link; link = link.next) {
+        if (link !== txn && trackedTxns.has(link))
+            break;
+        reportLongLivedLink(link, thresholdMs, now, reportBudget);
+    }
+}
 function startMonitoringTxns() {
     timer = setInterval(function () {
+        monitorTick++;
+        const checkedCommitPhaseChains = new Set();
+        const reportThresholdMs = (0, longLivedTransactions_ts_1.getReportThresholdMs)();
+        (0, longLivedTransactions_ts_1.rebaseLongLivedTransactionReports)(reportThresholdMs);
+        const reportBudget = (0, longLivedTransactions_ts_1.createLongLivedHolderReportBudget)(reportThresholdMs);
+        const reportNow = performance.now();
         // Both registries, in sequence rather than as a union: a root can be in each (it read, and a
         // later link blind-wrote), and the membership check is cheaper than allocating per tick.
         for (const txn of trackedTxns)
-            monitorTransaction(txn);
+            monitorTransaction(txn, checkedCommitPhaseChains, reportThresholdMs, reportNow, reportBudget);
         for (const txn of supervisedWriteRoots)
             if (!trackedTxns.has(txn))
-                monitorTransaction(txn);
+                monitorTransaction(txn, checkedCommitPhaseChains, reportThresholdMs, reportNow, reportBudget);
+        (0, longLivedTransactions_ts_1.finishLongLivedHolderReports)(reportBudget);
     }, txnExpiration).unref();
-    function monitorTransaction(txn) {
-        if (txn.rangeReadActive) {
-            txn.rangeReadActive = false;
-            txn.renewReadTimeout();
-        }
+    function monitorTransaction(txn, checkedCommitPhaseChains, reportThresholdMs, reportNow, reportBudget) {
+        reportIfLongLived(txn, reportThresholdMs, reportNow, reportBudget);
         {
+            const commitChainHead = txn.commitChainHead ?? txn;
             // Decay write recency once per tick for every tracked link, independent of the `timeout`
             // branches below — a tracked link that keeps its own idle limit alive by reading must not
             // thereby keep chainStillActive believing it was written recently too.
@@ -1577,6 +1883,21 @@ function startMonitoringTxns() {
                     harperLogger.warn?.(`Read iterators held a committed transaction's snapshot past the open-transaction limit; releasing it, from table: ${txn.db?.name + (url ? ' path: ' + url : '')}`);
                     txn.releaseReadTxn();
                 }
+                else if (shouldSpareCommitPhase(txn, checkedCommitPhaseChains)) {
+                    // Parked in commit()'s pre-commit await — a `before`/`beforeIntermediate` hook, in practice a
+                    // blob's durable file write, which for a multi-tens-of-MB payload legitimately outruns the
+                    // limit. The write set is sealed and the caller is awaiting this commit, so the limit's
+                    // premise (the application is holding a transaction open) does not hold, and poisoning here
+                    // would unlink the blobs the write still references (issue #2062). The grace is bounded
+                    // because the transaction still pins a read snapshot: a source that stalls rather than
+                    // finishing falls through to the abort below once it runs out. A canonical-source apply or
+                    // replay is spared for as long as it takes: neither aborting it (harper-pro#348) nor the
+                    // force-commit below — which would durably commit a record whose blob file is still being
+                    // written — is acceptable, and their blob sources are bounded by the receive-side idle
+                    // watchdog instead.
+                    harperLogger.warn?.(`Transaction has been in its commit phase past the open-transaction limit, waiting on pre-commit work (e.g. a large blob write); letting it complete, from table: ${txn.db?.name + (url ? ' path: ' + url : '')}`, ...(txn.startedFrom ? [`was started from ${txn.startedFrom.resourceName}.${txn.startedFrom.method}`] : []));
+                    txn.timeout = Math.max(txnExpiration, txn.timeoutBudget ?? 0);
+                }
                 else if (txn.hasPendingWrites() && chainStillActive(txn)) {
                     // A later link in the chain was written recently (writes re-arm only the link that
                     // receives them, and a multi-store transaction can be writing database B while this
@@ -1595,7 +1916,7 @@ function startMonitoringTxns() {
                     // those, keep the prior force-commit behavior below.
                     harperLogger.error(`Transaction was open too long and has been aborted after exceeding the open-transaction limit, from table: ${txn.db?.name + (url ? ' path: ' + url : '')}`, ...(txn.startedFrom ? [`was started from ${txn.startedFrom.resourceName}.${txn.startedFrom.method}`] : []), ...(DEBUG_LONG_TXNS ? ['starting stack trace', txn.stackTraces] : []));
                     try {
-                        txn.abortDueToTimeout();
+                        commitChainHead.abortDueToTimeout();
                     }
                     catch (error) {
                         harperLogger.debug?.(`Error aborting timed out transaction: ${error.message}`);
@@ -1634,6 +1955,11 @@ startMonitoringTxns();
 function resetReplayedWritesWarning() {
     replayedWritesWarned = false;
 }
+function setMaxOutstandingTxnDuration(ms) {
+    const previous = MAX_OUTSTANDING_TXN_DURATION;
+    MAX_OUTSTANDING_TXN_DURATION = ms;
+    return previous;
+}
 /** Test seam: whether the monitor supervises this logical transaction for its writes. */
 function isWriteSupervised(txn) {
     return supervisedWriteRoots.has(txn);