@harperfast/harper 5.2.9 → 5.3.0-alpha.1

package/resources/DatabaseTransaction.ts

@@ -1,7 +1,8 @@
 import { cleanupUnusedBlobs, collectRetainedFileIds } from './blob.ts';
 import { Transaction as LMDBTransaction } from 'lmdb';
 import { getNextMonotonicTime } from '../utility/lmdb/commonUtility.ts';
-import { ServerError } from '../utility/errors/hdbError.ts';
+import { ServerError, TransactionCommitConflictTimeoutError } from '../utility/errors/hdbError.ts';
+import { lockNotHeldError, type RecordLockHandle } from './recordLock.ts';
 import * as harperLogger from '../utility/logging/harper_logger.ts';
 import type { Context, Id } from './ResourceInterface.ts';
 import * as envMngr from '../utility/environment/environmentManager.ts';
@@ -14,6 +15,15 @@ const RETRY_NOW_VALUE = constants.RETRY_NOW_VALUE;
 import type { RootDatabaseKind } from './databases.ts';
 import type { Entry } from './RecordEncoder.ts';
 import { toBufferKey } from 'ordered-binary';
+import {
+	createLongLivedHolderReportBudget,
+	describeHolderCandidates,
+	finishLongLivedHolderReports,
+	getReportThresholdMs,
+	rebaseLongLivedTransactionReports,
+	reportLongLivedHolder,
+	type LongLivedHolderReportBudget,
+} from './longLivedTransactions.ts';
 
 const trackedTxns = new Set<DatabaseTransaction>();
 // Read options for a rotated generation's native transactions; shared because they never vary.
@@ -23,7 +33,7 @@ const SNAPSHOT_FREE = Object.freeze({ disableSnapshot: true });
 // is what the read-queue-depth metric counts, while this holds one entry per logical transaction — the
 // chain root — so a chain child can never become its own timeout root (issue #2231).
 const supervisedWriteRoots = new Set<DatabaseTransaction>();
-const MAX_OUTSTANDING_TXN_DURATION = convertToMS(envMngr.get(CONFIG_PARAMS.STORAGE_MAXTRANSACTIONQUEUETIME)) || 45000; // Allow write transactions to be queued for up to 45 seconds before we start rejecting them
+let MAX_OUTSTANDING_TXN_DURATION = convertToMS(envMngr.get(CONFIG_PARAMS.STORAGE_MAXTRANSACTIONQUEUETIME)) || 45000; // Allow write transactions to be queued for up to 45 seconds before we start rejecting them
 const DEBUG_LONG_TXNS = envMngr.get(CONFIG_PARAMS.STORAGE_DEBUGLONGTRANSACTIONS);
 export const TRANSACTION_STATE = {
 	CLOSED: 0, // the transaction has been committed or aborted and can no longer be used for writes (if read txn is active, it can be used for reads)
@@ -35,6 +45,12 @@ export const TRANSACTION_STATE = {
 	LINGERING: 2,
 };
 const MAX_RETRIES = 40;
+// Over-limit monitor ticks a transaction parked in its commit phase is spared before it is aborted
+// like any other over-time transaction. Sparing re-arms `timeout`, so each of these costs two
+// monitor intervals — roughly 10 minutes at the 30s default. Generous, so a legitimately large blob
+// write finishes, but bounded, so a pre-commit source that stalls instead of finishing can't pin its
+// read snapshot forever (issue #2062).
+export const COMMIT_PHASE_GRACE = 10;
 // Cap the per-retry backoff so replication-applied transactions, which retry conflicts without a
 // cap (see the commit rejection handler), don't grow the delay unbounded.
 const MAX_RETRY_DELAY_MS = 1000;
@@ -68,7 +84,33 @@ let outstandingCommitCount = 0;
 // the whole thread, regardless of how many distinct commits individually cross the threshold — see
 // the comment at the log site for why a per-commit-only dedup isn't enough under sustained overload.
 const OVERLOAD_LOG_MIN_INTERVAL_MS = 1000;
-let lastOverloadLogAt = -Infinity;
+// One cooldown per reporting site, not one shared: `shed` fires on every bystander write during a
+// wedge and would starve `abandon`, which names a different transaction and is the only server-side
+// record of why a request was failed. Mutates only when it grants, so a caller with its own
+// suppression as well (checkOverloaded's per-node `logged`) must test that first.
+const lastStuckCommitLogAt = { shed: -Infinity, abandon: -Infinity };
+function allowStuckCommitLog(site: 'shed' | 'abandon', now: number): boolean {
+	if (now - lastStuckCommitLogAt[site] <= OVERLOAD_LOG_MIN_INTERVAL_MS) return false;
+	lastStuckCommitLogAt[site] = now;
+	return true;
+}
+
+// Which database/table, which native transaction, and which request a stuck commit belongs to —
+// without it a wedge gives no indication of what to investigate (harper#2001).
+function describeCommitIdentity(
+	store: any,
+	startedFrom: { resourceName: string; method: string } | undefined,
+	nativeTransaction: any
+): string {
+	const nativeTransactionId = nativeTransaction?.id;
+	return (
+		`from table: ${store?.rootStore?.databaseName ?? '?'}.${store?.name ?? '?'}` +
+		(nativeTransactionId !== undefined ? ` (transaction ${nativeTransactionId})` : '') +
+		(startedFrom?.resourceName
+			? `, started from ${startedFrom.resourceName}${startedFrom.method ? '.' + startedFrom.method : ''}`
+			: '')
+	);
+}
 
 // Track a submitted commit until it settles. Every attempt is tracked unconditionally: a
 // coordinated retry round and a chained second-store commit are both issued from inside the
@@ -145,6 +187,20 @@ export function getOutstandingCommits(): { count: number; oldestAgeMs: number |
 // about the caller's pattern, not the individual commit.
 let replayedWritesWarned = false;
 
+/**
+ * Abort a detached native handle. RocksTransaction.abort() throws on one that was already
+ * committed or aborted, and every caller is a cleanup path whose own callers have no handler — a
+ * throw there would abandon the rest of the cleanup.
+ */
+function abortNativeTransaction(transaction: RocksTransaction | null | undefined, context: string): void {
+	if (transaction == null) return;
+	try {
+		transaction.abort();
+	} catch (error) {
+		harperLogger.debug?.(context, error);
+	}
+}
+
 // The analytics module registers a recorder here at load (dependency inversion, mirroring
 // `replicationConfirmation` below) so the storage layer doesn't statically import the analytics/server
 // modules. Unset until analytics loads, and when analytics is disabled the recorder call is cheap.
@@ -265,6 +321,9 @@ export type TransactionWrite = {
 	// overload accounting, the replay marker and a no-op write's removal all belong to the committer.
 	validate?: (txnTime: number, committedBy: DatabaseTransaction) => void;
 	fullUpdate?: boolean;
+	// The origin record version carried by an applied or replayed write. Bound it by the origin's
+	// transaction-log key so malformed or historical overloaded values cannot move ordering past the write.
+	recordVersion?: number;
 	saved?: boolean;
 	deferSave?: boolean;
 	skipReplicationConfirmation?: boolean;
@@ -303,8 +362,31 @@ export type TransactionWrite = {
 	// this write appended an audit entry, which references its saved blobs — they then belong to the
 	// audit trail (audit pruning deletes them), so the superseded-write cleanup must leave them alone
 	blobsAuditReferenced?: boolean;
+	// Lock handle set by Table._writeUpdate when the write is staged through a held record lock;
+	// used in DatabaseTransaction.save() to assign and track that handle's write versions.
+	lockHandle?: RecordLockHandle;
+	// Per-operation holder version, set once on first save() and reused on retry so that
+	// ImmediateTransaction's sequential immediateCommit saves don't collide: each write carries its
+	// own stamp independently of this.timestamp, which may not reset to 0 between saves.
+	lockStamp?: number;
+	// Version staged by a write that actually changed the record this round. It advances any
+	// same-key held handle's floor only after the native transaction commits successfully.
+	appliedRecordVersion?: number;
+	// Present only while a transaction owns record locks, keeping bookkeeping off ordinary writes.
+	trackRecordVersion?: boolean;
+	// Set by a table commit handler only when this retry round staged a record change.
+	recordVersionApplied?: boolean;
+	// Set by DatabaseTransaction.save() on the immediateCommit path: the Promise returned by the
+	// inner this.commit({ transaction }) call. The ImmediateTransaction outer commit resolves before
+	// this settles (fire-and-forget from the if-branch), so Table.save()'s lock-writable path awaits
+	// it to ensure the write is durable before resolving to the caller.
+	innerCommit?: MaybePromise<CommitResolution>;
 };
 
+export function getAppliedWriteVersion(recordVersion: number | undefined, txnLogKey: number): number {
+	return recordVersion == null ? txnLogKey : Math.min(recordVersion, txnLogKey);
+}
+
 /**
  * The state a preceding write in this transaction left for `operation`'s key, or undefined if this
  * is the first write to it. Within a transaction the writes are ordered by program order, and
@@ -364,7 +446,11 @@ export class DatabaseTransaction implements Transaction {
 	// link with no writes that is being read in a loop would otherwise masquerade as write activity and
 	// keep a write-holding head immortal.
 	declare writeTimeout: number;
+	writeTick = -1;
 	timeoutBudget = 0;
+	// Initialized rather than `declare`d so the class shape stays monomorphic on the write path. One clock
+	// read per handle is what lets the monitor age a handle without calling into the registry every tick.
+	handleOpenedAt = 0;
 	// save() only stages here; ImmediateTransaction overrides it to commit, which addWrite must not defer
 	saveCommits = false;
 	// True where save() puts the write into this transaction's native handle, which is what lets a scope
@@ -378,6 +464,13 @@ export class DatabaseTransaction implements Transaction {
 	declare next: DatabaseTransaction;
 	// The head of this multi-store chain, set when the link is created; absent on the head itself.
 	declare root?: DatabaseTransaction;
+	// When this logical commit first reached the storage engine, held on the chain root so every
+	// retry round and every chained store measures ONE elapsed wait (issue #2450). Deliberately not
+	// the per-attempt clock trackOutstandingCommit() keeps: that one drives thread-wide load
+	// shedding and must stay per-attempt, or a long uncapped source-apply retry would 503 every
+	// unrelated request on the thread. Cleared when the logical commit settles, so a reused
+	// transaction's next batch starts on a fresh budget.
+	declare commitStartedAt?: number;
 	// Whether this link is why its chain root is write-supervised (see endWriteSupervision).
 	declare writeSupervised?: boolean;
 	declare stale: boolean;
@@ -418,6 +511,28 @@ export class DatabaseTransaction implements Transaction {
 	// asks to stop reading a pinned snapshot, so re-pinning one for the rest of the scope would take
 	// back what it asked for.
 	snapshotFree = false;
+	// Set while commit() is parked in its pre-commit await (`before`/`beforeIntermediate` completions —
+	// in practice a blob's durable file write). The write set is sealed and the caller is awaiting the
+	// commit, so this is core's own I/O rather than an application holding a transaction open, which is
+	// what the open-transaction limit polices: the monitor spares it for COMMIT_PHASE_GRACE ticks
+	// instead of poisoning it (issue #2062).
+	committing = false;
+	commitPhaseTicks = 0;
+	declare commitChainHead?: DatabaseTransaction;
+	// O(1) lookup in recordLockFor; only lock() handles are registered here (no gate handles).
+	declare recordLocks?: Map<any, Map<unknown, RecordLockHandle>>;
+	// Tracks in-flight acquireRecordKey calls so concurrent lock() calls for the same key in one
+	// link (e.g. Promise.all([T.lock(id), T.lock(id)])) can coalesce rather than self-block.
+	declare pendingLocks?: Map<any, Map<unknown, Promise<RecordLockHandle>>>;
+
+	setCommitPhase(committing: boolean): void {
+		// A commit phase covers the sealed write set across the whole multi-store chain.
+		for (let txn: DatabaseTransaction = this; txn; txn = txn.next) {
+			txn.committing = committing;
+			txn.commitChainHead = committing ? this : undefined;
+			if (committing) txn.commitPhaseTicks = 0;
+		}
+	}
 
 	getReadTxn(disableSnapshot?: boolean): ReadTransaction {
 		this.readTxnRefCount = (this.readTxnRefCount || 0) + 1;
@@ -469,6 +584,7 @@ export class DatabaseTransaction implements Transaction {
 		this.transaction = transaction;
 		this.readTxnsUsed = 1;
 		this.baseReadRefConsumed = false;
+		this.handleOpenedAt = performance.now();
 	}
 
 	/**
@@ -509,6 +625,7 @@ export class DatabaseTransaction implements Transaction {
 		this.transaction = null;
 		this.readTxnsUsed = 0;
 		this.readTxnRefCount = 0;
+		this.handleOpenedAt = 0;
 		return transaction;
 	}
 
@@ -555,11 +672,7 @@ export class DatabaseTransaction implements Transaction {
 	 */
 	releaseReadTxn(): void {
 		const transaction = this.detachOwnedTransaction();
-		try {
-			transaction?.abort();
-		} catch (error) {
-			harperLogger.debug?.('releasing timed-out read transaction', error);
-		}
+		abortNativeTransaction(transaction, 'releasing timed-out read transaction');
 		this.completeDeferredContextRelease();
 	}
 
@@ -632,6 +745,70 @@ export class DatabaseTransaction implements Transaction {
 		}
 	}
 
+	registerRecordLock(handle: RecordLockHandle): void {
+		if (!this.recordLocks) this.recordLocks = new Map();
+		let storeMap = this.recordLocks.get(handle.store);
+		if (!storeMap) this.recordLocks.set(handle.store, (storeMap = new Map()));
+		storeMap.set(handle.keyId, handle);
+	}
+
+	recordLockFor(store: any, keyId: unknown): RecordLockHandle | undefined {
+		const storeMap = this.recordLocks?.get(store);
+		if (!storeMap) return undefined;
+		const h = storeMap.get(keyId);
+		if (!h) return undefined;
+		if (!h.released) return h;
+		// Prune released handles so they don't accumulate; an expired handle checking re-entrancy
+		// would otherwise be seen as the holder and incorrectly granted access.
+		storeMap.delete(keyId);
+		return undefined;
+	}
+
+	unregisterRecordLock(handle: RecordLockHandle): void {
+		const storeMap = this.recordLocks?.get(handle.store);
+		if (storeMap?.get(handle.keyId) === handle) storeMap.delete(handle.keyId);
+	}
+
+	registerPendingLock(store: any, keyId: unknown, pending: Promise<RecordLockHandle>): void {
+		if (!this.pendingLocks) this.pendingLocks = new Map();
+		let storeMap = this.pendingLocks.get(store);
+		if (!storeMap) this.pendingLocks.set(store, (storeMap = new Map()));
+		storeMap.set(keyId, pending);
+	}
+
+	pendingLockFor(store: any, keyId: unknown): Promise<RecordLockHandle> | undefined {
+		return this.pendingLocks?.get(store)?.get(keyId);
+	}
+
+	unregisterPendingLock(store: any, keyId: unknown): void {
+		this.pendingLocks?.get(store)?.delete(keyId);
+	}
+
+	/** Release every transaction-scoped lock() handle this link owns. */
+	releaseRecordLocks(): void {
+		const recordLocks = this.recordLocks;
+		if (!recordLocks) return;
+		for (const [store, storeMap] of recordLocks) {
+			for (const [keyId, handle] of storeMap) {
+				if (handle.hold) continue; // hold handles outlive the transaction; released by unlock()
+				handle.release();
+				storeMap.delete(keyId);
+			}
+			if (storeMap.size === 0) recordLocks.delete(store);
+		}
+		if (recordLocks.size === 0) this.recordLocks = undefined;
+	}
+
+	private noteCommittedLockVersions(): void {
+		const recordLocks = this.recordLocks;
+		if (!recordLocks) return;
+		for (const write of this.writes) {
+			if (write?.appliedRecordVersion == null) continue;
+			const handle = write.lockHandle ?? recordLocks.get(write.store)?.get(writeKeyId(write.key));
+			if (handle && !handle.released) handle.noteHolderVersion(write.appliedRecordVersion);
+		}
+	}
+
 	/**
 	 * Discard the staged write set (committed or aborted); the per-key chain must go with it so a
 	 * reused transaction never bases a write on a previous batch's staged state.
@@ -691,26 +868,28 @@ export class DatabaseTransaction implements Transaction {
 			// original per-request log was fixed to avoid, just shifted from per-request to per-commit.
 			// A commit skipped by the cooldown is NOT marked `logged`, so it still gets a log later if
 			// it's still the oldest once the cooldown clears, rather than going silent forever.
-			if (!oldestOutstandingCommit.logged && now - lastOverloadLogAt > OVERLOAD_LOG_MIN_INTERVAL_MS) {
+			if (!oldestOutstandingCommit.logged && allowStuckCommitLog('shed', now)) {
 				// Log once per stuck commit (not once per rejected request, harper#2001): a wedged
 				// thread otherwise logs nothing at all server-side while rejecting every write with a
 				// 503, which was the single biggest obstacle to root-causing a recurrence. The flag lives
 				// on the node itself, so if THIS commit settles while still over the limit and a
 				// different one is now oldest, that one logs too instead of staying silent forever.
 				oldestOutstandingCommit.logged = true;
-				lastOverloadLogAt = now;
-				const nativeTransactionId = oldestOutstandingCommit.nativeTransaction?.id;
-				const store = oldestOutstandingCommit.store;
-				const startedFrom = oldestOutstandingCommit.startedFrom;
 				harperLogger.error(
 					`Rejecting writes on this thread: a commit has been outstanding for ` +
 						`${Math.round(now - oldestOutstandingCommit.start)}ms (exceeds the ` +
-						`${MAX_OUTSTANDING_TXN_DURATION}ms limit), from table: ${store?.rootStore?.databaseName ?? '?'}.${store?.name ?? '?'}` +
-						(nativeTransactionId !== undefined ? ` (transaction ${nativeTransactionId})` : '') +
-						(startedFrom?.resourceName
-							? `, started from ${startedFrom.resourceName}${startedFrom.method ? '.' + startedFrom.method : ''}`
-							: '') +
-						`. Further record updates and publishes from new application requests on this thread ` +
+						`${MAX_OUTSTANDING_TXN_DURATION}ms limit), ` +
+						describeCommitIdentity(
+							oldestOutstandingCommit.store,
+							oldestOutstandingCommit.startedFrom,
+							oldestOutstandingCommit.nativeTransaction
+						) +
+						`.` +
+						describeHolderCandidates(
+							oldestOutstandingCommit.store?.rootStore?.path,
+							oldestOutstandingCommit.nativeTransaction?.id
+						) +
+						` Further record updates and publishes from new application requests on this thread ` +
 						`will be rejected with 503 until the commit settles or the process is restarted (deletes, ` +
 						`and writes applied from a canonical source, e.g. replication or a caching source, bypass this check).`
 				);
@@ -774,6 +953,7 @@ export class DatabaseTransaction implements Transaction {
 		// Independent write-recency signal for chainStillActive (see the field comment) — reads never
 		// touch this, only writes do.
 		this.writeTimeout = this.timeout;
+		this.writeTick = monitorTick;
 		this.linkWrite(operation);
 		this.writes.push(operation);
 		operation.stagedIn = this;
@@ -804,7 +984,32 @@ export class DatabaseTransaction implements Transaction {
 	}
 
 	save(operation: TransactionWrite, transaction?: RocksTransaction, reloadEntry = false, options?: CommitOptions) {
-		let txnTime = this.timestamp;
+		const lockHandle = operation.lockHandle;
+		// Guard: a write staged through an expired or released lock handle must not land.
+		// The handle's lease timer already unlocked the native key; another holder may have taken it.
+		if (lockHandle && (lockHandle.expired || lockHandle.released)) {
+			// Remove the operation from the staged set so subsequent writes on this context do not
+			// re-throw 409 due to a stale null-saved entry sitting in this.writes.
+			const failedIdx = this.writes.indexOf(operation);
+			if (failedIdx > -1) this.writes[failedIdx] = null;
+			throw lockNotHeldError(lockHandle);
+		}
+		// Lock-write timestamp rules.
+		if (lockHandle) {
+			if (this.open === TRANSACTION_STATE.CLOSED || this.saveCommits) {
+				// CLOSED path (second+ write per ImmediateTransaction cycle) OR the first write in
+				// an ImmediateTransaction (open=OPEN until commit sets it CLOSED, but saveCommits
+				// signals the per-write-commit semantics):  stamp with nextHolderVersion() so that
+				// each sequential save() gets its own monotonically increasing stamp — scoped or hold.
+				// The stamp stays on the operation and is consumed below rather than assigned to
+				// this.timestamp: pinning the link's clock would stamp every OTHER write staged on
+				// the same context before the commit resets it — a concurrent write in the caller's
+				// own Promise.all, or the next operation in a retry/replay save loop — with the
+				// lock's version, which LWW then silently drops against a newer record version.
+				if (!operation.lockStamp) operation.lockStamp = lockHandle.holderVersionCandidate();
+			}
+		}
+		let txnTime = operation.lockStamp ?? this.timestamp;
 		// Only an OPEN transaction accepts new staged writes. After commit, this.transaction may still
 		// be retained for outstanding read iterators; staging into it would silently discard the write
 		// when doneReadTxn() aborts the handle, so such writes commit immediately on a fresh
@@ -849,13 +1054,19 @@ export class DatabaseTransaction implements Transaction {
 			(transaction as RocksTransactionWithRetry).isRetry = true;
 		}
 		if (!txnTime) txnTime = this.timestamp = transaction.getTimestamp();
+		// `txnTime` is this transaction's timestamp — the key its entries take in the per-origin log.
+		// A write applied from elsewhere carries the origin's record version too, and that is what the
+		// record is stored at; the two coincide for every locally-originated write. Gated on the apply
+		// flags so an ordinary write never reads the property (harper#2412).
+		const writeVersion =
+			this.sourceApply || this.isReplay ? getAppliedWriteVersion(operation.recordVersion, txnTime) : txnTime;
 		if (reloadEntry || operation.entry === undefined) {
 			operation.entry = operation.store.getEntry(operation.key, { transaction });
 		}
 		if (!operation.saved) {
 			operation.saved = true;
 			// immediately execute in this transaction
-			if ((operation.validate?.(txnTime, this) as any) === false) {
+			if ((operation.validate?.(writeVersion, this) as any) === false) {
 				operation.commit = () => {}; // noop if we try again
 				return;
 			}
@@ -864,7 +1075,11 @@ export class DatabaseTransaction implements Transaction {
 			result = operation.beforeIntermediate?.() as Promise<void>;
 			if (result?.then) this.stageCompletion(result);
 		}
-		const completion = operation.commit(txnTime, operation.entry, this.retries > 0, transaction) as Promise<void>;
+		if (lockHandle || this.recordLocks) operation.trackRecordVersion = true;
+		if (operation.trackRecordVersion) operation.recordVersionApplied = false;
+		const completion = operation.commit(writeVersion, operation.entry, this.retries > 0, transaction) as Promise<void>;
+		if (operation.trackRecordVersion)
+			operation.appliedRecordVersion = operation.recordVersionApplied ? writeVersion : undefined;
 		if (typeof completion?.then === 'function') this.stageCompletion(completion);
 		// Sticky record that THIS write staged with its audit entry appended (log entries batch on the
 		// native transaction and are durably written by its commit attempt — even a failed one — so
@@ -876,7 +1091,13 @@ export class DatabaseTransaction implements Transaction {
 			operation.appendedAuditEntry = true;
 		}
 		if (immediateCommit) {
-			return this.commit({ ...options, transaction }); // immediately commit if the harper transaction is closed
+			// immediately commit if the harper transaction is closed
+			const innerCommit = this.commit({ ...options, transaction });
+			// Expose on the operation so the lock-writable Table.save() path can await the real
+			// native commit — without this the ImmediateTransaction outer commit resolves before
+			// the native transaction.commit() settles (fire-and-forget from the if-branch).
+			operation.innerCommit = innerCommit;
+			return innerCommit;
 		}
 	}
 
@@ -888,17 +1109,35 @@ export class DatabaseTransaction implements Transaction {
 		// reused across retries — the native layer resets it in place (fresh snapshot) on IsBusy/TryAgain —
 		// but reassigned to a fresh replay transaction when outstanding read iterators retain this.transaction
 		let transaction = options.transaction ?? this.transaction;
-		for (let i = 0; i < this.writes.length; i++) {
-			let operation = this.writes[i];
-			if (!operation || (this.retries === 0 && operation.saved)) continue;
-			this.save(operation, transaction, i < this.validated, options);
+		try {
+			for (let i = 0; i < this.writes.length; i++) {
+				let operation = this.writes[i];
+				if (!operation || (this.retries === 0 && operation.saved)) continue;
+				this.save(operation, transaction, i < this.validated, options);
+			}
+		} catch (error) {
+			this.abort();
+			throw error;
 		}
 		this.validated = this.writes.length;
 		const completions = this.completions;
 		if (completions.length > 0) this.completions = []; // reset
+		const stagedWrites = this.writes.length;
+		if (completions.length > 0) {
+			this.setCommitPhase(true);
+		}
 		return when(
 			completions.length > 0 ? Promise.all(completions) : null,
 			() => {
+				if (completions.length > 0) this.setCommitPhase(false);
+				// The transaction can be aborted underneath us while we are parked in the await above — by the
+				// monitor once the commit phase outlives its grace, or through the multi-store poison chain.
+				// abort() cleared the write set and released the handle, so resuming would commit nothing and
+				// resolve as SUCCESS: the caller is told its write landed when it was dropped, and a write
+				// carrying a blob is left holding an instance whose file was unlinked (issue #2062).
+				if (this.timedOut) throw transactionOpenTooLongError();
+				if (stagedWrites > 0 && this.writes.length === 0 && this.open === TRANSACTION_STATE.CLOSED)
+					throw new ServerError('Transaction was aborted while its commit was waiting on pre-commit work', 500);
 				if (this.writes.length > this.validated) {
 					// check just in case we got any more transactions while we were waiting, if so just recursively continue to finish the additional writes now
 					return this.commit(options);
@@ -1032,8 +1271,12 @@ export class DatabaseTransaction implements Transaction {
 					// claimed this per-database transaction in txnForContext and so can name the wrong
 					// table when a transaction spans more than one table in the same database.
 					trackOutstandingCommit(commitResolution, this.writes[0]?.store, this.startedFrom, transaction);
+					// Every retry round and every chained store re-enters here and must inherit the chain
+					// root's clock rather than restart it, so only the first submission stamps.
+					const chainRoot = this.root ?? this;
+					if (chainRoot.commitStartedAt == null) chainRoot.commitStartedAt = performance.now();
 					const completions = [];
-					return commitResolution.then(
+					const commitOutcome = commitResolution.then(
 						(commitResult) => {
 							if (commitResult === RETRY_NOW_VALUE) {
 								this.retries++;
@@ -1041,6 +1284,8 @@ export class DatabaseTransaction implements Transaction {
 								// Mark this specific native transaction as a retry so RocksTransactionLogStore
 								// skips re-writing its already-staged txn-log entries (#2).
 								(transaction as RocksTransactionWithRetry).isRetry = true;
+								const pastBudget = this.elapsedPastCommitBudget();
+								if (pastBudget) this.abandonCommitAfterDeadline(transaction, pastBudget);
 								// Mirror the ERR_BUSY cap/warn policy: non-sourceApply transactions abort
 								// at MAX_RETRIES; sourceApply transactions keep retrying with periodic warn.
 								if (this.retries > MAX_RETRIES) {
@@ -1100,9 +1345,11 @@ export class DatabaseTransaction implements Transaction {
 								if (write?.savedBlobs && (write.skipped || (write.superseded && !write.blobsAuditReferenced)))
 									cleanupUnusedBlobs(write.savedBlobs, collectRetainedFileIds(write.store.getEntry(write.key)?.value));
 							}
+							if (this.recordLocks) this.noteCommittedLockVersions();
 							// now reset transactions tracking; this transaction be reused and committed again
 							this.retries = 0; // reset per-native-transaction retry counter so a reused DatabaseTransaction's next batch starts fresh
 							this.clearWrites();
+							this.releaseRecordLocks();
 							if (options.doneWriting) this.endScopeOwnership();
 							this.releaseContext(!!options.doneWriting);
 							let txnTime = this.timestamp;
@@ -1157,6 +1404,10 @@ export class DatabaseTransaction implements Transaction {
 								// premature publish, and no fresh-transaction replay that would drop the entry.
 								// Mark the native transaction as a retry so RocksTransactionLogStore skips re-staging entries.
 								(transaction as RocksTransactionWithRetry).isRetry = true;
+								// Before the backoff gate below: the budget can already be spent on the first
+								// retry when an earlier store in this chain consumed it.
+								const pastBudget = this.elapsedPastCommitBudget();
+								if (pastBudget) this.abandonCommitAfterDeadline(transaction, pastBudget);
 								if (this.retries > 2) {
 									// Transactions applying data from a canonical source of truth (replication peer or
 									// external caching source) must never drop a write on a transient conflict: there is no
@@ -1205,24 +1456,45 @@ export class DatabaseTransaction implements Transaction {
 								// finished and its durability is unknown, so ownership goes with it.
 								this.endScopeOwnership();
 								this.releaseContext(!!options.doneWriting);
+								this.releaseRecordLocks();
+								this.timestamp = 0;
 								throw error;
 							}
 						}
 					);
+					// `commitOutcome` settles when the LOGICAL commit ends — its success branch awaits the
+					// chained stores' own commits — so releasing here covers every terminal exit (success,
+					// retry exhaustion, abandonment, terminal failure) in one place rather than five.
+					// Released through the RETURNED promise rather than a second subscriber on
+					// `commitOutcome`: a subscriber would mark a dropped commit rejection as handled and
+					// silence the unhandled-rejection that surfaces it.
+					return commitOutcome.then(
+						(resolution) => {
+							chainRoot.commitStartedAt = undefined;
+							return resolution;
+						},
+						(error) => {
+							chainRoot.commitStartedAt = undefined;
+							throw error;
+						}
+					);
 				}
 				for (const write of this.writes) {
 					if (write?.savedBlobs && (write.skipped || (write.superseded && !write.blobsAuditReferenced)))
 						cleanupUnusedBlobs(write.savedBlobs, collectRetainedFileIds(write.store.getEntry(write.key)?.value));
 				}
+				if (this.recordLocks) this.noteCommittedLockVersions();
 				this.clearWrites();
+				this.releaseRecordLocks();
 				if (options.doneWriting) this.endScopeOwnership();
 				this.releaseContext(!!options.doneWriting);
 				const txnResolution: CommitResolution = {
 					txnTime: this.timestamp,
 				};
+				this.timestamp = 0; // reset like the async path (~1279) so stale lock stamps don't persist
 				if (this.next) {
 					// now run any other transactions
-					options.timestamp = this.timestamp;
+					options.timestamp = txnResolution.txnTime;
 					// as above: the next store must not inherit this store's explicit native transaction
 					let nextResolution;
 					try {
@@ -1241,7 +1513,7 @@ export class DatabaseTransaction implements Transaction {
 								// scope resumable after a partially failed mid-scope commit.
 								this.completeMidScopeCommit(options);
 								return {
-									txnTime: this.timestamp,
+									txnTime: txnResolution.txnTime,
 									next: nextResolution,
 								};
 							},
@@ -1260,6 +1532,7 @@ export class DatabaseTransaction implements Transaction {
 				return txnResolution;
 			},
 			(error) => {
+				this.setCommitPhase(false);
 				this.abort();
 				throw error;
 			}
@@ -1308,6 +1581,7 @@ export class DatabaseTransaction implements Transaction {
 		// Defensively release any native handle whose reference bookkeeping was already consumed.
 		if (this.transaction) this.releaseReadTxn();
 		this.open = TRANSACTION_STATE.CLOSED;
+		this.timestamp = 0; // a lock stamp pinned for this write set must not leak into the next
 		this.drainCompletions();
 		try {
 			for (const write of this.writes) {
@@ -1317,6 +1591,7 @@ export class DatabaseTransaction implements Transaction {
 		} finally {
 			this.endScopeOwnership(); // the scope is over; nothing may rotate this instance again
 			this.clearWrites();
+			this.releaseRecordLocks();
 			// A timeout-poisoned abort (abortDueToTimeout()) is the one abort that is NOT "reuse-free":
 			// Resource.ts's dispatcher deliberately keeps joining a `timedOut` transaction (instead of
 			// starting a fresh one) so the rest of the logical operation fails atomically via the
@@ -1335,6 +1610,62 @@ export class DatabaseTransaction implements Transaction {
 			}
 		}
 	}
+	/** How long this logical commit may keep retrying: the thread-wide queue limit, or its own larger explicit budget. */
+	private commitConflictBudget(): number {
+		return Math.max(MAX_OUTSTANDING_TXN_DURATION, (this.root ?? this).timeoutBudget || 0);
+	}
+
+	/**
+	 * How long this logical commit has been retrying once it is past its budget, else 0. rocksdb-js
+	 * returns control from a parked commit every `ROCKSDB_JS_PARK_TIMEOUT_MS` even when the intent
+	 * holder never releases, so without this bound a request-path commit retries the attempt cap out
+	 * — minutes past the queue limit an operator configured (issue #2450).
+	 *
+	 * Source-applied writes are exempt for the same reason they are exempt from the attempt cap:
+	 * there is no resubscribe/sequence-resume path, so dropping one permanently diverges this node.
+	 */
+	private elapsedPastCommitBudget(): number {
+		if (this.sourceApply) return 0;
+		const startedAt = (this.root ?? this).commitStartedAt;
+		if (startedAt == null) return 0;
+		const elapsed = performance.now() - startedAt;
+		return elapsed > this.commitConflictBudget() ? elapsed : 0;
+	}
+
+	/**
+	 * Abandon a logical commit that stayed in write-intent conflict past its budget. Cleanup is
+	 * retry exhaustion's, so no link leaks a native handle or read snapshot; the error is distinct
+	 * because the condition is distinct — every attempt reported transient contention, so a later
+	 * request can succeed once the holder releases, which the generic exhaustion 500 does not say.
+	 *
+	 * Only a chain root may report `retryable`: a link commits solely from its predecessor's success
+	 * handler, so anywhere else in the chain an earlier store has already landed durable audit
+	 * entries and hooks that a replayed request would run twice. A head whose scope already rotated
+	 * through a mid-scope commit is in the same position.
+	 */
+	private abandonCommitAfterDeadline(headTransaction: RocksTransaction, elapsedMs: number): never {
+		const elapsed = Math.round(elapsedMs);
+		const budget = this.commitConflictBudget();
+		const retryable = !this.root && !this.snapshotFree;
+		if (allowStuckCommitLog('abandon', performance.now())) {
+			harperLogger.error(
+				`Abandoning a write transaction: its commit has been in write-intent conflict for ${elapsed}ms ` +
+					`(exceeds the ${budget}ms limit) across ${this.retries} retries, ` +
+					describeCommitIdentity(this.writes[0]?.store, this.startedFrom, headTransaction) +
+					`.` +
+					describeHolderCandidates(this.writes[0]?.store?.rootStore?.path, headTransaction?.id) +
+					` Another transaction holds a conflicting write intent and has not completed; the request is ` +
+					`failed with a 503${retryable ? '' : ' (not retryable — an earlier store in this transaction already committed)'} ` +
+					`rather than waiting further.`
+			);
+		}
+		this.abortChainAfterRetries(headTransaction);
+		throw new TransactionCommitConflictTimeoutError(
+			`Commit was in conflict with ongoing writes for ${elapsed}ms, exceeding the ${budget}ms limit; transaction abandoned after ${this.retries} retries`,
+			retryable
+		);
+	}
+
 	/**
 	 * Give up on a chain of linked transactions after exhausting conflict retries: poison every link
 	 * first, then abort each link's native transaction and release its DatabaseTransaction-level
@@ -1424,11 +1755,7 @@ export class DatabaseTransaction implements Transaction {
 			// throws. abort() rather than the native abort alone: it reclaims blobs a replayed write
 			// staged.
 			this.detachOwnedTransaction();
-			try {
-				transaction?.abort();
-			} catch (abortError) {
-				harperLogger.debug?.('aborting a transaction whose synchronous commit failed', abortError);
-			}
+			abortNativeTransaction(transaction, 'aborting a transaction whose synchronous commit failed');
 			try {
 				this.abort();
 			} catch (abortError) {
@@ -1455,6 +1782,20 @@ export interface Transaction {
 	abort?(): any;
 }
 
+export function shouldSpareCommitPhase(
+	txn: DatabaseTransaction,
+	checkedCommitPhaseChains: Set<DatabaseTransaction>
+): boolean {
+	if (!txn.committing) return false;
+	if (txn.sourceApply || txn.isReplay) return true;
+	const commitChainHead = txn.commitChainHead ?? txn;
+	if (!checkedCommitPhaseChains.has(commitChainHead)) {
+		checkedCommitPhaseChains.add(commitChainHead);
+		commitChainHead.commitPhaseTicks++;
+	}
+	return commitChainHead.commitPhaseTicks <= COMMIT_PHASE_GRACE;
+}
+
 export class ImmediateTransaction extends DatabaseTransaction {
 	isCommitting = false;
 	saveCommits = true;
@@ -1463,18 +1804,49 @@ export class ImmediateTransaction extends DatabaseTransaction {
 		this.db = db;
 	}
 	save(...args: any[]): any {
-		const transaction = args[0];
+		const operation = args[0]; // the staged write, not a transaction — commit() re-enters here with it
 		if (this.isCommitting) {
 			// if we are in the commit, do the save and force a reload so we get a read within the transaction
-			super.save(transaction, null as any, true);
+			super.save(operation, null as any, true);
 		} else {
 			this.isCommitting = true;
-			return when(this.commit(), () => {
+			// A synchronous throw from commit() (e.g. a 409 from an expired lock handle) would
+			// otherwise leave isCommitting latched at true, causing every subsequent save() in this
+			// context to take the fire-and-forget if-branch and silently drop writes.
+			let commitResult: any;
+			try {
+				commitResult = this.commit();
+			} catch (err) {
 				this.isCommitting = false;
-			});
+				throw err;
+			}
+			return when(
+				commitResult,
+				() => {
+					this.isCommitting = false;
+				},
+				(err: any) => {
+					// Async rejection (e.g. a rejected rocksdb commit promise) must also clear the
+					// latch; without this, every subsequent save() silently fire-and-forgets.
+					this.isCommitting = false;
+					throw err;
+				}
+			);
 		}
 	}
 
+	// Without an explicit transaction() a live lock is released only by unlock() or its lease, never
+	// by the per-write commit. Expired handles are pruned so a reused context does not retain every key.
+	releaseRecordLocks(): void {
+		const recordLocks = this.recordLocks;
+		if (!recordLocks) return;
+		for (const [store, storeMap] of recordLocks) {
+			for (const [keyId, handle] of storeMap) if (handle.released) storeMap.delete(keyId);
+			if (storeMap.size === 0) recordLocks.delete(store);
+		}
+		if (recordLocks.size === 0) this.recordLocks = undefined;
+	}
+
 	declare _timestamp: number;
 	// @ts-expect-error accessor overriding property
 	get timestamp() {
@@ -1553,6 +1925,7 @@ export function isJoinableScope(transaction: DatabaseTransaction | null | undefi
 }
 
 let timer;
+let monitorTick = 0;
 
 /**
  * True when a link other than `txn` in the same multi-store chain was written recently enough to
@@ -1572,16 +1945,96 @@ function chainStillActive(txn: DatabaseTransaction): boolean {
 	return false;
 }
 
+/**
+ * Name a chain link still holding its native handle past the reporting threshold, and why the
+ * branches below are not reaping it (harper#2471). Attribution only: it reads the recency clocks
+ * those branches maintain rather than calling `chainStillActive`, which would decay `writeTimeout`
+ * as a side effect, and it runs before them so it cannot reorder them.
+ */
+function reportLongLivedLink(
+	link: DatabaseTransaction,
+	thresholdMs: number,
+	now: number,
+	reportBudget: LongLivedHolderReportBudget
+): void {
+	if (!link.transaction || link.handleOpenedAt === 0) return;
+	const ageMs = now - link.handleOpenedAt;
+	if (ageMs < thresholdMs) return;
+	const states: string[] = [];
+	if (link.sourceApply) states.push('source-apply');
+	if (link.isReplay) states.push('replay');
+	if (link.committing) states.push('commit-phase');
+	if (link.open === TRANSACTION_STATE.CLOSED) states.push('closed-with-open-iterators');
+	if (monitorTick - link.writeTick <= 1) states.push('active');
+	if (states.length === 0) states.push('over-limit');
+	reportLongLivedHolder(
+		{
+			databasePath: (link.db as any)?.rootStore?.path,
+			nativeId: link.transaction.id,
+			openedAt: link.handleOpenedAt,
+			ageMs,
+			databaseName: (link.db as any)?.rootStore?.databaseName,
+			tableName: (link.db as any)?.name,
+			countPendingWrites: () => {
+				let pendingWrites = 0;
+				for (const write of link.writes) if (write) pendingWrites++;
+				return pendingWrites;
+			},
+			states,
+			timeoutBudget: link.timeoutBudget,
+			startedFrom: link.startedFrom,
+		},
+		reportBudget
+	);
+}
+
+/**
+ * Report every link of this logical transaction that holds its own native handle. A blind write to a
+ * second database attaches a handle to a `.next` link while only the root is supervised, so reporting
+ * the entry alone names an id the sweep's line can never be joined to when the child is the holder.
+ * Links that read are in `trackedTxns` and get their own visit from the tick.
+ */
+function reportIfLongLived(
+	txn: DatabaseTransaction,
+	thresholdMs: number,
+	now: number,
+	reportBudget: LongLivedHolderReportBudget
+): void {
+	if (thresholdMs === 0) return;
+	for (let link: DatabaseTransaction = txn; link; link = link.next) {
+		if (link !== txn && trackedTxns.has(link)) break;
+		reportLongLivedLink(link, thresholdMs, now, reportBudget);
+	}
+}
+
 function startMonitoringTxns() {
 	timer = setInterval(function () {
+		monitorTick++;
+		const checkedCommitPhaseChains = new Set<DatabaseTransaction>();
+		const reportThresholdMs = getReportThresholdMs();
+		rebaseLongLivedTransactionReports(reportThresholdMs);
+		const reportBudget = createLongLivedHolderReportBudget(reportThresholdMs);
+		const reportNow = performance.now();
 		// Both registries, in sequence rather than as a union: a root can be in each (it read, and a
 		// later link blind-wrote), and the membership check is cheaper than allocating per tick.
-		for (const txn of trackedTxns) monitorTransaction(txn);
-		for (const txn of supervisedWriteRoots) if (!trackedTxns.has(txn)) monitorTransaction(txn);
+		for (const txn of trackedTxns)
+			monitorTransaction(txn, checkedCommitPhaseChains, reportThresholdMs, reportNow, reportBudget);
+		for (const txn of supervisedWriteRoots)
+			if (!trackedTxns.has(txn))
+				monitorTransaction(txn, checkedCommitPhaseChains, reportThresholdMs, reportNow, reportBudget);
+		finishLongLivedHolderReports(reportBudget);
 	}, txnExpiration).unref();
 
-	function monitorTransaction(txn: DatabaseTransaction) {
+	function monitorTransaction(
+		txn: DatabaseTransaction,
+		checkedCommitPhaseChains: Set<DatabaseTransaction>,
+		reportThresholdMs: number,
+		reportNow: number,
+		reportBudget: LongLivedHolderReportBudget
+	) {
+		reportIfLongLived(txn, reportThresholdMs, reportNow, reportBudget);
 		{
+			const commitChainHead = txn.commitChainHead ?? txn;
 			// Decay write recency once per tick for every tracked link, independent of the `timeout`
 			// branches below — a tracked link that keeps its own idle limit alive by reading must not
 			// thereby keep chainStillActive believing it was written recently too.
@@ -1609,6 +2062,25 @@ function startMonitoringTxns() {
 						}`
 					);
 					txn.releaseReadTxn();
+				} else if (shouldSpareCommitPhase(txn, checkedCommitPhaseChains)) {
+					// Parked in commit()'s pre-commit await — a `before`/`beforeIntermediate` hook, in practice a
+					// blob's durable file write, which for a multi-tens-of-MB payload legitimately outruns the
+					// limit. The write set is sealed and the caller is awaiting this commit, so the limit's
+					// premise (the application is holding a transaction open) does not hold, and poisoning here
+					// would unlink the blobs the write still references (issue #2062). The grace is bounded
+					// because the transaction still pins a read snapshot: a source that stalls rather than
+					// finishing falls through to the abort below once it runs out. A canonical-source apply or
+					// replay is spared for as long as it takes: neither aborting it (harper-pro#348) nor the
+					// force-commit below — which would durably commit a record whose blob file is still being
+					// written — is acceptable, and their blob sources are bounded by the receive-side idle
+					// watchdog instead.
+					harperLogger.warn?.(
+						`Transaction has been in its commit phase past the open-transaction limit, waiting on pre-commit work (e.g. a large blob write); letting it complete, from table: ${
+							(txn.db as any)?.name + (url ? ' path: ' + url : '')
+						}`,
+						...(txn.startedFrom ? [`was started from ${txn.startedFrom.resourceName}.${txn.startedFrom.method}`] : [])
+					);
+					txn.timeout = Math.max(txnExpiration, txn.timeoutBudget ?? 0);
 				} else if (txn.hasPendingWrites() && chainStillActive(txn)) {
 					// A later link in the chain was written recently (writes re-arm only the link that
 					// receives them, and a multi-store transaction can be writing database B while this
@@ -1632,7 +2104,7 @@ function startMonitoringTxns() {
 						...(DEBUG_LONG_TXNS ? ['starting stack trace', txn.stackTraces] : [])
 					);
 					try {
-						txn.abortDueToTimeout();
+						commitChainHead.abortDueToTimeout();
 					} catch (error) {
 						harperLogger.debug?.(`Error aborting timed out transaction: ${error.message}`);
 					}
@@ -1670,6 +2142,12 @@ export function resetReplayedWritesWarning() {
 	replayedWritesWarned = false;
 }
 
+export function setMaxOutstandingTxnDuration(ms: number): number {
+	const previous = MAX_OUTSTANDING_TXN_DURATION;
+	MAX_OUTSTANDING_TXN_DURATION = ms;
+	return previous;
+}
+
 /** Test seam: whether the monitor supervises this logical transaction for its writes. */
 export function isWriteSupervised(txn: DatabaseTransaction): boolean {
 	return supervisedWriteRoots.has(txn);