@harperfast/harper 5.2.0 → 5.2.2

package/resources/DatabaseTransaction.ts

@@ -31,22 +31,112 @@ const MAX_RETRIES = 40;
 // Cap the per-retry backoff so replication-applied transactions, which retry conflicts without a
 // cap (see the commit rejection handler), don't grow the delay unbounded.
 const MAX_RETRY_DELAY_MS = 1000;
-let outstandingCommit, outstandingCommitStart;
-// Identity references for the commit that armed `outstandingCommit`, kept for the one-time
-// checkOverloaded() log below (harper#2001) — `outstandingCommit` itself is otherwise anonymous, so
-// a stuck commit gives no indication of which database/table/resource to investigate. Snapshotted
-// at arm time (not read from `this.writes`/`this.startedFrom` lazily off the DatabaseTransaction
-// object) because that object can be reused for a later immediate commit while the original native
-// commit is still wedged — its resolve handler runs clearWrites() on the SAME object, which would
-// blank or replace the identity out from under a deferred read. Three reference assignments, no
-// allocation, and no lingering reference to the writes/entries graph (the native transaction handle
-// is retained, but its lifetime is already bounded by the pending outstandingCommit promise).
-let outstandingCommitStore: any;
-let outstandingCommitStartedFrom: { resourceName: string; method: string } | undefined;
-let outstandingCommitNativeTransaction: any;
-// Ensures the checkOverloaded() rejection is logged once per stuck commit, not once per rejected
-// request — under load a wedged thread can reject hundreds of requests per second.
-let outstandingCommitLogged = false;
+// Every native write commit currently outstanding on this worker thread, oldest first.
+// checkOverloaded() sheds new application writes once the OLDEST of these has been outstanding
+// past MAX_OUTSTANDING_TXN_DURATION, so a commit whose promise never settles (harper#2001) is
+// detected whichever transaction submitted it. A linked list rather than a Set because the write
+// path reads the oldest on every write and must not allocate an iterator to do it, and because
+// commits settle out of order and so have to unlink in constant time.
+interface OutstandingCommit {
+	start: number;
+	prev: OutstandingCommit | undefined;
+	next: OutstandingCommit | undefined;
+	// Identity for the one-time checkOverloaded() log below (harper#2001) — otherwise a stuck commit
+	// gives no indication of which database/table/resource to investigate. Snapshotted at arm time
+	// (not read from `this.writes`/`this.startedFrom` lazily off the DatabaseTransaction object)
+	// because that object can be reused for a later immediate commit while this native commit is
+	// still wedged — its resolve handler runs clearWrites() on the SAME object, which would blank or
+	// replace the identity out from under a deferred read. Per-node (not a single module-level slot)
+	// so that every outstanding commit carries its own identity and its own `logged` flag: a second
+	// commit that is still stuck once the first settles becomes the new oldest and logs on its own.
+	store: any;
+	startedFrom: { resourceName: string; method: string } | undefined;
+	nativeTransaction: any;
+	logged: boolean;
+}
+let oldestOutstandingCommit: OutstandingCommit | undefined;
+let newestOutstandingCommit: OutstandingCommit | undefined;
+let outstandingCommitCount = 0;
+// Caps the stuck-commit log (checkOverloaded() below) to at most one line per this interval across
+// the whole thread, regardless of how many distinct commits individually cross the threshold — see
+// the comment at the log site for why a per-commit-only dedup isn't enough under sustained overload.
+const OVERLOAD_LOG_MIN_INTERVAL_MS = 1000;
+let lastOverloadLogAt = -Infinity;
+
+// Track a submitted commit until it settles. Every attempt is tracked unconditionally: a
+// coordinated retry round and a chained second-store commit are both issued from inside the
+// preceding commit's own resolve handler, which runs before any reaction that could release a
+// single shared slot — so anything conditional on "is something already outstanding" skips them
+// and leaves a wedged retry or chain invisible to checkOverloaded() forever. Each attempt is timed
+// from its own submission, keeping the overload window per-attempt rather than cumulative over a
+// retry ladder. `.then(untrack, untrack)` also marks an ERR_BUSY rejection handled, so this
+// tracking never surfaces as an unhandled rejection alongside the caller's own handler.
+// Exported only so unit tests can drive the list with controllable promises: the unlink order that
+// matters (a middle or tail node settling first) cannot be forced through real writes, and a node
+// left linked would 503 every write on this thread forever. commit() below is the sole caller.
+// Also the single source for write-queue-depth accounting (getTransactionQueueDepths below): every
+// native commit this function tracks is, by definition, exactly the write-queue backlog — see the
+// comment there for why that used to be a second, separately-maintained counter.
+// `store`/`startedFrom`/`nativeTransaction` are the identity snapshot for checkOverloaded()'s stuck-
+// commit log (harper#2001); omit them (as the test seam below does) when a caller has none to give.
+export function trackOutstandingCommit(
+	commitResolution: Promise<number | void>,
+	store?: any,
+	startedFrom?: { resourceName: string; method: string },
+	nativeTransaction?: any
+): void {
+	// Guards against a future caller passing a non-Promise: today commit() always hands this a real
+	// Promise, but an unguarded link here would leave a node permanently wedged in the list (503ing
+	// every write on this thread) with no settlement to ever unlink it.
+	if (typeof commitResolution?.then !== 'function') return;
+	const outstanding: OutstandingCommit = {
+		start: performance.now(),
+		prev: newestOutstandingCommit,
+		next: undefined,
+		store,
+		startedFrom,
+		nativeTransaction,
+		logged: false,
+	};
+	if (newestOutstandingCommit != null) newestOutstandingCommit.next = outstanding;
+	else oldestOutstandingCommit = outstanding;
+	newestOutstandingCommit = outstanding;
+	outstandingCommitCount++;
+	// Doubles as the write-queue-depth high-water mark (see getTransactionQueueDepths): every
+	// outstanding commit is a write-queue entry, so the peak of one is the peak of the other.
+	if (outstandingCommitCount > writeTxnQueueDepthHighWater) writeTxnQueueDepthHighWater = outstandingCommitCount;
+	// Guards against double-untracking the same node: `.then(untrack, untrack)` below means a promise
+	// that both resolves and later has its rejection handler independently triggered (or is tracked via
+	// a shared/misused resolution) could otherwise run the unlink twice, corrupting the list or driving
+	// outstandingCommitCount negative.
+	let untracked = false;
+	const untrack = () => {
+		if (untracked) return;
+		untracked = true;
+		if (outstanding.prev != null) outstanding.prev.next = outstanding.next;
+		else oldestOutstandingCommit = outstanding.next;
+		if (outstanding.next != null) outstanding.next.prev = outstanding.prev;
+		else newestOutstandingCommit = outstanding.prev;
+		outstandingCommitCount--;
+	};
+	commitResolution.then(untrack, untrack);
+}
+
+/**
+ * How many write commits are outstanding on this thread and how long the oldest has been waiting
+ * (`oldestAgeMs` is undefined when none is). `oldestAgeMs` is exactly the value checkOverloaded()
+ * rejects on, exposed so a commit that never settles (harper#2001) can be observed directly rather
+ * than inferred from the 503s it eventually produces.
+ */
+export function getOutstandingCommits(): { count: number; oldestAgeMs: number | undefined } {
+	return {
+		count: outstandingCommitCount,
+		oldestAgeMs: oldestOutstandingCommit ? performance.now() - oldestOutstandingCommit.start : undefined,
+	};
+}
+// Once per process: committing under open read iterators forces a write replay, so the warning is
+// about the caller's pattern, not the individual commit.
+let replayedWritesWarned = false;
 
 // The analytics module registers a recorder here at load (dependency inversion, mirroring
 // `replicationConfirmation` below) so the storage layer doesn't statically import the analytics/server
@@ -80,29 +170,20 @@ function recordCommitLatency(commitResolution: Promise<number | void>, submitted
 
 // Queue-depth gauges surfaced through the analytics pipeline (write-transaction-queue-depth /
 // read-transaction-queue-depth). Per-thread state; the analytics aggregator sums across threads.
-// `writeTxnQueueDepth` counts write commits handed to the storage engine but not yet resolved —
-// this is the backlog that, when it drains too slowly, produces the "Outstanding write transactions
-// have too long of queue" overload error. Read depth is derived from the live `trackedTxns` set
-// (every tracked transaction holds an open read snapshot). We also retain a high-water mark per
-// sampling window because the queue can fill and drain within a single (~1s) analytics period, so an
+// The write depth is `outstandingCommitCount` itself (maintained above by trackOutstandingCommit) —
+// write commits handed to the storage engine but not yet resolved are exactly the same set of native
+// commits the overload check tracks, and keeping one counter instead of two removes the duplicate
+// per-commit bookkeeping (and the drift risk: a code path that updates one but not the other, as the
+// replay path did before this fix). Read depth is derived from the live `trackedTxns` set (every
+// tracked transaction holds an open read snapshot). We also retain a high-water mark per sampling
+// window because the queue can fill and drain within a single (~1s) analytics period, so an
 // instantaneous sample taken at emit time would routinely miss the spike operators need to see.
 // RocksDB-write-path only: LMDB routes through the separate LMDBTransaction.commit()/getReadTxn()
 // overrides (resources/LMDBTransaction.ts), which maintain their own unrelated `trackedTxns` set and
 // do not call into this accounting.
-let writeTxnQueueDepth = 0;
 let writeTxnQueueDepthHighWater = 0;
 let readTxnQueueDepthHighWater = 0;
 
-function enterWriteQueue() {
-	if (++writeTxnQueueDepth > writeTxnQueueDepthHighWater) writeTxnQueueDepthHighWater = writeTxnQueueDepth;
-}
-function leaveWriteQueue() {
-	// Floor at zero: accounting is balanced by construction (every enterWriteQueue has exactly one
-	// matching settlement), but the guard is cheap insurance against a future call-site imbalance
-	// producing a negative depth that would corrupt every subsequent sample.
-	if (writeTxnQueueDepth > 0) writeTxnQueueDepth--;
-}
-
 /**
  * Returns the current write/read transaction queue depths for this thread along with the high-water
  * mark observed since the previous call, then resets the high-water marks to the current depth so the
@@ -113,12 +194,12 @@ export function getTransactionQueueDepths() {
 	// dominates the current size here — no need to reconcile against `readDepth` before reporting.
 	const readDepth = trackedTxns.size;
 	const depths = {
-		writeDepth: writeTxnQueueDepth,
+		writeDepth: outstandingCommitCount,
 		writeMaxDepth: writeTxnQueueDepthHighWater,
 		readDepth,
 		readMaxDepth: readTxnQueueDepthHighWater,
 	};
-	writeTxnQueueDepthHighWater = writeTxnQueueDepth;
+	writeTxnQueueDepthHighWater = outstandingCommitCount;
 	readTxnQueueDepthHighWater = readDepth;
 	return depths;
 }
@@ -248,6 +329,13 @@ export class DatabaseTransaction implements Transaction {
 	readTxnRefCount: number;
 	readTxnsUsed: number;
 	timeout: number;
+	// Write recency, tracked separately from `timeout`: set only by addWrite, never by a read. `timeout`
+	// is re-armed by reads too (on a link with no pending writes of its own, see the fast path in
+	// getReadTxn), so chainStillActive can't use it to mean "this link was written recently" — a `.next`
+	// link with no writes that is being read in a loop would otherwise masquerade as write activity and
+	// keep a write-holding head immortal.
+	declare writeTimeout: number;
+	timeoutBudget = 0;
 	validated = 0;
 	timestamp = 0;
 	retries = 0;
@@ -256,6 +344,10 @@ export class DatabaseTransaction implements Transaction {
 	// Whether this read handle's base reference (readTxnsUsed starts at 1 in getReadTxn) has been
 	// consumed by a commit round; iterator references are consumed only by doneReadTxn().
 	declare baseReadRefConsumed?: boolean;
+	// Set when a final commit/abort wanted to release the context's back-reference (see
+	// releaseContext()) but outstanding read iterators were still using this transaction —
+	// doneReadTxn() completes the release once the last iterator drains.
+	declare pendingContextRelease?: boolean;
 	declare startedFrom?: {
 		resourceName: string;
 		method: string;
@@ -278,10 +370,24 @@ export class DatabaseTransaction implements Transaction {
 	// open-transaction limit. Once poisoned, any further addWrite/commit throws transactionOpenTooLongError
 	// so the request rolls back cleanly instead of silently committing a partial write set (issue #1407).
 	declare timedOut?: boolean;
+	// Set once the retained read handle's write intents have been released (see commit()'s
+	// outstanding-iterators branch), so a retry round cannot re-fire the release.
+	declare writesAbandoned?: boolean;
 
 	getReadTxn(disableSnapshot?: boolean): ReadTransaction {
 		this.readTxnRefCount = (this.readTxnRefCount || 0) + 1;
-		this.timeout = txnExpiration; // reset the timeout
+		// The limit is an IDLE limit. Writes always re-arm it (see addWrite), but reads only do so
+		// while no uncommitted writes are held: staged writes hold write intents that other writers'
+		// coordinated-retry commits park on, so a handler that wrote once and then only reads — an
+		// orphaned long-poll whose client had already gone, in harper#2001 — must not keep those
+		// intents alive by reading. A transaction that keeps writing stays alive; so does a purely
+		// read-only one. A committed transaction re-arms too: its intents went with the commit, and
+		// the monitor bounds its retained read snapshot separately.
+		// `writes`/`next` are checked inline first: this is a hot path and the dominant case is a
+		// single-store transaction that has never written.
+		if ((this.writes.length === 0 && !this.next) || this.open !== TRANSACTION_STATE.OPEN || !this.hasPendingWrites()) {
+			this.timeout = Math.max(txnExpiration, this.timeoutBudget);
+		}
 		if (this.transaction) {
 			if ((this.transaction as any).openTimer) (this.transaction as any).openTimer = 0;
 			return this.transaction;
@@ -328,6 +434,7 @@ export class DatabaseTransaction implements Transaction {
 			trackedTxns.delete(this);
 			this.transaction?.abort();
 			this.transaction = null;
+			this.completeDeferredContextRelease();
 		}
 	}
 
@@ -346,6 +453,19 @@ export class DatabaseTransaction implements Transaction {
 			harperLogger.debug?.('releasing timed-out read transaction', error);
 		}
 		this.transaction = null;
+		this.completeDeferredContextRelease();
+	}
+
+	/**
+	 * Complete a context release that releaseContext() deferred because outstanding read iterators
+	 * were still using this transaction (see releaseContext()) — called once the last one drains,
+	 * whether that happens naturally (doneReadTxn()) or is forced by the long-transaction monitor
+	 * (releaseReadTxn()).
+	 */
+	private completeDeferredContextRelease(): void {
+		if (!this.pendingContextRelease) return;
+		this.pendingContextRelease = false;
+		if (this.#context?.transaction === this) this.#context.transaction = null;
 	}
 
 	disregardReadTxn(): void {
@@ -378,25 +498,71 @@ export class DatabaseTransaction implements Transaction {
 		this.writesByKey = undefined;
 	}
 
+	/**
+	 * Drop this transaction's back-reference from its context once completed (commit or abort),
+	 * so a long-lived context (e.g. an MQTT subscription context held open for the life of a
+	 * suspended delivery loop) doesn't keep pinning a finished transaction in memory. Guarded by
+	 * identity: a context already re-pointed at a different (e.g. reused) transaction is untouched.
+	 *
+	 * `final` must be false for an in-callback explicit `context.transaction.commit()` — the
+	 * "commit in the middle" pattern intentionally keeps recommitting and adding writes to the
+	 * SAME instance (see the comment above about a transaction being "reused and committed
+	 * again"), so releasing here would strand those later writes with no transaction to join.
+	 * Only resources/transaction.ts's own wrapper commit (`{ doneWriting: true }`, once the
+	 * caller's callback has fully returned) and abort() are truly final.
+	 *
+	 * A final commit can still have outstanding read iterators streaming through this.transaction
+	 * (see the outstanding-iterators branch in commit()) — those keep this instance meaningfully
+	 * alive (a fresh write on the same context must not join a DIFFERENT, already-replayed
+	 * transaction) until doneReadTxn() drains the last one, so the release is deferred to there.
+	 *
+	 * Sets `.transaction` to `null` rather than deleting the property: `Context.transaction` is
+	 * typed `DatabaseTransaction | null | undefined` precisely to document this released state, and
+	 * `delete` would repeatedly force a long-lived, hot context (e.g. an MQTT subscription context
+	 * releasing/reattaching a transaction per message) into V8's slower dictionary-mode property
+	 * storage.
+	 */
+	private releaseContext(final: boolean): void {
+		if (!final) return;
+		if (this.readTxnsUsed > 0) {
+			this.pendingContextRelease = true;
+			return;
+		}
+		if (this.#context?.transaction === this) this.#context.transaction = null;
+	}
+
 	checkOverloaded() {
 		if (
-			outstandingCommit &&
+			oldestOutstandingCommit &&
 			!this.overloadChecked &&
-			performance.now() - outstandingCommitStart > MAX_OUTSTANDING_TXN_DURATION
+			performance.now() - oldestOutstandingCommit.start > MAX_OUTSTANDING_TXN_DURATION
 		) {
-			if (!outstandingCommitLogged) {
+			const now = performance.now();
+			// Also rate-limited across the whole overload episode (not just deduped per commit): under
+			// sustained heavy load, many distinct commits can each individually age past the limit in
+			// quick succession as earlier ones finally settle, which without this cap would turn one
+			// overload episode into a growing stream of ERROR lines — the same "flood" harper#2001's
+			// original per-request log was fixed to avoid, just shifted from per-request to per-commit.
+			// A commit skipped by the cooldown is NOT marked `logged`, so it still gets a log later if
+			// it's still the oldest once the cooldown clears, rather than going silent forever.
+			if (!oldestOutstandingCommit.logged && now - lastOverloadLogAt > OVERLOAD_LOG_MIN_INTERVAL_MS) {
 				// Log once per stuck commit (not once per rejected request, harper#2001): a wedged
 				// thread otherwise logs nothing at all server-side while rejecting every write with a
-				// 503, which was the single biggest obstacle to root-causing a recurrence.
-				outstandingCommitLogged = true;
-				const nativeTransactionId = outstandingCommitNativeTransaction?.id;
+				// 503, which was the single biggest obstacle to root-causing a recurrence. The flag lives
+				// on the node itself, so if THIS commit settles while still over the limit and a
+				// different one is now oldest, that one logs too instead of staying silent forever.
+				oldestOutstandingCommit.logged = true;
+				lastOverloadLogAt = now;
+				const nativeTransactionId = oldestOutstandingCommit.nativeTransaction?.id;
+				const store = oldestOutstandingCommit.store;
+				const startedFrom = oldestOutstandingCommit.startedFrom;
 				harperLogger.error(
 					`Rejecting writes on this thread: a commit has been outstanding for ` +
-						`${Math.round(performance.now() - outstandingCommitStart)}ms (exceeds the ` +
-						`${MAX_OUTSTANDING_TXN_DURATION}ms limit), from table: ${outstandingCommitStore?.rootStore?.databaseName ?? '?'}.${outstandingCommitStore?.name ?? '?'}` +
+						`${Math.round(now - oldestOutstandingCommit.start)}ms (exceeds the ` +
+						`${MAX_OUTSTANDING_TXN_DURATION}ms limit), from table: ${store?.rootStore?.databaseName ?? '?'}.${store?.name ?? '?'}` +
 						(nativeTransactionId !== undefined ? ` (transaction ${nativeTransactionId})` : '') +
-						(outstandingCommitStartedFrom?.resourceName
-							? `, started from ${outstandingCommitStartedFrom.resourceName}${outstandingCommitStartedFrom.method ? '.' + outstandingCommitStartedFrom.method : ''}`
+						(startedFrom?.resourceName
+							? `, started from ${startedFrom.resourceName}${startedFrom.method ? '.' + startedFrom.method : ''}`
 							: '') +
 						`. Further record updates and publishes from new application requests on this thread ` +
 						`will be rejected with 503 until the commit settles or the process is restarted (deletes, ` +
@@ -410,6 +576,13 @@ export class DatabaseTransaction implements Transaction {
 
 	addWrite(operation: TransactionWrite) {
 		if (this.timedOut) throw transactionOpenTooLongError();
+		// A write is activity: it re-arms the idle limit on this link even though the reads it
+		// performs no longer do (see getReadTxn), so a transaction that keeps writing stays alive
+		// and only an idle one holding write intents is reaped.
+		this.timeout = Math.max(txnExpiration, this.timeoutBudget ?? 0);
+		// Independent write-recency signal for chainStillActive (see the field comment) — reads never
+		// touch this, only writes do.
+		this.writeTimeout = this.timeout;
 		this.linkWrite(operation);
 		this.writes.push(operation);
 		if (!operation.deferSave) {
@@ -535,6 +708,14 @@ export class DatabaseTransaction implements Transaction {
 					this.writes = this.writes.filter((write) => write); // filter out removed entries
 					if (this.writes.length > 0) {
 						if (!options.transaction) {
+							if (!replayedWritesWarned) {
+								replayedWritesWarned = true;
+								harperLogger.warn?.(
+									`Committing while read iterators are still open: ${this.writes.length} staged write(s) must be re-staged and committed on a second transaction, doubling their write work` +
+										(this.startedFrom ? `, from ${this.startedFrom.resourceName}.${this.startedFrom.method}` : '') +
+										`. Fully consume (or close) iterators before committing to avoid this. Logged once per process.`
+								);
+							}
 							// Deliberately NOT marked isRetry and NOT carrying over the original's onCommit:
 							// audit/txn-log entries batch natively on the transaction they were staged into and
 							// are only durably written by that transaction's commit attempt (an abort discards
@@ -557,6 +738,28 @@ export class DatabaseTransaction implements Transaction {
 						// re-staged the writes into it
 						commitResolution = transaction.commit() as Promise<void>;
 						recordCommitLatency(commitResolution, performance.now());
+						// Write-queue-depth accounting for this replay commit happens uniformly below, via
+						// trackOutstandingCommit(commitResolution) — see that function's comment. Omitting
+						// dedicated accounting here (as a prior version of this replay path did) used to leave
+						// write-transaction-queue-depth, the one metric that can observe a commit that never
+						// settles (harper#2001), reading zero for exactly this path.
+					}
+					// No commit will ever run on the retained handle — the replay above owns these
+					// writes — so this is the only place its write intents can be released. Left in
+					// place, other writers' coordinated-retry commits park on them until the last
+					// iterator finishes (harper#2001). Reads through the handle, including
+					// read-your-own-writes, keep working. Once only: a coordinated-retry or backoff
+					// round re-enters this branch on the same retained handle. Fenced like the other
+					// post-submit steps here: the replay commit is already in flight, so a throw must
+					// not skip onCommit/the chain-store commit below. Optional: rocksdb-js < 2.7
+					// lacks the method.
+					if (!this.writesAbandoned) {
+						this.writesAbandoned = true;
+						try {
+							(this.transaction as { abandonWrites?: () => void } | null)?.abandonWrites?.();
+						} catch (error) {
+							harperLogger.warn?.('Failed to release write intents on a retained read transaction', error);
+						}
 					}
 				} else {
 					// no more reads need to be performed, just commit/abort based if there are any writes
@@ -571,30 +774,19 @@ export class DatabaseTransaction implements Transaction {
 							// Promise<number | void>; it is handled in the resolve callback below.
 							commitResolution = transaction.commit();
 							// Record how long this commit stays outstanding (submit → settle) as a distribution
-							// metric. This is the same clock the overload check uses (outstandingCommitStart is
-							// stamped at submit), so a rising p99/p999 is the leading indicator for the
+							// metric. This is the same clock the overload check uses (trackOutstandingCommit
+							// stamps each attempt at submit), so a rising p99/p999 is the leading indicator for the
 							// "Outstanding write transactions have too long of queue" (503) rejection. A transient-
-							// conflict retry rejects this promise and issues a fresh commit(), but outstandingCommit
-							// only re-arms for the backoff path (retries > 2, below) — the coordinated-retry and
-							// retries<=2 paths re-commit synchronously while outstandingCommit is still set, so
-							// recording per attempt here counts more attempts than the overload check ever arms for.
+							// conflict retry rejects this promise and issues a fresh commit(), which is tracked as
+							// its own attempt, so recording per attempt matches the overload semantics.
 							// commitResolution's declared type (Promise<number | void> | void) doesn't narrow to
 							// Promise<void> here because the widening union defeats flow analysis on the prior
 							// cast assignment; re-assert it — this branch's commit() result is always a Promise.
 							recordCommitLatency(commitResolution as Promise<void>, performance.now());
-							// Count this commit against the write queue depth until the storage engine
-							// resolves it. A transient-conflict retry rejects this promise and issues a
-							// fresh commit() (re-entering here), so the enter/leave stays balanced. leaveWriteQueue
-							// never throws, so the settled promise resolves and needs no rejection handling of its own.
-							// The thenable guard protects against a future caller passing a non-Promise
-							// `commitResolution` (today it is always rocksdb-js's async Transaction.commit()
-							// result, guaranteed to be a Promise).
-							enterWriteQueue();
-							if (commitResolution && typeof (commitResolution as any).then === 'function') {
-								commitResolution.then(leaveWriteQueue, leaveWriteQueue);
-							} else {
-								leaveWriteQueue();
-							}
+							// Write-queue-depth accounting for this commit happens uniformly below, via
+							// trackOutstandingCommit(commitResolution) — see that function's comment. A
+							// transient-conflict retry rejects this promise and issues a fresh commit()
+							// (re-entering here), which trackOutstandingCommit tracks as its own attempt.
 						} else {
 							try {
 								commitResolution = transaction.abort();
@@ -609,43 +801,10 @@ export class DatabaseTransaction implements Transaction {
 				}
 
 				if (commitResolution) {
-					// Known gap (pre-existing, not introduced here): this branch only arms when
-					// outstandingCommit is currently unset. A coordinated-retry re-commit (below) or a
-					// chained this.next.commit() runs synchronously inside this same commit's .then
-					// handler, one microtask before the .catch().finally() below clears outstandingCommit
-					// — so a retry or a second store's commit that itself wedges is never armed, and
-					// neither checkOverloaded() nor this log will ever see it. Only the first store's
-					// first commit attempt is covered.
-					if (!outstandingCommit) {
-						outstandingCommit = commitResolution;
-						outstandingCommitStart = performance.now();
-						// Snapshot the store/origin now rather than reading them off `this` lazily in
-						// checkOverloaded(): `this` (a DatabaseTransaction) can be reused for a later immediate
-						// commit while this native commit is still wedged, and that later commit's resolve
-						// handler runs clearWrites() on this SAME object — a lazy read would then blank or
-						// mis-attribute the identity out from under the deferred log. Read the table off the
-						// write itself, not this.db, which is whichever table first claimed this per-database
-						// transaction in txnForContext and so can name the wrong table when a transaction spans
-						// more than one table in the same database. Both are stable references (a Store, and an
-						// object set once and never mutated), so this holds no allocation and no lingering
-						// reference to the transaction/writes/entries graph.
-						outstandingCommitStore = this.writes[0]?.store;
-						outstandingCommitStartedFrom = this.startedFrom;
-						outstandingCommitNativeTransaction = transaction;
-						outstandingCommitLogged = false;
-						outstandingCommit
-							// if `commitResolution` rejects with and `ERR_BUSY` error, the retry logic
-							// will correct course, but the reject will still be propagated on the
-							// `outstandingCommit` promise and needs to be caught and silenced
-							.catch(() => {})
-							.finally(() => {
-								outstandingCommit = null;
-								outstandingCommitStore = undefined;
-								outstandingCommitStartedFrom = undefined;
-								outstandingCommitNativeTransaction = undefined;
-								outstandingCommitLogged = false;
-							});
-					}
+					// Read the table off the write itself, not this.db, which is whichever table first
+					// claimed this per-database transaction in txnForContext and so can name the wrong
+					// table when a transaction spans more than one table in the same database.
+					trackOutstandingCommit(commitResolution, this.writes[0]?.store, this.startedFrom, transaction);
 					const completions = [];
 					return commitResolution.then(
 						(commitResult) => {
@@ -721,6 +880,7 @@ export class DatabaseTransaction implements Transaction {
 							// now reset transactions tracking; this transaction be reused and committed again
 							this.retries = 0; // reset per-native-transaction retry counter so a reused DatabaseTransaction's next batch starts fresh
 							this.clearWrites();
+							this.releaseContext(!!options.doneWriting);
 							this.next = null;
 							let txnTime = this.timestamp;
 							this.timestamp = 0; // reset the timestamp as well
@@ -802,6 +962,11 @@ export class DatabaseTransaction implements Transaction {
 								} catch (abortError) {
 									harperLogger.debug?.('aborting transaction after failed commit', abortError);
 								}
+								// A terminal failure is just as final as a success — release the context's
+								// back-reference here too, or transaction.ts's onComplete() (which has no
+								// rejection handler of its own) would leave a long-lived context pinning this
+								// CLOSED wrapper forever.
+								this.releaseContext(!!options.doneWriting);
 								throw error;
 							}
 						}
@@ -812,6 +977,7 @@ export class DatabaseTransaction implements Transaction {
 						cleanupUnusedBlobs(write.savedBlobs, collectRetainedFileIds(write.store.getEntry(write.key)?.value));
 				}
 				this.clearWrites();
+				this.releaseContext(!!options.doneWriting);
 				const txnResolution: CommitResolution = {
 					txnTime: this.timestamp,
 				};
@@ -846,6 +1012,13 @@ export class DatabaseTransaction implements Transaction {
 		}
 		// reset the transaction
 		this.clearWrites();
+		// A timeout-poisoned abort (abortDueToTimeout()) is the one abort that is NOT "reuse-free":
+		// Resource.ts's dispatcher deliberately keeps joining a `timedOut` transaction (instead of
+		// starting a fresh one) so the rest of the logical operation fails atomically via the
+		// poison check in addWrite()/commit(), rather than silently landing a later write on a
+		// brand-new transaction after an earlier one was rolled back (#1411). Releasing here would
+		// make that check see `undefined?.timedOut` and take the "start fresh" branch instead.
+		this.releaseContext(!this.timedOut);
 	}
 	/**
 	 * Give up on a chain of linked transactions after exhausting conflict retries: poison every link
@@ -935,6 +1108,7 @@ export interface CommitResolution {
 	next?: CommitResolution;
 }
 export interface Transaction {
+	timeoutBudget?: number;
 	commit(options): MaybePromise<CommitResolution>;
 	abort?(): any;
 }
@@ -973,9 +1147,31 @@ export class ImmediateTransaction extends DatabaseTransaction {
 
 let timer;
 
+/**
+ * True when a link other than `txn` in the same multi-store chain was written recently enough to
+ * still be active — i.e. its `writeTimeout` (set only by addWrite, see the field comment) hasn't
+ * decayed to zero. Writes re-arm only the link that receives them, so a chain writing database B
+ * while its head only reads A would otherwise be aborted by the head's own decay.
+ */
+function chainStillActive(txn: DatabaseTransaction): boolean {
+	for (let link: DatabaseTransaction = txn.next; link; link = link.next) {
+		// A write-only link (e.g. a blind write to a second database, never itself read) never calls
+		// getReadTxn, so it's never added to trackedTxns and the main loop below never decays it.
+		// Decay it here instead, so an idle write-only link eventually expires rather than keeping the
+		// whole chain immortal (harper#2001's blind-write shape).
+		if (!trackedTxns.has(link) && link.writeTimeout > 0) link.writeTimeout -= txnExpiration;
+		if (link.writeTimeout > 0) return true;
+	}
+	return false;
+}
+
 function startMonitoringTxns() {
 	timer = setInterval(function () {
 		for (const txn of trackedTxns) {
+			// Decay write recency once per tick for every tracked link, independent of the `timeout`
+			// branches below — a tracked link that keeps its own idle limit alive by reading must not
+			// thereby keep chainStillActive believing it was written recently too.
+			if (txn.writeTimeout > 0) txn.writeTimeout -= txnExpiration;
 			if (txn.timeout <= 0) {
 				const url = (txn.getContext() as any)?.url;
 				if (txn.open === TRANSACTION_STATE.CLOSED) {
@@ -991,6 +1187,12 @@ function startMonitoringTxns() {
 						}`
 					);
 					txn.releaseReadTxn();
+				} else if (txn.hasPendingWrites() && chainStillActive(txn)) {
+					// A later link in the chain was written recently (writes re-arm only the link that
+					// receives them, and a multi-store transaction can be writing database B while this
+					// head only reads A). The logical transaction is still active, so re-arm this link
+					// rather than aborting the whole chain out from under it.
+					txn.timeout = Math.max(txnExpiration, txn.timeoutBudget ?? 0);
 				} else if (txn.hasPendingWrites() && !txn.sourceApply && !txn.isReplay) {
 					// Abort and surface an error rather than force-committing a partial write set: silently
 					// committing on the application's behalf breaks atomicity and can leave orphaned
@@ -1026,7 +1228,7 @@ function startMonitoringTxns() {
 					} catch (error) {
 						harperLogger.debug?.(`Error committing timed out transaction: ${error.message}`);
 					}
-					txn.timeout = txnExpiration;
+					txn.timeout = Math.max(txnExpiration, txn.timeoutBudget ?? 0);
 				}
 			} else {
 				txn.timeout -= txnExpiration;
@@ -1037,6 +1239,15 @@ function startMonitoringTxns() {
 
 startMonitoringTxns();
 
+/**
+ * Test seam: re-arms the once-per-process replay warning. The whole unit suite shares one process,
+ * so whichever test first drives a commit under open iterators consumes the warning for every test
+ * after it.
+ */
+export function resetReplayedWritesWarning() {
+	replayedWritesWarned = false;
+}
+
 export function setTxnExpiration(ms) {
 	clearInterval(timer);
 	txnExpiration = ms;

package/resources/ResourceInterface.ts

@@ -67,8 +67,14 @@ export interface Context {
 	login?: (username: string, password: string) => Promise<string>;
 	/** Describes the current cookie-based session if it is present and grants the capacity to delete it. authentication.enableSessions must be turned on in the harperdb-config.yaml  */
 	session?: Session;
-	/**	 The database transaction object */
-	transaction?: DatabaseTransaction;
+	/**
+	 * The database transaction object. `undefined` means none was ever attached; `null` means one
+	 * was attached and has since completed and released its back-reference (DatabaseTransaction.ts's
+	 * releaseContext()) — kept `null` rather than deleting the property so a long-lived, hot context
+	 * (e.g. an MQTT subscription context releasing/reattaching a transaction per message) doesn't
+	 * repeatedly force V8 to deoptimize it into dictionary-mode property storage.
+	 */
+	transaction?: DatabaseTransaction | null;
 	/**	 If the operation that will be performed with this context should check user authorization	 */
 	authorize?: boolean;
 	/**	 The last modification time of any data that has been accessed with this context	 */