@harperfast/harper 5.3.0 → 5.3.1

package/resources/Table.ts

@@ -1259,6 +1259,18 @@ function scopeViolation(
 	);
 }
 
+function rescope(resolved: ResolvedRecordLockOptions, tableReplicates: boolean): ResolvedRecordLockOptions {
+	if (resolved.scopeRequested) return resolved;
+	const scope = tableReplicates ? 'cluster' : 'node';
+	return scope === resolved.scope ? resolved : { ...resolved, scope };
+}
+
+function transportUnavailable(databaseName: string): LockUnavailableError {
+	return new LockUnavailableError(
+		`Cluster-scoped record locks are not available on ${databaseName}: no record lock transport is registered`
+	);
+}
+
 /** Distinguishes bare lock options from a record target (id, URL, {id:...}). */
 function isPlainOptions(value: unknown): boolean {
 	return (
@@ -1282,6 +1294,26 @@ setLockCoordinatorResolver(
 	}
 );
 
+// Valid for one synchronous notify pass, in which a key's subscribers on this thread all receive the same
+// freshly decoded audit record; cleared after it so it pins no store or record once delivery is done.
+let memoizedEntryAuditRecord: any;
+let memoizedEntryStore: any;
+let memoizedEntryId: Id;
+let memoizedEntry: Entry | undefined;
+function clearEntryMemo() {
+	memoizedEntryAuditRecord = memoizedEntryStore = memoizedEntryId = memoizedEntry = undefined;
+}
+function currentEntryForAudit(store: any, id: Id, auditRecord: any): Entry | undefined {
+	if (auditRecord !== memoizedEntryAuditRecord || store !== memoizedEntryStore || id !== memoizedEntryId) {
+		if (memoizedEntryAuditRecord === undefined) queueMicrotask(clearEntryMemo);
+		memoizedEntry = store.getEntry(id);
+		memoizedEntryAuditRecord = auditRecord;
+		memoizedEntryStore = store;
+		memoizedEntryId = id;
+	}
+	return memoizedEntry;
+}
+
 export function makeTable(options): TableResourceClass {
 	const {
 		primaryKey,
@@ -1580,12 +1612,25 @@ export function makeTable(options): TableResourceClass {
 			return;
 		return { key, descriptor };
 	}
+	const COMMIT_BASE_METHODS = new Set(['put', 'patch', 'delete']);
+	function entryBeforeWrite(loadedEntry: Entry | undefined, id: Id, transaction: any, reloadsCommitBase: boolean) {
+		if (loadedEntry != null) return loadedEntry;
+		if (isRocksDB && reloadsCommitBase) {
+			// save() reads this write's base from the staging snapshot (harper#2259); the read handle is still opened
+			// here because it is what gives the staged write coordinated conflict retries
+			transaction.getReadTxn();
+			return undefined;
+		}
+		return primaryStore.getEntry(id, { transaction: transaction.getReadTxn() });
+	}
 	class TableResource<Record extends object = any> extends Resource<Record> {
 		#record: any; // the stored/frozen record from the database and stored in the cache (should not be modified directly)
 		#changes: any; // the changes to the record that have been made (should not be modified directly)
 		#version?: number; // version of the record
 		#entry?: Entry; // the entry from the database
 		#savingOperation?: any; // operation for the record is currently being saved
+		#baseReadTxn?: any; // staging handle this instance's uncached pre-load read through
+		#baseReadEntry?: Entry; // what that read returned, reusable as the commit base only while it is still #entry
 		#lockHandle?: RecordLockHandle; // the record lock acquired by lock() — scoped or hold
 		#lockWritable?: boolean; // set by #reloadLocked to let save() stage lock-writable updates
 		#writeGeneration?: WriteGeneration;
@@ -2361,15 +2406,29 @@ export function makeTable(options): TableResourceClass {
 				if (readTxn?.isDone) {
 					throw new Error('You can not read from a transaction that has already been committed/aborted');
 				}
+				const readsCommitBase =
+					isRocksDB &&
+					COMMIT_BASE_METHODS.has(resourceOptions?.method) &&
+					!resourceOptions?.ensureLoaded &&
+					readTxn &&
+					!(readTxn as any).snapshotDisabled;
 				return loadLocalRecord(
 					id,
 					request,
-					{ transaction: readTxn, ensureLoaded: resourceOptions?.ensureLoaded },
+					{
+						transaction: readTxn,
+						ensureLoaded: resourceOptions?.ensureLoaded,
+						uncachedRead: readsCommitBase || undefined,
+					},
 					sync,
 					(entry) => {
 						if (entry) {
 							TableResource._updateResource(this, entry);
 						} else this.#record = null;
+						if (readsCommitBase) {
+							this.#baseReadTxn = readTxn;
+							this.#baseReadEntry = entry;
+						}
 						if (request.onlyIfCached) {
 							// don't go into the loading from source condition, but HTTP spec says to
 							// return 504 (rather than 404) if there is no content and the cache-control header
@@ -2401,6 +2460,14 @@ export function makeTable(options): TableResourceClass {
 				throw error;
 			}
 		}
+		// Reusable only for the key this instance read and while #entry is the entry that read returned: a source fill
+		// or retry replaces #entry, and ensureLoaded() can evict it in place, so ensureLoaded() drops the receipt.
+		#commitBaseTxn(id: Id) {
+			const baseReadTxn = this.#baseReadTxn;
+			if (!baseReadTxn || this.#baseReadEntry !== this.#entry) return;
+			const receiverId = this.getId();
+			if (Object.is(id, receiverId) || writeKeyId(id) === writeKeyId(receiverId)) return baseReadTxn;
+		}
 		static _updateResource(resource, entry) {
 			resource.#entry = entry;
 			resource.#record = entry?.value ?? null;
@@ -2412,6 +2479,7 @@ export function makeTable(options): TableResourceClass {
 		 * @returns
 		 */
 		ensureLoaded() {
+			this.#baseReadTxn = undefined;
 			const loadedFromSource = ensureLoadedFromSource(
 				(this.constructor as any).source,
 				this.getId(),
@@ -3655,6 +3723,7 @@ export function makeTable(options): TableResourceClass {
 								isRocksDB && audit && txnLogKey !== txnTime
 									? [{ version: txnLogKey, nodeId: options?.nodeId }]
 									: undefined,
+							localOnly: options?.localOnly,
 						},
 						'invalidate'
 					);
@@ -3723,6 +3792,7 @@ export function makeTable(options): TableResourceClass {
 								isRocksDB && audit && txnLogKey !== txnTime
 									? [{ version: txnLogKey, nodeId: options?.nodeId }]
 									: undefined,
+							localOnly: options?.localOnly,
 						},
 						'relocate',
 						false,
@@ -3900,7 +3970,7 @@ export function makeTable(options): TableResourceClass {
 			const id = target != null ? requestTargetToId(target as RequestTargetOrId) : this.getId();
 			checkValidId(id);
 			this.#assertLiveHandle(id);
-			const resolved = resolveLockOptions(options);
+			const resolved = resolveLockOptions(options, TableResource.replicate !== false);
 			const context = this.getContext();
 			const link = txnForContext(context);
 			const keyId = writeKeyId(id);
@@ -3912,11 +3982,7 @@ export function makeTable(options): TableResourceClass {
 				(resolved.scopeRequested || isClusterLockRequired(databaseName)) &&
 				!getClusterLockTransport(databaseName)
 			)
-				return Promise.reject(
-					new LockUnavailableError(
-						`Cluster-scoped record locks are not available on ${databaseName}: no record lock transport is registered`
-					)
-				);
+				return Promise.reject(transportUnavailable(databaseName));
 			const held = this.#lockHandle;
 			if (held && !held.isExpired() && held.keyId === keyId) {
 				// Re-entrant: upgrade to hold if requested, then preserve staged changes.
@@ -4007,7 +4073,11 @@ export function makeTable(options): TableResourceClass {
 						clearTimeout(followerTimer);
 						const acquired = link.recordLockFor(primaryStore, keyId);
 						if (acquired && !acquired.isExpired()) {
-							const violation = scopeViolation(acquired, resolved, databaseName);
+							const violation = scopeViolation(
+								acquired,
+								rescope(resolved, TableResource.replicate !== false),
+								databaseName
+							);
 							if (violation) throw violation;
 							if (resolved.hold && !acquired.hold) {
 								detachScopedUpgradeWrite(link, keyId, acquired);
@@ -4056,8 +4126,9 @@ export function makeTable(options): TableResourceClass {
 				// caller's whole timeout, long enough for harper-pro to register the transport on this
 				// worker. Using the snapshot would take the native key alone and hand back a node-scoped
 				// handle while a peer that already had the transport is granted the same key.
+				const current = rescope(resolved, TableResource.replicate !== false);
 				try {
-					if (resolved.scope !== 'node') coordinator = TableResource.lockCoordinator ?? coordinator;
+					coordinator = current.scope === 'cluster' ? (TableResource.lockCoordinator ?? coordinator) : undefined;
 				} catch (error) {
 					// The getter fails closed on an unusable node identity, and that has to reach the caller
 					// the same way it does before the wait. Swallowing it let an implicit cluster lock fall
@@ -4066,6 +4137,15 @@ export function makeTable(options): TableResourceClass {
 					handle.release();
 					throw error as Error;
 				}
+				// The entry check cannot cover this: the wait is where the call became cluster-scoped.
+				if (
+					current.scope === 'cluster' &&
+					!coordinator &&
+					(current.scopeRequested || isClusterLockRequired(databaseName))
+				) {
+					handle.release();
+					throw transportUnavailable(databaseName);
+				}
 				if (coordinator) {
 					try {
 						// Not a 423 when the budget is gone, and not a skip either: the native wait can consume
@@ -4383,7 +4463,9 @@ export function makeTable(options): TableResourceClass {
 					}
 				};
 			}
-			const entry = this.#entry ?? primaryStore.getEntry(id, { transaction: transaction.getReadTxn() });
+			const reloadsCommitBase = options?.isCopyApply !== true;
+			const entry = entryBeforeWrite(this.#entry, id, transaction, reloadsCommitBase);
+			const baseReadTxn = this.#commitBaseTxn(id);
 			const writeToSource = () => {
 				if (!(this.constructor as any).source || (context as any)?.source) return;
 				if (fullUpdate) {
@@ -4411,11 +4493,12 @@ export function makeTable(options): TableResourceClass {
 				key: id,
 				store: primaryStore,
 				entry,
+				baseReadTxn,
 				nodeName: (context as any)?.nodeName,
 				fullUpdate,
 				chainsStagedState: true,
 				// copy-apply rows keep their pre-read base: one read per row, healed by the post-copy replay
-				reloadCommitBase: options?.isCopyApply !== true,
+				reloadCommitBase: reloadsCommitBase,
 				deferSave: true,
 				// the origin's record version on an applied write; absent for a locally-originated one
 				recordVersion: options?.version,
@@ -5303,12 +5386,14 @@ export function makeTable(options): TableResourceClass {
 			const transaction = txnForContext(context);
 			assertDerivedIndexAdmission(options, transaction);
 			checkValidId(id);
-			const entry = this.#entry ?? primaryStore.getEntry(id, { transaction: transaction.getReadTxn() });
+			const entry = entryBeforeWrite(this.#entry, id, transaction, true);
+			const baseReadTxn = this.#commitBaseTxn(id);
 
 			const write: any = {
 				key: id,
 				store: primaryStore,
 				entry,
+				baseReadTxn,
 				chainsStagedState: true,
 				reloadCommitBase: true,
 				nodeName: (context as any)?.nodeName,
@@ -5371,6 +5456,7 @@ export function makeTable(options): TableResourceClass {
 									isRocksDB && audit && txnLogKey !== txnTime
 										? [{ version: txnLogKey, nodeId: options?.nodeId }]
 										: undefined,
+								localOnly: options?.localOnly,
 							},
 							'delete'
 						);
@@ -5998,8 +6084,12 @@ export function makeTable(options): TableResourceClass {
 				if (target.offset || target.limit !== undefined) results = results.slice(offset, end);
 				results.onDone = () => {
 					results.onDone = null; // ensure that it isn't called twice
+					txn.unregisterReadIterator(results);
 					txn.doneReadTxn();
 				};
+				// Recorded ownership: if the request dies before anything consumes these results, the
+				// transaction closes them itself rather than leaving its read snapshot pinned.
+				txn.registerReadIterator(results);
 				results.selectApplied = true;
 				results.getColumns = getColumns;
 				return results;
@@ -6812,7 +6902,11 @@ export function makeTable(options): TableResourceClass {
 			function failSubscription(error: any) {
 				if (subscription.closed) return;
 				harperLogger.error?.('Error in real-time subscription:', error);
-				subscription.close(error);
+				try {
+					subscription.close(error);
+				} catch (listenerError) {
+					harperLogger.error?.('Error in real-time subscription listener:', listenerError);
+				}
 			}
 			function eventFromAudit(id: Id, auditRecord: any, localTime: number, beginTxn?: boolean) {
 				let type = auditRecord.type;
@@ -6821,7 +6915,7 @@ export function makeTable(options): TableResourceClass {
 					type === 'put' || type === 'patch' || type === 'delete' || type === 'invalidate' || type === 'relocate';
 				if (isMutation && !includeSuperseded) {
 					if (id === undefined) return;
-					const entry: Entry = primaryStore.getEntry(id);
+					const entry = currentEntryForAudit(primaryStore, id, auditRecord);
 					if (!entry || entry.version !== auditRecord.version) return;
 					if (getFullRecord) {
 						value = entry?.value;
@@ -7007,7 +7101,7 @@ export function makeTable(options): TableResourceClass {
 						existingEntry?.value ?? null,
 						existingEntry,
 						txnTime,
-						0,
+						(existingEntry?.metadataFlags ?? 0) & LOCAL_ONLY,
 						true,
 						{
 							user: (context as any)?.user,
@@ -7017,6 +7111,7 @@ export function makeTable(options): TableResourceClass {
 							viaNodeId: options?.viaNodeId,
 							transaction,
 							tableToTrack: tableName,
+							auditLocalOnly: options?.localOnly,
 						},
 						'message',
 						false,
@@ -8684,6 +8779,13 @@ export function makeTable(options): TableResourceClass {
 						// regardless of this state.
 						transaction.next.open = TRANSACTION_STATE.CLOSED;
 					}
+					// A poison flag must travel with `open`, or a link created after the poisoning (a
+					// handler touching this database for the first time post-poison) sees CLOSED but not
+					// the reason, takes save()'s immediateCommit path, and commits on behalf of a request
+					// that was supposed to have been cut off.
+					if (transaction.timedOut) transaction.next.timedOut = true;
+					if (transaction.disconnected) transaction.next.disconnected = true;
+					if (transaction.postSubmitPoisoned) transaction.next.postSubmitPoisoned = true;
 					transaction = transaction.next;
 					transaction.db = primaryStore;
 					return transaction;
@@ -8697,6 +8799,7 @@ export function makeTable(options): TableResourceClass {
 			if (context) {
 				context.transaction = transaction;
 				if (context.timestamp) transaction.timestamp = context.timestamp;
+				if (!context.sourceApply) transaction.requestSignal = context.signal;
 			}
 			return transaction;
 		}

package/resources/analytics/write.ts

@@ -15,6 +15,7 @@ import * as fs from 'node:fs';
 import { getAnalyticsHostnameTable, nodeIds, stableNodeId } from './hostnames.ts';
 import { METRIC } from './metadata.ts';
 import { getTransactionQueueDepths, setCommitLatencyRecorder } from '../DatabaseTransaction.ts';
+import { contextStorage } from '../transaction.ts';
 import { RocksDatabase, type TransactionLogStats } from '@harperfast/rocksdb-js';
 
 const log = forComponent('analytics').conditional;
@@ -126,7 +127,7 @@ export function recordAction(value: Value, metric: string, path?: string, method
 	} else {
 		recordNewAction(key, value, metric, path, method, type);
 	}
-	if (!sendAnalyticsTimeout) sendAnalytics();
+	if (!sendAnalyticsTimeout) contextStorage.exit(sendAnalytics);
 }
 
 server.recordAnalytics = recordAction;
@@ -151,7 +152,10 @@ export function addAnalyticsListener(callback) {
 const IDEAL_PERCENTILES = [0.01, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.99, 0.999, 1];
 
 /**
- * Periodically send analytics data back to the main thread for storage
+ * Periodically send analytics data back to the main thread for storage. Callers arm it through
+ * contextStorage.exit(): the first sample of a period comes from some request, but the flush and the
+ * scheduled tasks it starts are process-wide work, which under that request's context would write as
+ * its user and, once its client disconnected, have every write rejected by its aborted signal.
  */
 function sendAnalytics() {
 	analyticsStart ||= performance.now();

package/resources/audit-retention.md

@@ -0,0 +1,203 @@
+# Audit retention and cleanup
+
+Design notes for the audit/transaction-log retention lifecycle: the floor that makes a short replay
+detectable, the self-rearming cleanup cadence, and the removal-loop rules. Indexed from the root
+[DESIGN.md](../DESIGN.md). Moved out of `resources/DESIGN.md` intact (harper#2711) so that file stays
+under its 1000-line budget; no note's content changed in the move.
+
+## Audit retention floor
+
+`Table.subscribe`'s `startTime` replay just begins wherever the audit log now begins, so a consumer
+resuming below the retention horizon is silently handed a short replay. The floor records what
+pruning removed (harper#2447); it is internal, with deliberately no public accessor. **A resume is
+not checked against it** but against the database generation (next section): this floor also steers
+reconciliation, survives a restore, and absorbs at `Infinity`.
+
+**The one consumer today is not a resume**: `Table.commit`'s out-of-order reconciliation reads the
+floor before entering the audit walk (harper#2642). The walk terminates at the incoming write only by
+reaching an audit entry at or below its version, so below the floor it cannot — it runs the whole
+retained chain, one RocksDB end-of-log scan per step, to an outcome the floor already determines.
+Two things that consumer does differently from a cursor check, and both are deliberate: it compares
+the write's record version rather than a log key, because that is what the walk's own loop condition
+compares; and it treats the `Infinity` unknown floor as **walk anyway** rather than as "not safe",
+because here the conservative direction is to do the work, not to skip it. Below the floor a write
+contributes only its commutative operations — a plain field's survival depends on what newer writes
+did to that key, which is exactly what the pruned history no longer answers.
+
+**The invariant: every path that prunes audit history raises the floor BEFORE removing anything.**
+There are five, and the ordering is the whole guarantee — a floor written after the removal is lost
+if the process dies in between, and the surviving lower floor then certifies a cursor whose history
+is gone. Over-reporting (a floor covering more than the prune actually removed) costs a consumer one
+unnecessary resync; under-reporting loses its data with no signal. So `raiseAuditFloor` is called
+first and a throw from it is what stops the prune.
+
+| Prune path                                                                   | Engine                                                                      |
+| ---------------------------------------------------------------------------- | --------------------------------------------------------------------------- |
+| `scheduleAuditCleanup` retention loop (`auditStore.ts`)                      | LMDB                                                                        |
+| `scheduleAuditCleanup` → `purgeLogs`                                         | RocksDB                                                                     |
+| `purgeAgedLogs` (boot/recovery, called from `replayLogs.ts`)                 | RocksDB                                                                     |
+| `Table.deleteHistory`                                                        | LMDB (`RocksTransactionLogStore.remove()` is a no-op, so it must NOT raise) |
+| `delete_transaction_logs_before` whole-database branch (`ResourceBridge.ts`) | RocksDB                                                                     |
+
+Things that are easy to get wrong here:
+
+- **The floor cannot be derived from the surviving log.** For four of the five paths the oldest
+  surviving entry would do, because they prune a database-wide time prefix. `Table.deleteHistory`
+  removes one table's entries from a database-scoped log, so a sibling's entry survives _below_ the
+  newest entry it removed, and a floor taken from that survivor certifies cursors over removed history.
+- **The record's presence is the trust marker.** `Symbol.for('audit-floor')` is not `last-removed`,
+  which the LMDB retention loop still maintains (#2338 — do not remove it): that one records where the
+  loop got to, after the fact, so a value there cannot be told apart from a write-ahead, verified floor.
+- **A store with no floor record is one whose retention history we cannot account for** (the empty
+  audit store a migration leaves, a table-scoped backup without `include_audit`), so `openAuditStore`
+  stamps `max(Date.now(), newest retained key)` as a one-time resync epoch. There is no
+  permissive-baseline case: creating the audit DBI proves only that it was absent.
+- **That epoch is a guess, and it is recorded as one.** Surviving state cannot see history a legacy
+  selective prune removed, so a rolled-back clock can stamp it below entries that are gone (#2458).
+  Refusing to stamp is worse — `AUDIT_FLOOR_UNKNOWN` is absorbing — so `establishAuditFloor` writes the
+  epoch under `Symbol.for('audit-floor-bootstrap')` first, then stamps the floor from that record.
+
+  **The record's presence is the signal; comparing it against the floor is not.** Worked example: a
+  v4-era `deleteHistory` removes tableA up to t=1000 while tableB's newest survivor is 900; a
+  rolled-back clock stamps bootstrap=floor=900; a later pass raises the floor to 950 — and a cursor at
+  970 still sits over tableA's missing 950–1000. No timestamp can close that window, which is why
+  resumable positions are bound to a generation instead: a position naming one postdates tracking.
+
+  **Ordering:** the record is written first, so a crash between the two writes leaves a record with no
+  floor, which the next open retries. **Undecodable bytes are overwritten** (unlike the floor, where a
+  present record may be a deliberate unknown): keeping them pinned the store to unknown forever.
+
+- **`getHistory` is not in the floor's time domain.** The floor is an audit-log key (`subscribe`'s
+  `localTime`); `getHistory` reports each entry's origin `version` under that name, which a backdated
+  or replicated write makes differ, so its cursors cannot be compared against the floor.
+- **On RocksDB the floor tracks the configured retention horizon, not retained reality.** Whole-log-file
+  purge granularity means the branch cannot know which entries a purge will drop, and the floor is
+  written first, so each pass advances it to `Date.now() - auditRetention/(1+priority²)` whether a
+  file was dropped or not. Entries below that horizon are often still on disk, and a cursor among
+  them is told to resync — conservative in the safe direction only. LMDB can see a single eligible
+  entry, so it raises off the first one it finds instead.
+- **A prune's clamped floor can sit past `Date.now() + 1`.** `boundedAuditPruneEnd` records `newest + 1`
+  when the newest log key is at or past the clock, and log keys are fractional `getNextMonotonicTime`
+  values, not `Date.now()`, so for the rest of that key's millisecond the floor exceeds the wall clock
+  by more than one. Bound it by the newest key, never the wall clock alone (`auditFloor.test.js`, the
+  far-future `deleteHistory` case — it failed twice on main that way).
+- **Untrustworthy metadata resolves to `Infinity`, not to a number.** A wrong-length record, or eight
+  bytes decoding to NaN/negative, must not become a floor: `cursor < NaN` is false, so a consumer
+  spelling the check that way would read corrupt metadata as safe.
+- **A copy keeps this floor honest for the log it carried.** A restore carries its log, so the floor
+  stands; a branch or migration carries none, so its stamp raises a finite floor to the copy's epoch.
+  Which history the database is belongs to the generation.
+
+---
+
+## Audit-entry removal loops must track every `removeAuditEntry()`/`removeEntry()` promise
+
+`scheduleAuditCleanup` (`auditStore.ts`) and `Table.deleteHistory` (`Table.ts`, the LMDB path behind
+`delete_transaction_logs_before`) both iterate a range of audit records and remove each one. Any loop
+that removes audit/primary-store entries in a batch must attach a rejection handler to every removal
+immediately and drain all tracked promises before returning — never stash a per-iteration promise in
+an outer variable to await only the last one: an overwritten promise's rejection is never awaited or
+caught, and surfaces later as an unhandled rejection with no log to explain it. `Table.deleteHistory`
+allows up to 1,000 LMDB removals in flight
+(ten for RocksDB) so storage writes batch without growing an unbounded pending
+set. Live removals are tracked in a `Set`, and each one removes itself and wakes at most one parked
+producer when it settles, so any completion releases the loop. In these removal loops, do not repeatedly
+race the live set: each race attaches another reaction to every long-pending removal. Both phases drain
+their tracked removals before settling, including when iteration throws. `scheduleAuditCleanup` remains
+sequential because it is an automatic background loop, and it ends a pass at its first failed removal
+rather than continuing past it: its `deleted` count (the backoff input) and the `last-removed` marker it
+writes then cover a contiguous removed prefix — nothing above that marker was removed by the pass — and
+the next pass retries the failed entry first. Past a failure there is no key the marker can truthfully
+record, and counting failures as progress re-armed an all-failing pass at 10 ms. The guarantee is
+pass-local: the marker is written after its removals, `deleteHistory` never writes it, and a marker an
+older build persisted past a failed entry is not repaired; completeness is the audit floor's question.
+Stopping costs liveness only for an entry that fails on every pass, which is why `removeAuditEntry` must
+fail only when `auditStore.remove()` does (below). Regression: `auditLog.test.js` "ends a pass at a failed
+removal".
+
+Individual removal failures are logged and excluded from the returned count, but a purge that attempted
+at least one removal and completed none rejects with the first error after both phases have drained.
+Without that, `delete_transaction_logs_before` reports a successful `entries_deleted: 0` whether nothing
+was eligible or the store rejected every write, and an operator pruning to bound disk growth has no signal
+that pruning did nothing. Drain first, then decide: a failing store should still get every removal it can
+accept, and a single success means the purge made progress and reports normally.
+
+The optional primary-store cleanup snapshots each tombstone's key and version before yielding and passes
+that version to `remove()`. LMDB enforces the condition natively. Harper's RocksDB adapter re-reads and
+removes inside one native transaction, retrying a conflict once, because rocksdb-js's `remove()` accepts
+an options object rather than an LMDB-style version argument. Never replace this with a separate live read
+followed by an unconditional remove: a record recreated between those operations would be deleted.
+
+`removeAuditEntry` has a second, nested version of the same hazard: for a `'delete'`-type audit record it
+also invokes a per-table delete callback (`addDeleteRemovalCallback`) that removes the corresponding
+primary-store tombstone. That callback's promise must be returned and joined with the audit-store
+removal (currently via `Promise.all`, with the callback's own rejection — and a throwing tombstone
+lookup, such as lmdb-js `getEntry(undefined)` for an undecodable recordId — caught and logged through
+`warnContained`, so neither a failed tombstone cleanup nor a throwing log sink gets misreported as a failed
+audit-entry removal) — otherwise the tombstone removal is fire-and-forget and the same detached-rejection
+hazard reappears one level down.
+A tombstone whose cleanup fails this way is not swept automatically — `scheduleAuditCleanup`'s automatic
+pass never retries it, since the audit entry that would have triggered a retry is already gone. It sits
+in the primary store until an operator runs `delete_transaction_logs_before` with `cleanup_deleted_records: true`.
+
+## Audit retention cleanup is a self-rearming, engine-independent lifecycle
+
+One call to `scheduleAuditCleanup` establishes a retention cadence that ends when the root store closes
+(or immediately in process-wide read-only mode). Storage-engine selection changes the work inside each pass, not whether the timer,
+serialization barrier, error containment, and re-arm exist. LMDB removes bounded batches of audit
+entries; RocksDB asks rocksdb-js to purge conservatively eligible log segments before the same time
+cutoff. Disk-pressure callbacks may accelerate the next pass and shorten the effective window, but
+ordinary retention progress must not depend on pressure.
+
+The two engines do not share a cadence rule, because their units of progress differ. LMDB's adaptive
+backoff reads a per-entry delete count: it speeds up while entries are being removed and doubles while
+idle. Rocks reclaims whole segments whose eligibility changes only on rotation/flush, so the same
+signal would only make it rescan the same files — its delay is instead a pure function of the
+pressure-adjusted retention window (a tenth of it, floored at `DEFAULT_AUDIT_CLEANUP_DELAY`).
+
+Exactly one Rocks purge loop exists per store, and that is owned by the **arming** sites, not the
+re-arm: `onStorageReclamation` registers its handler only on the last worker (it takes no
+`skipThreadCheck`), and the store-open arm gates on the same index. The last-worker conjunct on the
+re-arm is therefore unreachable through either of those paths; it is a backstop for a direct caller
+of the exported `scheduleAuditCleanup`, because a store-wide segment purge looping on every worker is
+duplicated work. If a future change passes `skipThreadCheck: true` at the registration site, that
+backstop — not the registration — becomes the thing keeping the loop single.
+
+Both re-arm guards are **Rocks-only**. The LMDB arm re-arms unconditionally, so it neither yields to
+an already-pending pass (a pressure-armed 100ms pass can be cancelled and replaced by the idle
+backoff) nor restricts itself to one worker — pre-existing LMDB behavior, not an invariant.
+
+Two things a purge does **not** need to coordinate, both load-bearing for the continuous cadence.
+Unlinking a segment a consumer has mapped is safe **on POSIX**: the inode outlives the unlink, and the
+mapping cache (`_logBuffers`) holds `WeakRef`s, with a strong ref only on the newest segment, which is
+never purge-eligible — so nothing pins a purged inode and no cross-worker cache invalidation is
+required. Windows does not share that property: deleting a mapped segment raises a sharing violation,
+so the purge throws, is warn-logged, re-arms, and makes no progress for as long as a consumer holds the
+mapping. The continuous cadence therefore turns a Windows retention stall into a steady state rather
+than a one-off, and nothing covers it — the Rocks retention integration test skips win32.
+What is _not_ covered is the segment a lagging consumer has not mapped yet: `TransactionLog.query()`'s
+iterator returns `done` when its next segment cannot be mapped, indistinguishable from being caught up
+(rocksdb-js `src/transaction-log-reader.ts`). A consumer that far behind needs a full copy rather than
+log replay, so the gap is a missing escalation signal in the reader, not a reason to hold retention —
+tracked as HarperFast/rocksdb-js#805. Continuous retention is what moves it from unreachable-in-steady-state
+to routine: a peer offline longer than `logging.auditRetention` now resumes into a purged prefix and is
+recorded as caught up, and `txnlogReplayGapBytes` observes the gap without escalating on it.
+
+Retirement is two things, and teardown needs both. `stopAuditCleanup()` latches the loop closed and
+cancels the pending timer, and it **returns a drain barrier** — a promise that settles once the pass
+already running has finished. The barrier is what makes closing stores safe: lmdb-js stamps the DBI
+number into its write instruction synchronously and the native writer consumes it later
+(`node_modules/lmdb/write.js`), so a pass suspended inside `await removeAuditEntry()` still has a
+delete pending against the primary and audit DBIs, and LMDB forbids closing a DBI an existing
+transaction has modified. `dropDatabase()` and the legacy arm of `Table.dropTable()` await it.
+`closeDatabase()` and branch `close()` close commit admission, then drain tracked Rocks transactions,
+table maintenance, audit cleanup, and derived-index work before closing stores. Every environment
+touch remaining in a resumed cleanup pass — cursor advance, cursor release, marker write, re-arm —
+also re-checks `rootStore.status`. `resetDatabases()` closes LMDB roots with no retirement call at all,
+so that re-check is a routine
+path rather than a defensive one.
+
+The last-removed marker is retained until it commits. A rejected write is logged and carried to the
+next pass rather than dropped: a pass that deletes nothing never reaches the write again, so one
+transient failure would otherwise leave the recorded boundary permanently behind the entries that
+were already removed.