@harperfast/harper 5.2.9 → 5.3.0-alpha.1

package/resources/recordLock.ts

@@ -0,0 +1,253 @@
+import harperLogger from '../utility/logging/harper_logger.ts';
+import { ClientError } from '../utility/errors/hdbError.ts';
+
+/**
+ * Exclusive per-record locks (harper#483, Phase 0: one node, every worker thread).
+ *
+ * The sole authority is the rocksdb-js process-wide key lock: one `locks` map per DBDescriptor
+ * shared by every worker thread's handle. Nothing is written to the store or audit log. Lock and
+ * unlock are pure in-memory operations; the record's version and bytes are unchanged.
+ */
+
+export const DEFAULT_LOCK_LEASE_MS = 30_000;
+/** Below this a generation can expire before the caller's work completes, handing back a lock already lost. */
+export const MIN_LOCK_LEASE_MS = 100;
+export const MAX_LOCK_LEASE_MS = 300_000;
+export const DEFAULT_LOCK_TIMEOUT_MS = 30_000;
+export const MAX_LOCK_TIMEOUT_MS = 300_000;
+const LOCK_KEY_PREFIX = Symbol.for('record-lock');
+
+export interface RecordLockOptions {
+	/** Lease in ms; the lock expires on its own if the holder never releases it (default 30 s, max 5 min) */
+	lease?: number;
+	/** How long to wait for a contended lock in ms before failing with 423 (default 30 s, max 5 min) */
+	timeout?: number;
+	/** A held lock outlives the acquiring transaction; it is released by `unlock()` or by its lease */
+	hold?: boolean;
+}
+
+export interface ResolvedRecordLockOptions {
+	lease: number;
+	timeout: number;
+	hold: boolean;
+}
+
+/**
+ * The handle returned by `lock()` and held by each gated write until its transaction completes.
+ * `release()` is synchronous: it calls `store.unlock(key)` which is ownerless and fires any
+ * waiting callbacks cross-thread. The `expired` flag is set in the holder thread by the lease
+ * timer, so a stale holder cannot accidentally call unlock after its lease already fired and
+ * released the key to a new holder.
+ */
+export interface RecordLockHandle {
+	store: any;
+	key: any[]; // lockAttemptKey, passed to store.tryLock / store.unlock
+	keyId: unknown; // writeKeyId(id), used to scan the per-transaction recordLocks array
+	expiresAt: number;
+	hold: boolean;
+	released: boolean;
+	expired: boolean;
+	/** Monotonic timestamp at which tryLock succeeded; used as the base version for holder writes. */
+	acquiredAt: number;
+	/** Advance, but never lower, the version floor from the transaction that owns the actual write timestamp. */
+	noteHolderVersion(version: number): void;
+	/** Preserve a caller timestamp as a candidate floor without treating it as a committed write. */
+	noteCandidateFloor(version: number): void;
+	/** Compute a holder-write version without advancing the floor until that write commits. */
+	holderVersionCandidate(floor?: number): number;
+	/**
+	 * Return the next version to stamp a holder write with. Starts at acquiredAt when no transaction
+	 * version has been noted; otherwise advances by the minimum monotonic step from the recorded floor.
+	 */
+	nextHolderVersion(): number;
+	release(): boolean;
+	/**
+	 * Flip this handle to hold mode and reset its lease timer. Used during scoped→hold upgrades
+	 * so all instances that reference this handle object remain valid (no retire + replace).
+	 */
+	upgradeToHold(lease: number): void;
+}
+
+function requireDuration(name: string, value: unknown, fallback: number, min: number, max: number): number {
+	if (value === undefined) return fallback;
+	if (typeof value !== 'number' || !Number.isFinite(value) || value < min)
+		throw new ClientError(
+			`Lock option ${name} must be a number of milliseconds of at least ${min}, but received ${value}`
+		);
+	if (value > max) throw new ClientError(`Lock option ${name} must not exceed ${max}ms, but received ${value}`);
+	return value;
+}
+
+export function resolveLockOptions(options?: RecordLockOptions | null): ResolvedRecordLockOptions {
+	if (options != null && typeof options !== 'object')
+		throw new ClientError(`Lock options must be an object, but received ${typeof options}`);
+	const hold = options?.hold ?? false;
+	if (typeof hold !== 'boolean') throw new ClientError(`Lock option hold must be a boolean, but received ${hold}`);
+	return {
+		lease: requireDuration('lease', options?.lease, DEFAULT_LOCK_LEASE_MS, MIN_LOCK_LEASE_MS, MAX_LOCK_LEASE_MS),
+		timeout: requireDuration('timeout', options?.timeout, DEFAULT_LOCK_TIMEOUT_MS, 1, MAX_LOCK_TIMEOUT_MS),
+		hold,
+	};
+}
+
+/**
+ * The 409 for a write staged through a handle that is no longer live. A lease that ran out and a
+ * lock that was handed back are different causes, and a caller told the wrong one debugs the wrong
+ * thing — a commit-released scoped handle is not a timeout.
+ */
+export function lockNotHeldError(handle: Pick<RecordLockHandle, 'expired'>): ClientError {
+	return new ClientError(handle.expired ? 'Record lock lease expired' : 'Record lock already released', 409);
+}
+
+/** The advisory key for this (table, record) pair, distinct from getFromSource's bare-id single-flight lock. */
+export function lockAttemptKey(tableId: number, id: any): any[] {
+	return Array.isArray(id) ? [LOCK_KEY_PREFIX, tableId, ...id] : [LOCK_KEY_PREFIX, tableId, id];
+}
+
+const warnedLeaseTimerStores = new WeakSet();
+
+class KeyLockHandle implements RecordLockHandle {
+	store: any;
+	key: any[];
+	keyId: unknown;
+	expiresAt: number;
+	hold: boolean;
+	released = false;
+	expired = false;
+	acquiredAt: number;
+	#lastHolderVersion: number | undefined;
+	#candidateFloor: number | undefined;
+	#timer: ReturnType<typeof setTimeout> | undefined;
+
+	constructor(store: any, key: any[], keyId: unknown, lease?: number, hold = false, acquiredAt?: number) {
+		this.store = store;
+		this.key = key;
+		this.keyId = keyId;
+		this.expiresAt = lease != null ? Date.now() + lease : Infinity;
+		this.hold = hold;
+		this.acquiredAt = acquiredAt ?? store.getMonotonicTimestamp();
+		if (lease != null) {
+			this.#timer = setTimeout(() => this.#onLeaseExpire(), lease).unref();
+		}
+	}
+
+	noteHolderVersion(version: number): void {
+		this.#lastHolderVersion = Math.max(this.#lastHolderVersion ?? version, version);
+	}
+
+	noteCandidateFloor(version: number): void {
+		this.#candidateFloor = Math.max(this.#candidateFloor ?? version, version);
+	}
+
+	holderVersionCandidate(floor?: number): number {
+		// A small nonzero step prevents exact LWW ties without materially moving past the observed floor.
+		const MIN_STEP = 0.000488;
+		return Math.max(
+			this.#lastHolderVersion != null ? this.#lastHolderVersion + MIN_STEP : this.acquiredAt,
+			this.acquiredAt,
+			this.#candidateFloor ?? -Infinity,
+			floor ?? -Infinity
+		);
+	}
+
+	nextHolderVersion(): number {
+		const next = this.holderVersionCandidate();
+		this.#lastHolderVersion = next;
+		return next;
+	}
+
+	#onLeaseExpire() {
+		if (this.released) return;
+		this.released = true;
+		this.expired = true;
+		try {
+			this.store.unlock(this.key);
+		} catch (err) {
+			if (!warnedLeaseTimerStores.has(this.store)) {
+				warnedLeaseTimerStores.add(this.store);
+				harperLogger.warn?.('record lock lease timer failed (store may have been dropped)', err);
+			}
+		}
+	}
+
+	release(): boolean {
+		if (this.released) return false;
+		this.released = true;
+		clearTimeout(this.#timer);
+		try {
+			this.store.unlock(this.key);
+		} catch (err) {
+			if (!warnedLeaseTimerStores.has(this.store)) {
+				warnedLeaseTimerStores.add(this.store);
+				harperLogger.warn?.('record lock release failed (store may have been dropped)', err);
+			}
+		}
+		return true;
+	}
+
+	upgradeToHold(lease: number): void {
+		this.hold = true;
+		clearTimeout(this.#timer);
+		this.expiresAt = Date.now() + lease;
+		this.#timer = setTimeout(() => this.#onLeaseExpire(), lease).unref();
+	}
+}
+
+export function makeKeyLockHandle(
+	store: any,
+	key: any[],
+	keyId: unknown,
+	lease?: number,
+	hold = false,
+	acquiredAt?: number
+): RecordLockHandle {
+	return new KeyLockHandle(store, key, keyId, lease, hold, acquiredAt);
+}
+
+/**
+ * Acquire the native key lock for a record, looping until acquired or deadline passes. Re-entrant
+ * within a transaction: if the transaction already holds this key (non-expired handle in its
+ * recordLocks map) the existing handle is returned without a second tryLock.
+ */
+export async function acquireRecordKey(
+	txn: { recordLockFor(store: any, keyId: unknown): RecordLockHandle | undefined },
+	store: any,
+	key: any[],
+	keyId: unknown,
+	waitMs: number,
+	lease?: number,
+	hold = false
+): Promise<RecordLockHandle> {
+	const existing = txn.recordLockFor(store, keyId);
+	if (existing && !existing.released && !existing.expired) return existing;
+
+	const deadline = Date.now() + waitMs;
+	// One persistent wake slot: at most one onUnlocked callback is registered per waiter at any
+	// time.  After the callback fires (holder released), hasCallback is cleared so the next
+	// tryLock can register a fresh callback for the new holder.  When the timeout fires instead
+	// (the holder has not released yet), hasCallback stays true and subsequent retries skip the
+	// callback registration, preventing unbounded accumulation on a hot key.
+	let wakeResolve: (() => void) | undefined;
+	let hasCallback = false;
+	while (true) {
+		const onUnlocked = hasCallback
+			? undefined
+			: () => {
+					hasCallback = false;
+					wakeResolve?.();
+				};
+		const acquired = store.tryLock(key, onUnlocked);
+		if (acquired) return makeKeyLockHandle(store, key, keyId, lease, hold, store.getMonotonicTimestamp());
+		if (onUnlocked) hasCallback = true; // callback now registered; do not register another until it fires
+		const remaining = deadline - Date.now();
+		if (remaining <= 0) throw new ClientError(`Record is locked and was not released in time`, 423);
+		await new Promise<void>((resolve) => {
+			const t = setTimeout(resolve, remaining).unref();
+			wakeResolve = () => {
+				clearTimeout(t);
+				resolve();
+			};
+		});
+		wakeResolve = undefined;
+	}
+}

package/resources/replayLogs.ts

@@ -40,13 +40,36 @@ export function structuresWouldShrink(existing: any, updated: any): boolean {
 	return false;
 }
 
-export function replayLogs(rootStore: RocksDatabase, tables: any): Promise<void> {
-	if (!isMainThread) return; // ideally we don't do it like this, but for now this is predictable
-	return new Promise((resolve) => {
+/** The total wall-clock budget (ms) a replay may spend, as configured for this process. */
+export function replayTimeBudgetMs(): number {
+	const configuredReplayTimeout = Number(envGet(CONFIG_PARAMS.REPLICATION_REPLAYTIMEOUT));
+	return configuredReplayTimeout > 0 ? configuredReplayTimeout : REPLAY_WALL_CLOCK_LIMIT_MS;
+}
+
+/**
+ * `electedReplayer` marks a caller that has already won a cross-thread election for this store —
+ * the branch claim (branchDatabase.ts) — and so may replay off the main thread. It also makes the
+ * replay strict: the promise settles (never hangs), and a failure to apply the tail — a commit or
+ * write error, a stall or wall-clock abort — rejects instead of quietly serving a rewound store.
+ * Undecodable entries and a torn tail stay tolerated in both modes: a crash tears the last frame of
+ * a log by construction, and end-of-log is the designed reading of it (see replayLogsGuards.ts). A
+ * **mid-log** break is different — entries behind it were acknowledged and are now quarantined — so
+ * an elected replay rejects on one rather than publish a branch as a trustworthy point-in-time copy
+ * over known lost writes; boot replay still logs it and continues (harper#2016, harper#2063).
+ */
+export function replayLogs(rootStore: RocksDatabase, tables: any, electedReplayer?: boolean): Promise<void> {
+	if (!isMainThread && !electedReplayer) return Promise.resolve(); // ideally we don't do it like this, but for now this is predictable
+	return new Promise((resolve, reject) => {
 		const acquired = rootStore.tryLock('replayLogs', async () => {
 			resolve();
 		});
-		if (!acquired) return;
+		if (!acquired) {
+			// An elected replayer is the store's sole opener, so a held lock is a protocol violation;
+			// hanging its awaited promise would wedge the branch claim in CREATING. The boot-scan
+			// caller keeps the settle-on-unlock semantics (nothing awaits it).
+			if (electedReplayer) reject(new Error(`The replay lock for ${(rootStore as any).databaseName} is already held`));
+			return;
+		}
 		// Shed transaction-log files already older than the audit retention window before
 		// replaying. A node that crash-loops during recovery never reaches the steady-state
 		// cleanup loop, so without this its aged backlog only grows and enlarges each subsequent
@@ -75,9 +98,13 @@ export function replayLogs(rootStore: RocksDatabase, tables: any): Promise<void>
 			tableById.set(table.tableId, table);
 		}
 		// replay all the logs
-		let transaction: DatabaseTransaction;
+		let transaction: DatabaseTransaction | undefined;
 		let lastTimestamp = 0;
 		let writes = 0;
+		// Writes staged on the currently-open transaction, so a discard can be taken back out of `writes`.
+		let stagedWrites = 0;
+		// Records dropped because a corrupt frame truncated the transaction they belong to.
+		let discardedWrites = 0;
 		let skipped = 0;
 		// Track forward progress so a backlog of unwritable entries can't grind the boot thread
 		// forever (harper#1266). `noProgressRun` counts every entry processed without a successful
@@ -89,13 +116,33 @@ export function replayLogs(rootStore: RocksDatabase, tables: any): Promise<void>
 		let lastProgressTime = replayStartTime;
 		// Total wall-clock budget (ms) for replay, configurable via `replication.replayTimeout`;
 		// falls back to the 10-minute default (harper#1316).
-		const configuredReplayTimeout = Number(envGet(CONFIG_PARAMS.REPLICATION_REPLAYTIMEOUT));
-		const replayTimeoutMs = configuredReplayTimeout > 0 ? configuredReplayTimeout : REPLAY_WALL_CLOCK_LIMIT_MS;
+		const replayTimeoutMs = replayTimeBudgetMs();
+		// Set on the first unrecoverable failure in elected mode; rejects below instead of publishing
+		// a store the replay knows is incomplete.
+		let strictFailure: Error | undefined;
+		const strictAbort = (error: Error, staged: DatabaseTransaction | undefined): void => {
+			try {
+				staged?.abort();
+			} catch (abortError) {
+				logger.warn('Error aborting a strict replay transaction', abortError);
+			}
+			strictFailure = error;
+		};
 		const txnLog: RocksTransactionLogStore = (rootStore as any).auditStore;
-		for (const auditRecord of txnLog.getRange({ startFromLastFlushed: true, readUncommitted: true }) as any) {
+		const entries = txnLog.getRange({
+			startFromLastFlushed: true,
+			readUncommitted: true,
+			trackCorruptTransactions: true,
+		});
+		for (const auditRecord of entries as any) {
 			if (noProgressRun > 0 && shouldAbortStalledReplay(noProgressRun, performance.now() - lastProgressTime)) {
+				const stallDiagnostic = `Aborting transaction-log replay in ${(rootStore as any).databaseName} database: ${noProgressRun} consecutive audit entries with no successful write (${skipped} skipped as unrecoverable, ${writes} replayed so far). This backlog is making no forward progress and was blocking startup (harper#1266) — typically a peer transaction log whose values reference unresolvable shared structures (harper#1163), or a backlog for a dropped table.`;
+				if (electedReplayer) {
+					strictAbort(new Error(stallDiagnostic), transaction);
+					break;
+				}
 				logger.fatal(
-					`Aborting transaction-log replay in ${(rootStore as any).databaseName} database: ${noProgressRun} consecutive audit entries with no successful write (${skipped} skipped as unrecoverable, ${writes} replayed so far). This backlog is making no forward progress and was blocking startup (harper#1266) — typically a peer transaction log whose values reference unresolvable shared structures (harper#1163), or a backlog for a dropped table. Continuing boot without replaying the remainder; shed or relocate the oversized/undecodable peer transaction log(s), or re-clone this node, to recover the unreplayed data.`
+					`${stallDiagnostic} Continuing boot without replaying the remainder; shed or relocate the oversized/undecodable peer transaction log(s), or re-clone this node, to recover the unreplayed data.`
 				);
 				break;
 			}
@@ -105,6 +152,7 @@ export function replayLogs(rootStore: RocksDatabase, tables: any): Promise<void>
 				nodeId,
 				recordId,
 				version,
+				txnLogKey,
 				residencyId,
 				expiresAt,
 				originatingOperation,
@@ -165,14 +213,35 @@ export function replayLogs(rootStore: RocksDatabase, tables: any): Promise<void>
 					warnedReplayHappening = true;
 					console.warn('Harper was not properly shutdown, replaying transaction logs to synchronize database');
 				}
-				if (lastTimestamp !== version) {
-					lastTimestamp = version;
+				// Transactions are delimited by the log key — which is also what `truncatedVersions` records —
+				// and each write is replayed at its own stored record version. The two differ for a source
+				// fill, and stamping such a record at the log key would move its version forward and make a
+				// later legitimate write in between look stale (harper#2411).
+				if (lastTimestamp !== txnLogKey) {
+					const torn = entries.corruptFrameStop.truncatedVersions.has(lastTimestamp);
+					lastTimestamp = txnLogKey;
 					try {
-						// commit the last transaction since we are starting a new one
-						transaction?.directCommitSync();
+						// commit the last transaction since we are starting a new one, unless a corrupt
+						// frame swallowed the rest of it — half of a source transaction must never become
+						// durable, so it is dropped whole and stays in the log for a repaired retry
+						if (torn) {
+							writes -= stagedWrites;
+							discardedWrites += stagedWrites;
+							transaction?.abort();
+						} else transaction?.directCommitSync();
 					} catch (error) {
-						logger.error('Error committing replay transaction', error);
+						// directCommitSync aborts and detaches its transaction on failure; no cleanup here.
+						// The torn branch already backed stagedWrites out of writes before this try, so
+						// only a failed COMMIT (never applied) needs it backed out here too — otherwise a
+						// commit failure left `writes` counting records that never actually landed.
+						if (!torn) writes -= stagedWrites;
+						if (electedReplayer) {
+							strictFailure = error;
+							break;
+						}
+						logger.error(`Error ${torn ? 'discarding a torn' : 'committing'} replay transaction`, error);
 					}
+					stagedWrites = 0;
 					// Abort if replay has exceeded the total wall-clock budget even while making progress
 					// (harper#1316, facet a). shouldAbortStalledReplay resets its counters on every write,
 					// so a slow-but-progressing replay (deep out-of-order audit chain walk per entry) can
@@ -181,15 +250,18 @@ export function replayLogs(rootStore: RocksDatabase, tables: any): Promise<void>
 					// is not yet staged, so aborting never tears a same-version (same source-transaction)
 					// write batch in half. Re-clone to recover the unreplayed remainder.
 					if (shouldAbortSlowReplay(performance.now() - replayStartTime, replayTimeoutMs)) {
-						logger.fatal(
-							`Aborting transaction-log replay in ${(rootStore as any).databaseName} database: replay has exceeded the wall-clock time limit (${writes} written, ${skipped} skipped). The transaction log contains a pathologically deep out-of-order write history that is too expensive to reconcile during boot (harper#1316). Re-clone this node from a healthy leader to recover the unreplayed data.`
-						);
-						transaction = undefined as any; // already committed above; nothing staged for the new version
+						const slowMessage = `Aborting transaction-log replay in ${(rootStore as any).databaseName} database: replay has exceeded the wall-clock time limit (${writes} written, ${skipped} skipped). The transaction log contains a pathologically deep out-of-order write history that is too expensive to reconcile during boot (harper#1316). Re-clone this node from a healthy leader to recover the unreplayed data.`;
+						transaction = undefined; // already committed above; nothing staged for the new version
+						if (electedReplayer) {
+							strictFailure = new Error(slowMessage);
+							break;
+						}
+						logger.fatal(slowMessage);
 						break;
 					}
 					transaction = new DatabaseTransaction();
 					transaction.db = primaryStore;
-					transaction.timestamp = version;
+					transaction.timestamp = txnLogKey;
 					// retries=1 routes operation.commit() through its retry path (no duplicate audit staging)
 					transaction.retries = 1;
 					// Explicit replay marker: skips schema validation (harper#1316) and makes save() stamp
@@ -198,8 +270,9 @@ export function replayLogs(rootStore: RocksDatabase, tables: any): Promise<void>
 					transaction.isReplay = true;
 				}
 				context.transaction = transaction;
-				const options = { context, residencyId, nodeId, originatingOperation };
+				const options = { context, residencyId, nodeId, originatingOperation, version };
 				writes++;
+				stagedWrites++;
 				switch (type) {
 					case 'put':
 						tableInstance._writeUpdate(recordId, record, true, options);
@@ -274,6 +347,10 @@ export function replayLogs(rootStore: RocksDatabase, tables: any): Promise<void>
 				noProgressRun = 0;
 				lastProgressTime = performance.now();
 			} catch (err) {
+				if (electedReplayer) {
+					strictAbort(err, transaction);
+					break;
+				}
 				// A write that threw made no forward progress either — count it toward the stall
 				// bound so a continuous stream of throwing writes can't grind the boot thread
 				// indefinitely (and the per-entry error log below can't spam unboundedly). harper#1266
@@ -283,18 +360,62 @@ export function replayLogs(rootStore: RocksDatabase, tables: any): Promise<void>
 				});
 			}
 		}
-		try {
-			transaction?.directCommitSync();
-		} catch (error) {
-			logger.error('Error committing replay transaction', error);
+		const finalTorn = entries.corruptFrameStop.truncatedVersions.has(lastTimestamp);
+		if (!strictFailure) {
+			try {
+				if (finalTorn) {
+					writes -= stagedWrites;
+					discardedWrites += stagedWrites;
+					transaction?.abort();
+				} else transaction?.directCommitSync();
+			} catch (error) {
+				// directCommitSync aborts and detaches its transaction on failure; no cleanup here.
+				// Mirrors the interior version-boundary catch above: a failed commit never applied, so
+				// it must not stay counted in `writes`.
+				if (!finalTorn) writes -= stagedWrites;
+				if (electedReplayer) strictFailure = error;
+				logger.error(`Error ${finalTorn ? 'discarding a torn' : 'committing'} replay transaction`, error);
+			}
+		}
+		// `breaks` also counts a torn tail — the designed, benign reading of a crash's last frame, and
+		// the reporter already logged it at `warn`. Only a mid-log break lost entries, so it alone earns
+		// this quarantine/repair summary; duplicating it for a torn tail would tell an operator to
+		// repair or re-clone the node on every ordinary unclean shutdown.
+		if (entries.corruptFrameStop.midLogBreak) {
+			logger.error(
+				`Transaction-log replay in ${(rootStore as any).databaseName} database stopped at a corrupt entry after replaying ${writes} records. Every entry after the break is quarantined — neither replayed nor replicated — and ${discardedWrites} record(s) of the transaction the break truncated were discarded rather than applied in part. Repair the transaction log or re-clone this node to recover them.`
+			);
+			// electedReplayer must not resolve past known loss — see the doc comment above. Only set
+			// strictFailure if nothing has already failed the replay for an unrelated reason, so this
+			// diagnostic never shadows a more specific commit/write error.
+			if (electedReplayer && !strictFailure) {
+				strictFailure = new Error(
+					`Elected replay in ${(rootStore as any).databaseName} database stopped at a mid-log corrupt transaction-log frame; entries behind the break were acknowledged and are now quarantined. Refusing to publish a branch over known lost writes — repair the transaction log or re-clone this node.`
+				);
+			}
 		}
 		if (writes > 0) logger.warn(`Replayed ${writes} records in ${(rootStore as any).databaseName} database`);
 		if (skipped > 0)
 			logger.warn(
 				`Skipped ${skipped} unrecoverable audit entries in ${(rootStore as any).databaseName} database during replay`
 			);
-		// we never actually release the lock because we only want to ever run one time
-		// rootStore.unlock('replayLogs');
+		if (strictFailure) {
+			// A failed strict replay must be re-runnable: the branch claim resets for a retry, so the
+			// lock must not be what survives to wedge it. Only a COMPLETED replay keeps the lock
+			// forever (the once-per-boot guarantee). Reject BEFORE unlocking: unlock wakes tryLock
+			// callbacks — including this holder's own resolve() — and the first settle must be the
+			// rejection, or a failed replay reads as success.
+			reject(strictFailure);
+			try {
+				rootStore.unlock('replayLogs');
+			} catch (unlockError) {
+				logger.warn('Error releasing the replay lock after a failed replay', unlockError);
+			}
+		} else {
+			// we never actually release the lock because we only want to ever run one time
+			// rootStore.unlock('replayLogs');
+			resolve();
+		}
 	});
 }
 function asBinary(buffer) {