@harperfast/harper 5.3.0-beta.3 → 5.3.0-beta.4

package/resources/derivedIndexRegistry.ts

@@ -1,5 +1,127 @@
+import { DerivedIndexLagError } from '../utility/errors/hdbError.ts';
+
 const registrations = new WeakMap<object, Map<number, number>>();
 const admissions = new WeakMap<object, Map<number, Array<() => string | undefined>>>();
+type LockStore = { status?: string; tryLock(key: string): boolean; unlock(key: string): void };
+type FenceWaitOptions = { shouldContinue?: () => boolean; timeoutMilliseconds?: number };
+const DEFAULT_FENCE_WAIT_MILLISECONDS = 70_000;
+
+function fullTextClearLockKey(tableId: number): string {
+	return `derived-index:fulltext:${tableId}:table-clear`;
+}
+
+function fullTextRetirementLockKey(tableName: string): string {
+	return `derived-index:fulltext:${tableName}:retirement`;
+}
+
+/** Hold this node-wide fence for the full duration of an asynchronous table clear. */
+export function acquireFullTextClearFence(rootStore: LockStore, tableId: number): (() => void) | undefined {
+	const key = fullTextClearLockKey(tableId);
+	if (!rootStore.tryLock(key)) return;
+	let held = true;
+	return () => {
+		if (!held) return;
+		held = false;
+		rootStore.unlock(key);
+	};
+}
+
+/** Probe without waiting; callers hold the schema lock so a clear cannot start between this and persistence. */
+export function fullTextClearInProgress(rootStore: LockStore, tableId: number): boolean {
+	const release = acquireFullTextClearFence(rootStore, tableId);
+	if (!release) return true;
+	release();
+	return false;
+}
+
+/** Wait until the current clear releases its fence; callers retry admission afterwards. */
+export function waitForFullTextClear(
+	rootStore: LockStore,
+	tableId: number,
+	options?: FenceWaitOptions
+): Promise<boolean> {
+	return waitForFence(rootStore, () => acquireFullTextClearFence(rootStore, tableId), 'table clear', options);
+}
+
+/** Fence same-name recreation while a dropped table's native directories are being retired. */
+export function acquireFullTextRetirementFence(rootStore: LockStore, tableName: string): (() => void) | undefined {
+	const key = fullTextRetirementLockKey(tableName);
+	if (!rootStore.tryLock(key)) return;
+	let held = true;
+	return () => {
+		if (!held) return;
+		held = false;
+		rootStore.unlock(key);
+	};
+}
+
+/** Probe under the schema lock so recreation cannot pass between the probe and catalog persistence. */
+export function fullTextRetirementInProgress(rootStore: LockStore, tableName: string): boolean {
+	const release = acquireFullTextRetirementFence(rootStore, tableName);
+	if (!release) return true;
+	release();
+	return false;
+}
+
+/** Wait until the current retirement releases its fence; the caller must recheck its generation afterwards. */
+export function waitForFullTextRetirement(
+	rootStore: LockStore,
+	tableName: string,
+	options?: FenceWaitOptions
+): Promise<boolean> {
+	return waitForFence(
+		rootStore,
+		() => acquireFullTextRetirementFence(rootStore, tableName),
+		`retirement of full-text storage for '${tableName}'`,
+		options
+	);
+}
+
+export function waitForFullTextRetirementLease(
+	rootStore: LockStore,
+	tableName: string,
+	options?: FenceWaitOptions
+): Promise<(() => void) | undefined> {
+	return waitForFenceLease(
+		rootStore,
+		() => acquireFullTextRetirementFence(rootStore, tableName),
+		`retirement of full-text storage for '${tableName}'`,
+		options
+	);
+}
+
+async function waitForFence(
+	rootStore: LockStore,
+	acquire: () => (() => void) | undefined,
+	description: string,
+	options: FenceWaitOptions = {}
+): Promise<boolean> {
+	const release = await waitForFenceLease(rootStore, acquire, description, options);
+	if (!release) return false;
+	release();
+	return true;
+}
+
+async function waitForFenceLease(
+	rootStore: LockStore,
+	acquire: () => (() => void) | undefined,
+	description: string,
+	options: FenceWaitOptions = {}
+): Promise<(() => void) | undefined> {
+	const deadline = Date.now() + (options.timeoutMilliseconds ?? DEFAULT_FENCE_WAIT_MILLISECONDS);
+	let retryDelayMilliseconds = 1;
+	for (;;) {
+		if (options.shouldContinue && !options.shouldContinue()) return;
+		if (rootStore.status !== undefined && rootStore.status !== 'open')
+			throw new Error(`Cannot wait for ${description} on a ${rootStore.status} store`);
+		const release = acquire();
+		if (release) return release;
+		if (Date.now() >= deadline)
+			throw new DerivedIndexLagError(`Timed out waiting for ${description}; retry the operation`);
+		await new Promise((resolve) => setTimeout(resolve, retryDelayMilliseconds));
+		retryDelayMilliseconds = Math.min(retryDelayMilliseconds * 2, 50);
+	}
+}
 
 /** `admission` returns a reason while writes to these tables must be rejected. */
 export function registerDerivedIndexTables(

package/resources/derivedIndexRuntime.ts

@@ -188,6 +188,10 @@ export type DerivedIndexRegistration = {
 	backend: DerivedIndexBackend;
 	projections: ReadonlyMap<number, (record: unknown) => unknown>;
 	options?: DerivedIndexRunnerOptions;
+	/** Shared readiness identity when it must be narrower than the stable backend/lock identity. */
+	readinessId?: string;
+	/** False once this table generation has been superseded. Must be synchronous and allocation-free. */
+	isCurrent?: () => boolean;
 };
 
 /**
@@ -246,6 +250,8 @@ const READINESS_STATES: DerivedIndexReadinessState[] = [
 	'needs-rebuild',
 	'unavailable',
 ];
+const READINESS_BACKEND_FAILED = READINESS_STATES.length;
+const READINESS_RETRYING = READINESS_BACKEND_FAILED + 1;
 const READINESS_REASONS: DerivedIndexReadinessReason[] = [
 	'none',
 	'cursor-missing',
@@ -333,6 +339,7 @@ export class DerivedIndexRuntime {
 			this.#listening = true;
 		}
 		runner.wake(true);
+		let releaseStarted = false;
 		return () => {
 			if (this.#runners.get(registration.backend.id) === runner) {
 				this.#runners.delete(registration.backend.id);
@@ -343,6 +350,8 @@ export class DerivedIndexRuntime {
 				}
 				this.#stopListeningIfIdle();
 			}
+			if (releaseStarted) return this.#retryRunnerRelease(registration.backend.id, runner);
+			releaseStarted = true;
 			return this.#track(runner, runner.stop());
 		};
 	}
@@ -417,7 +426,7 @@ export class DerivedIndexRuntime {
 
 	#pollCoverage(backendId: string, group: CoverageWaitGroup) {
 		try {
-			const views = getReadinessViews(this.#logStore, backendId);
+			const views = getReadinessViews(this.#logStore, this.#runners.get(backendId)?.readinessId ?? backendId);
 			const before = readReadiness(views);
 			const time = Atomics.load(views.coverage, 0);
 			const after = readReadiness(views);
@@ -443,23 +452,31 @@ export class DerivedIndexRuntime {
 	/** Force a rebuild (or retry one that became `unavailable`). Returns false when the backend cannot be rebuilt by the runtime. */
 	requestRebuild(backendId: string): boolean {
 		const held = this.#heldRunners.get(backendId);
-		if (held) {
-			const released = held.runner.retryRelease();
-			this.#pendingStops.add(released);
-			released.then(
-				() => {
-					this.#pendingStops.delete(released);
-					this.#pendingStops.delete(held.stopped);
-					if (this.#heldRunners.get(backendId) === held) this.#heldRunners.delete(backendId);
-				},
-				() => {}
-			);
-		}
+		if (held) void this.#retryRunnerRelease(backendId, held.runner);
 		const runner = this.#runners.get(backendId);
 		if (runner) return runner.requestRebuild();
 		return held !== undefined;
 	}
 
+	#retryRunnerRelease(backendId: string, runner: DerivedIndexRunner): Promise<void> {
+		const held = this.#heldRunners.get(backendId);
+		const released = runner.retryRelease();
+		this.#pendingStops.add(released);
+		released.then(
+			() => {
+				this.#pendingStops.delete(released);
+				if (held?.runner === runner) this.#pendingStops.delete(held.stopped);
+				if (this.#heldRunners.get(backendId)?.runner === runner) this.#heldRunners.delete(backendId);
+			},
+			() => {
+				if (held?.runner === runner && held.stopped !== released) this.#pendingStops.delete(held.stopped);
+				const current = this.#heldRunners.get(backendId);
+				if (!current || current.runner === runner) this.#heldRunners.set(backendId, { runner, stopped: released });
+			}
+		);
+		return released;
+	}
+
 	/** Resolves once every runner has released ownership and its backend shutdown has settled. */
 	stop(): Promise<void> {
 		if (this.#stopping) return this.#stopping;
@@ -485,8 +502,15 @@ export class DerivedIndexRuntime {
 
 	#stopListening() {
 		if (!this.#listening) return;
-		this.#logStore.rootStore.off?.('committed', this.#onCommit);
-		this.#listening = false;
+		try {
+			this.#logStore.rootStore.off?.('committed', this.#onCommit);
+		} catch (error) {
+			// An already-closed store cannot emit again and some native stores reject listener access
+			// after close. Preserve errors from a live store, where losing the listener would be unsafe.
+			if (this.#logStore.rootStore.status === 'open') throw error;
+		} finally {
+			this.#listening = false;
+		}
 	}
 }
 
@@ -624,6 +648,7 @@ class DerivedIndexRunner {
 	#unregisterTables: () => void;
 	#ownerEpoch?: bigint;
 	#readinessBuffer: SharedReadinessBuffer;
+	#wakeBuffer: SharedReadinessBuffer;
 	#sharedViews: SharedViews;
 	#resetting?: Promise<void>;
 	status: DerivedIndexRunnerStatus = { state: 'idle' };
@@ -632,6 +657,10 @@ class DerivedIndexRunner {
 		return this.#registration.backend.id;
 	}
 
+	get readinessId() {
+		return this.#registration.readinessId ?? this.id;
+	}
+
 	retryRelease(): Promise<void> {
 		if (!this.#heldLock) return this.#stopResult ?? Promise.resolve();
 		this.#heldLock = false;
@@ -666,7 +695,13 @@ class DerivedIndexRunner {
 			typeof root?.getSync === 'function' &&
 			typeof root?.removeSync === 'function';
 		this.#lagBudget = effectiveLagBudget(options);
-		this.#readinessBuffer = readinessBuffer(logStore, registration.backend.id, () => this.#notified());
+		const notified = () => this.#notified();
+		this.#readinessBuffer = readinessBuffer(
+			logStore,
+			this.readinessId,
+			this.readinessId === this.id ? notified : undefined
+		);
+		this.#wakeBuffer = this.readinessId === this.id ? this.#readinessBuffer : wakeBuffer(logStore, this.id, notified);
 		this.#sharedViews = sharedViewsOf(this.#readinessBuffer);
 		try {
 			registration.backend.attach({
@@ -678,6 +713,7 @@ class DerivedIndexRunner {
 			);
 		} catch (error) {
 			this.#readinessBuffer.cancel?.();
+			if (this.#wakeBuffer !== this.#readinessBuffer) this.#wakeBuffer.cancel?.();
 			throw error;
 		}
 		this.#unregisterTables = registerDerivedIndexTables(
@@ -700,11 +736,7 @@ class DerivedIndexRunner {
 		if (this.#stopped || this.#rebuilding) return;
 		if (!fromBackend && !forCoverage && this.#lagBudget > 0) this.#unreadSince ??= this.#options.now();
 		if (this.status.state === 'unavailable') {
-			if (
-				this.#heldLock ||
-				Atomics.load(this.#shared().words, READINESS_STATE) === READINESS_STATES.indexOf('unavailable')
-			)
-				return;
+			if (this.#heldLock || isUnavailableReadinessState(Atomics.load(this.#shared().words, READINESS_STATE))) return;
 			this.status = { state: 'idle' };
 		}
 		// A shared rebuild request must reach an owner parked on backpressure or backoff at its next wake.
@@ -742,6 +774,7 @@ class DerivedIndexRunner {
 		try {
 			this.#unsubscribeBackend?.();
 			this.#readinessBuffer.cancel?.();
+			if (this.#wakeBuffer !== this.#readinessBuffer) this.#wakeBuffer.cancel?.();
 		} catch (error) {
 			logger.warn?.(`Derived index '${this.id}' cleanup hook threw`, error);
 		}
@@ -883,9 +916,9 @@ class DerivedIndexRunner {
 			return true;
 		}
 		// The owner may be another worker that never idles: leave the request where every runner looks,
-		// and notify whoever holds the buffer's callback.
+		// and notify every generation sharing the stable runner lock.
 		Atomics.store(this.#shared().words, READINESS_REBUILD_REQUEST, 1);
-		this.#readinessBuffer.notify?.();
+		this.#wakeBuffer.notify?.();
 		this.wake(true);
 		return true;
 	}
@@ -927,12 +960,14 @@ class DerivedIndexRunner {
 	 * The lock is taken without an unlock callback. rocksdb-js queues such a callback as a
 	 * thread-safe function of the caller's env, and on Node 22 a callback left behind by a worker that
 	 * was terminated aborts the process when another thread unlocks (HarperFast/rocksdb-js, pending).
-	 * A successor is woken by the releasing owner's `notify()` on the readiness buffer — which
-	 * tolerates a dead listener's env — and by the retry timer below.
+	 * A successor is woken by the releasing owner's `notify()` on the stable-id wake buffer — which
+	 * tolerates a dead listener's env — and by the retry timer below. Registrations without a
+	 * generation-specific readiness id keep using the readiness buffer itself.
 	 */
 	#acquire() {
 		// Commit wakes are frequent; neither a parked nor a backing-off runner may re-probe on each one.
 		if (this.#waitingForLock || this.#lockBackoff) return;
+		if (this.#registration.isCurrent && !this.#registration.isCurrent()) return;
 		if (this.#releasing) {
 			this.#releasing.then(() => this.wake(true));
 			return;
@@ -982,6 +1017,15 @@ class DerivedIndexRunner {
 		this.#reachedEndOfLog = false;
 		try {
 			if (!reviving) this.#ownerEpoch = this.#mintEpoch();
+			// A worker that could not construct this generation may have published a setup-only
+			// failure before this healthy backend acquired the stable runner lock. Ownership is
+			// stronger evidence: clear only that sentinel, never a runner-published unavailable state.
+			Atomics.compareExchange(
+				this.#shared().words,
+				READINESS_STATE,
+				READINESS_BACKEND_FAILED,
+				READINESS_STATES.indexOf('unknown')
+			);
 			this.status = { state: 'running', ownerEpoch: this.#ownerEpoch };
 			const condemned = this.#readCondemnation();
 			const shared = this.getReadiness();
@@ -1101,6 +1145,10 @@ class DerivedIndexRunner {
 
 	#drain() {
 		if (!this.#owned || this.#stopped || this.#rebuilding) return;
+		if (this.#registration.isCurrent && !this.#registration.isCurrent()) {
+			this.#release();
+			return;
+		}
 		if (this.#canRebuild() && this.#takeSharedRebuildRequest()) {
 			if (this.#rebuildTimer) {
 				clearTimeout(this.#rebuildTimer);
@@ -2083,10 +2131,10 @@ class DerivedIndexRunner {
 				logger.error(`Failed to release derived index runner '${backend.id}'`, error);
 			}
 			// A skip left standing by an absent or throwing `notify` swallows a peer's release instead.
-			if (this.#readinessBuffer.notify) {
+			if (this.#wakeBuffer.notify) {
 				this.#skipNextNotify = !this.#stopped;
 				try {
-					this.#readinessBuffer.notify();
+					this.#wakeBuffer.notify();
 				} catch (error) {
 					this.#skipNextNotify = false;
 					logger.warn?.(`Derived index '${backend.id}' could not notify peers of its release`, error);
@@ -2134,6 +2182,16 @@ function readinessBuffer(
 	) as SharedReadinessBuffer;
 }
 
+function wakeBuffer(
+	logStore: RocksTransactionLogStore,
+	backendId: string,
+	callback: () => void
+): SharedReadinessBuffer {
+	return logStore.getUserSharedBuffer(`derived-index:${backendId}:wake`, new ArrayBuffer(1), {
+		callback,
+	}) as SharedReadinessBuffer;
+}
+
 type SharedViews = {
 	words: Int32Array;
 	epoch: BigInt64Array;
@@ -2151,12 +2209,19 @@ function sharedViewsOf(buffer: ArrayBufferLike): SharedViews {
 const readinessViews = new WeakMap<object, Map<string, SharedViews>>();
 
 function readReadiness({ words, epoch }: SharedViews): DerivedIndexReadiness {
-	const state = READINESS_STATES[Atomics.load(words, READINESS_STATE)] ?? 'unknown';
-	const reason = READINESS_REASONS[Atomics.load(words, READINESS_REASON)] ?? 'none';
+	const stateIndex = Atomics.load(words, READINESS_STATE);
+	const setupFailed = stateIndex === READINESS_BACKEND_FAILED;
+	const retrying = stateIndex === READINESS_RETRYING;
+	const state = setupFailed ? 'unavailable' : retrying ? 'unknown' : (READINESS_STATES[stateIndex] ?? 'unknown');
+	const reason = setupFailed
+		? 'backend-failed'
+		: retrying
+			? 'none'
+			: (READINESS_REASONS[Atomics.load(words, READINESS_REASON)] ?? 'none');
 	const readiness: DerivedIndexReadiness = {
 		state,
 		ownerEpoch: Atomics.load(epoch, 0),
-		rebuildAttempts: Atomics.load(words, READINESS_ATTEMPTS),
+		rebuildAttempts: retrying || setupFailed ? 0 : Atomics.load(words, READINESS_ATTEMPTS),
 	};
 	if (reason !== 'none') readiness.reason = reason;
 	return readiness;
@@ -2170,6 +2235,68 @@ export function readDerivedIndexReadiness(
 	return readReadiness(getReadinessViews(logStore, backendId));
 }
 
+/** Publish setup-time state before a backend runner exists. */
+export function publishDerivedIndexReadiness(
+	logStore: RocksTransactionLogStore,
+	backendId: string,
+	state: DerivedIndexReadinessState,
+	reason: DerivedIndexReadinessReason = 'none'
+): void {
+	const views = getReadinessViews(logStore, backendId);
+	if (state === 'unknown') Atomics.store(views.words, READINESS_ATTEMPTS, 0);
+	Atomics.store(views.words, READINESS_REASON, READINESS_REASONS.indexOf(reason));
+	Atomics.store(views.words, READINESS_STATE, READINESS_STATES.indexOf(state));
+}
+
+/** Publish a setup failure only while no elected runner has published a stronger state. */
+export function publishDerivedIndexUnavailableIfUnknown(
+	logStore: RocksTransactionLogStore,
+	backendId: string
+): boolean {
+	const { words, epoch } = getReadinessViews(logStore, backendId);
+	// RETRYING is an explicit setup retry. Epochs are monotonic, so a prior
+	// owner must not prevent that retry from returning to an observable failure.
+	const retryEpoch = Atomics.load(epoch, 0);
+	if (
+		Atomics.compareExchange(words, READINESS_STATE, READINESS_RETRYING, READINESS_BACKEND_FAILED) === READINESS_RETRYING
+	) {
+		// Close the same owner-election race as the unknown-state path below. A
+		// runner that acquired this epoch owns readiness, even if its worker has
+		// not published ready yet.
+		if (Atomics.load(epoch, 0) !== retryEpoch) {
+			Atomics.compareExchange(words, READINESS_STATE, READINESS_BACKEND_FAILED, READINESS_STATES.indexOf('unknown'));
+			return false;
+		}
+		return true;
+	}
+	if (Atomics.load(epoch, 0) !== 0n) return false;
+	const published =
+		Atomics.compareExchange(words, READINESS_STATE, READINESS_STATES.indexOf('unknown'), READINESS_BACKEND_FAILED) ===
+		READINESS_STATES.indexOf('unknown');
+	if (!published) return false;
+	// Close the race with an owner acquiring between the epoch probe and the state CAS.
+	if (Atomics.load(epoch, 0) !== 0n) {
+		Atomics.compareExchange(words, READINESS_STATE, READINESS_BACKEND_FAILED, READINESS_STATES.indexOf('unknown'));
+		return false;
+	}
+	return true;
+}
+
+/** Rearm setup only while no peer has replaced the unavailable state. */
+export function retryDerivedIndexUnavailable(logStore: RocksTransactionLogStore, backendId: string): boolean {
+	const { words } = getReadinessViews(logStore, backendId);
+	const unavailable = READINESS_STATES.indexOf('unavailable');
+	if (Atomics.compareExchange(words, READINESS_STATE, unavailable, READINESS_RETRYING) === unavailable) return true;
+	return (
+		Atomics.compareExchange(words, READINESS_STATE, READINESS_BACKEND_FAILED, READINESS_RETRYING) ===
+		READINESS_BACKEND_FAILED
+	);
+}
+
+function isUnavailableReadinessState(stateIndex: number): boolean {
+	return stateIndex === READINESS_STATES.indexOf('unavailable') || stateIndex === READINESS_BACKEND_FAILED;
+}
+
 function getReadinessViews(logStore: RocksTransactionLogStore, backendId: string): SharedViews {
 	let byBackend = readinessViews.get(logStore);
 	if (!byBackend) readinessViews.set(logStore, (byBackend = new Map()));