@harperfast/harper 5.2.13 → 5.3.0-beta.1

package/resources/indexes/hnswDerivedIndex.ts

@@ -0,0 +1,531 @@
+import { ClientError } from '../../utility/errors/hdbError.ts';
+import { loggerWithTag } from '../../utility/logging/logger.ts';
+import type { Id } from '../ResourceInterface.ts';
+import type { RocksTransactionLogStore } from '../RocksTransactionLogStore.ts';
+import {
+	DERIVED_INDEX_ACCEPTED,
+	DERIVED_INDEX_DEFERRED,
+	DerivedIndexRuntime,
+	readDerivedIndexReadiness,
+	readDerivedIndexCoverage,
+	sameDerivedIndexPositions,
+	type DerivedIndexBackend,
+	type DerivedIndexBackendHost,
+	type DerivedIndexBackendStateChange,
+	type DerivedIndexBatch,
+	type DerivedIndexCursor,
+	type DerivedIndexCoverage,
+	type DerivedIndexPositions,
+	type DerivedIndexDeliveryResult,
+	type DerivedIndexFlushReason,
+	type DerivedIndexMutation,
+	type DerivedIndexReadiness,
+} from '../derivedIndexRuntime.ts';
+
+const logger = loggerWithTag('HNSW');
+
+/** The one durable cursor vector of a file-primary HNSW index, stored beside its node mappings. */
+export const DERIVED_INDEX_CURSOR_KEY = Symbol.for('derived-index-cursor');
+export const DEFAULT_MAX_INDEX_LAG_MILLISECONDS = 3000;
+export const MAX_WAIT_FOR_INDEX_MILLISECONDS = 30_000;
+
+export type DerivedNativeIndexHost = {
+	readiness: () => DerivedIndexReadiness;
+	coverage: (maxLagMilliseconds: number) => DerivedIndexCoverage;
+	waitForCoverage: (since: bigint, timeout: number, signal?: AbortSignal) => Promise<void>;
+	requestRebuild: () => boolean;
+};
+
+// Bounds on the queue between the runtime's delivery and the native applier. The runtime already
+// chunks a delivery by records and estimated bytes; this caps how many chunks may wait.
+const QUEUE_CAPACITY_BYTES = 64 * 1024 * 1024;
+const APPLY_SLICE_MILLIS = 5;
+const BARRIER_IDLE_MULTIPLE = 3;
+const BARRIER_IDLE_CEILING_MILLISECONDS = 7_500;
+// Writes to an index this far behind fail with a retryable 503 (see the runtime's lag policy).
+const DEFAULT_MAX_LAG_MILLISECONDS = 30_000;
+
+/**
+ * The native index methods the backend drives. `applyDerivedValue` inserts, replaces or removes
+ * one primary key's vector and stages its mapping as pending; `flushDerived` is the durability
+ * barrier that publishes the pending mappings; `resetDerivedStorage` destroys the native file.
+ */
+export interface DerivedNativeIndex {
+	readonly indexStore: any;
+	applyDerivedValue(primaryKey: Id, vector: number[] | undefined, version?: number): void;
+	flushDerived(): Promise<void>;
+	resetDerivedStorage(): void;
+	assertDerivedValue(vector: unknown, label: string): void;
+	attachDerivedHost(host: DerivedNativeIndexHost): void;
+}
+
+/**
+ * `DerivedIndexBackend` over a file-primary HNSW index. `deliver()` only enqueues; an applier drains
+ * the queue in bounded time slices so a 4096-record chunk at ~0.35 ms per native insert does not
+ * hold the event loop for a second and a half. The barrier is the plane's `msync`, after which the
+ * pending mappings and then the cursor vector are published — in that order, so a crash between
+ * any two steps leaves a cursor that replays (idempotently) over a mapping that was never published,
+ * never a published cursor over native state the barrier did not cover.
+ */
+export class HnswDerivedIndexBackend implements DerivedIndexBackend {
+	readonly id: string;
+	#index: DerivedNativeIndex;
+	#host?: DerivedIndexBackendHost;
+	#wake?: (change?: DerivedIndexBackendStateChange) => void;
+	#queue: DerivedIndexBatch[] = [];
+	#queuedBytes = 0;
+	#position = 0;
+	#scheduled = false;
+	#appliedCursor?: DerivedIndexCursor;
+	#appliedEpoch?: bigint;
+	#flushRequested = false;
+	#flushing?: Promise<void>;
+	#interruptBarrierAfter = 0;
+	#resetting?: Promise<void>;
+	#unindexable = 0;
+
+	constructor(id: string, index: DerivedNativeIndex) {
+		this.id = id;
+		this.#index = index;
+	}
+
+	attach(host: DerivedIndexBackendHost): void {
+		this.#host = host;
+	}
+
+	onStateChange(wake: (change?: DerivedIndexBackendStateChange) => void): () => void {
+		this.#wake = wake;
+		return () => {
+			if (this.#wake === wake) this.#wake = undefined;
+		};
+	}
+
+	getDurableCursor(): DerivedIndexCursor | undefined {
+		const cursor = this.#index.indexStore.getSync(DERIVED_INDEX_CURSOR_KEY);
+		return cursor === undefined ? undefined : (cursor as DerivedIndexCursor);
+	}
+
+	publishCoverage(positions: DerivedIndexPositions, ownerEpoch: bigint): void {
+		if (!this.#host?.isOwnerEpoch(ownerEpoch)) return;
+		const cursor = this.getDurableCursor();
+		if (!cursor) throw new Error('Cannot publish native index coverage without a durable cursor');
+		if (cursor.coverage && sameDerivedIndexPositions(cursor.coverage, positions)) return;
+		this.#index.indexStore.putSync(DERIVED_INDEX_CURSOR_KEY, { ...cursor, coverage: positions });
+	}
+
+	deliver(batch: DerivedIndexBatch): DerivedIndexDeliveryResult {
+		if (this.#queuedBytes >= QUEUE_CAPACITY_BYTES) return DERIVED_INDEX_DEFERRED;
+		this.#queue.push(batch);
+		this.#queuedBytes += batch.bytes;
+		this.#schedule();
+		return DERIVED_INDEX_ACCEPTED;
+	}
+
+	flush(_reason: DerivedIndexFlushReason): void {
+		this.#flushRequested = true;
+		if (this.#queue.length === 0 && !this.#scheduled) this.#runFlush();
+	}
+
+	/**
+	 * Drop what is queued and resolve once no barrier is in flight. The runtime quiesces an epoch
+	 * before it mints the next, so nothing of a newer epoch can be queued here yet.
+	 */
+	async shutdown(_epoch: bigint): Promise<void> {
+		this.#dropQueue();
+		await Promise.allSettled([this.#flushing, this.#resetting]);
+	}
+
+	/**
+	 * The cursor goes first: a crash anywhere after this line reopens cursorless and rebuilds, never
+	 * on a format-valid cursor over a destroyed graph.
+	 */
+	reset(_epoch: bigint): Promise<void> {
+		this.#dropQueue();
+		this.#appliedCursor = undefined;
+		this.#resetting = (async () => {
+			await this.#flushing;
+			this.#index.indexStore.removeSync(DERIVED_INDEX_CURSOR_KEY);
+			this.#index.resetDerivedStorage();
+			await this.#index.indexStore.clear();
+		})();
+		return this.#resetting.finally(() => (this.#resetting = undefined));
+	}
+
+	#dropQueue() {
+		this.#queue.length = 0;
+		this.#queuedBytes = 0;
+		this.#position = 0;
+	}
+
+	#schedule() {
+		if (this.#scheduled) return;
+		this.#scheduled = true;
+		setImmediate(() => {
+			this.#scheduled = false;
+			this.#applySlice();
+		});
+	}
+
+	#applySlice() {
+		// Application pauses while a barrier is in flight so the barrier publishes exactly the
+		// mappings it covers; the flush reschedules application when it settles.
+		if (this.#flushing) return;
+		const host = this.#host!;
+		const until = performance.now() + APPLY_SLICE_MILLIS;
+		const wasFull = this.#queuedBytes >= QUEUE_CAPACITY_BYTES;
+		let advancedCursor = false;
+		while (this.#queue.length > 0 && performance.now() < until) {
+			const batch = this.#queue[0];
+			if (!host.isOwnerEpoch(batch.ownerEpoch)) {
+				this.#queue.shift();
+				this.#queuedBytes -= batch.bytes;
+				this.#position = 0;
+				continue;
+			}
+			const records = batch.records;
+			while (this.#position < records.length && performance.now() < until) {
+				try {
+					this.#apply(records[this.#position++]);
+				} catch (error) {
+					this.#queue.length = 0;
+					this.#queuedBytes = 0;
+					this.#position = 0;
+					logger.error(`Derived index '${this.id}' could not apply a record; requesting a rebuild`, error);
+					this.#wake?.('failed');
+					return;
+				}
+			}
+			if (this.#position < records.length) break;
+			this.#queue.shift();
+			this.#queuedBytes -= batch.bytes;
+			this.#position = 0;
+			this.#appliedEpoch = batch.ownerEpoch;
+			if (batch.through) {
+				this.#appliedCursor = batch.through;
+				advancedCursor = true;
+				if (this.#mayInterruptForBarrier()) break;
+			}
+		}
+		if (this.#queue.length > 0) {
+			if (wasFull && this.#queuedBytes < QUEUE_CAPACITY_BYTES) this.#wake?.('changed');
+			if (this.#position === 0 && advancedCursor && this.#mayInterruptForBarrier()) this.#runFlush(true);
+			else this.#schedule();
+			return;
+		}
+		if (wasFull) this.#wake?.('changed');
+		if (this.#flushRequested) this.#runFlush();
+	}
+
+	#apply(mutation: DerivedIndexMutation) {
+		const { recordId, logVersion, state } = mutation;
+		if (state.kind !== 'record' || state.projection == null) {
+			this.#index.applyDerivedValue(recordId, undefined, logVersion);
+			return;
+		}
+		try {
+			this.#index.applyDerivedValue(recordId, state.projection as number[], state.version);
+		} catch (error) {
+			// The projection validated the vector, but the plane's dimensionality is only known to the
+			// worker holding it: a mismatch is this record's fault, not the index's. Same rule as the
+			// runtime's own 4xx handling — remove any entry and count it, never rebuild.
+			if (!(error instanceof ClientError)) throw error;
+			if (this.#unindexable++ === 0)
+				logger.warn?.(`Derived index '${this.id}' skipped a record its plane cannot hold: ${error.message}`);
+			this.#index.applyDerivedValue(recordId, undefined, state.version);
+		}
+	}
+
+	#mayInterruptForBarrier(): boolean {
+		return this.#flushRequested && performance.now() >= this.#interruptBarrierAfter;
+	}
+
+	#runFlush(interrupting = false) {
+		if (this.#flushing) return;
+		this.#flushRequested = false;
+		const started = performance.now();
+		const cursor = this.#appliedCursor;
+		const epoch = this.#appliedEpoch;
+		const host = this.#host!;
+		this.#flushing = (async () => {
+			await this.#index.flushDerived();
+			if (epoch !== undefined && !host.isOwnerEpoch(epoch)) return;
+			if (cursor) {
+				const coverage = this.getDurableCursor()?.coverage;
+				this.#index.indexStore.putSync(DERIVED_INDEX_CURSOR_KEY, coverage ? { ...cursor, coverage } : cursor);
+			}
+		})();
+		this.#flushing.then(
+			() => {
+				this.#flushing = undefined;
+				if (interrupting) {
+					const idle = Math.min(
+						BARRIER_IDLE_MULTIPLE * (performance.now() - started),
+						BARRIER_IDLE_CEILING_MILLISECONDS
+					);
+					this.#interruptBarrierAfter = performance.now() + idle;
+				}
+				this.#wake?.('changed');
+				if (this.#queue.length > 0) this.#schedule();
+				else if (this.#flushRequested) this.#runFlush();
+			},
+			(error) => {
+				this.#flushing = undefined;
+				logger.error(`Derived index '${this.id}' durability barrier failed`, error);
+				this.#wake?.('failed');
+			}
+		);
+	}
+}
+
+// One generation owns each physical backend; destructive cleanup also awaits superseded generations.
+type RegisteredTable = { current: { Table: any }; owners: Set<{ Table: any }> };
+type RegisteredBackend = {
+	tableId: number;
+	settle: () => Promise<void>;
+};
+type Registered = {
+	runtime: DerivedIndexRuntime;
+	tables: Map<number, RegisteredTable>;
+	backends: Map<string, RegisteredBackend>;
+	tableBackends: Map<number, Set<RegisteredBackend>>;
+	droppingTables: Set<number>;
+	retryUnavailable: Set<string>;
+};
+const runtimes = new WeakMap<object, Registered>();
+const retryUnavailableByStore = new Set<string>();
+
+function retryUnavailableKey(auditStore: RocksTransactionLogStore, backendId: string): string {
+	return `${auditStore.rootStore.path}\0${backendId}`;
+}
+
+function markUnavailableRetry(registered: Registered, auditStore: RocksTransactionLogStore, backendId: string): void {
+	registered.retryUnavailable.add(backendId);
+	retryUnavailableByStore.add(retryUnavailableKey(auditStore, backendId));
+}
+
+function consumeUnavailableRetry(
+	registered: Registered,
+	auditStore: RocksTransactionLogStore,
+	backendId: string
+): boolean {
+	const local = registered.retryUnavailable.delete(backendId);
+	const storeScoped = retryUnavailableByStore.delete(retryUnavailableKey(auditStore, backendId));
+	return local || storeScoped;
+}
+
+function runtimeFor(auditStore: RocksTransactionLogStore): Registered {
+	let registered = runtimes.get(auditStore);
+	if (registered) return registered;
+	const tables = new Map<number, RegisteredTable>();
+	const runtime = new DerivedIndexRuntime(
+		auditStore,
+		(tableId, recordId) => {
+			const entry = tables.get(tableId)?.current.Table.primaryStore.getEntry(recordId);
+			return entry?.value == null ? undefined : { version: entry.version, value: entry.value };
+		},
+		{
+			scanRecords: (tableId) =>
+				tables
+					.get(tableId)!
+					.current.Table.primaryStore.getRange({ versions: true, snapshot: false })
+					.map(({ key, value, version }) => ({ recordId: key, version, value })),
+		}
+	);
+	registered = {
+		runtime,
+		tables,
+		backends: new Map(),
+		tableBackends: new Map(),
+		droppingTables: new Set(),
+		retryUnavailable: new Set(),
+	};
+	runtimes.set(auditStore, registered);
+	return registered;
+}
+
+const warnedAuditIndexes = new Set<string>();
+
+/**
+ * Register every post-commit custom index of a table with the shared derived-index runtime of its
+ * database. Returns the release for the table's registrations, or undefined when it has none. Runs
+ * on every worker; the runtime elects one owner per index.
+ */
+export function attachDerivedIndexes(Table: any):
+	| {
+			close(dropping?: boolean): Promise<void>;
+			restoreAfterFailedDrop(): ReturnType<typeof attachDerivedIndexes>;
+			completeDrop(dropped?: boolean): void;
+	  }
+	| undefined {
+	const hnswAttributes = Table.attributes.filter(
+		(attribute: any) => attribute.indexed?.type === 'HNSW' && Table.indices[attribute.name]?.customIndex
+	);
+	const attributes = hnswAttributes.filter((attribute: any) => Table.indices[attribute.name]?.customIndex?.postCommit);
+	if (Table.audit !== true && attributes.length > 0) {
+		throw new ClientError(
+			`Table '${Table.databaseName}.${Table.tableName}' must enable audit logging before using a post-commit derived index`
+		);
+	}
+	if (!Table.auditStore) return;
+	const auditStore = Table.auditStore as RocksTransactionLogStore;
+	const registered = runtimeFor(auditStore);
+	if (registered.droppingTables.has(Table.tableId)) return;
+	for (const attribute of hnswAttributes) {
+		const indexStore = Table.indices[attribute.name];
+		const id = `hnsw:${indexStore.name}`;
+		// A transient JS-backed declaration can precede the first native registration while a
+		// schema is being installed. Only an explicit transition away from an already-registered
+		// native backend should make a later native registration eligible to rearm an exhausted
+		// rebuild budget.
+		if (attribute.indexed.nativePlane === false && !indexStore.customIndex.postCommit && registered.backends.has(id))
+			markUnavailableRetry(registered, auditStore, id);
+	}
+	if (attributes.length === 0) return;
+	const installed = { Table };
+	let registeredTable = registered.tables.get(Table.tableId);
+	if (registeredTable) {
+		registeredTable.owners.add(installed);
+		registeredTable.current = installed;
+	} else {
+		registeredTable = { current: installed, owners: new Set([installed]) };
+		registered.tables.set(Table.tableId, registeredTable);
+	}
+	const releases: Array<() => Promise<void>> = [];
+	const backendIds: string[] = [];
+	const registeredBackends = new Map<string, RegisteredBackend>();
+	for (const attribute of attributes) {
+		const indexStore = Table.indices[attribute.name];
+		const index = indexStore.customIndex as DerivedNativeIndex & { postCommit: true };
+		const id = `hnsw:${indexStore.name}`;
+		const warningKey = `${Table.databaseName}.${Table.tableName}.${indexStore.name}`;
+		if (!warnedAuditIndexes.has(warningKey)) {
+			warnedAuditIndexes.add(warningKey);
+			logger.warn?.(
+				`Derived index ${indexStore.name} requires auditing; the audit API retains full record history for the configured retention window`
+			);
+		}
+		const resolver = Table.propertyResolvers?.[attribute.name];
+		const label = `Vector for attribute "${attribute.name}"`;
+		index.attachDerivedHost({
+			readiness: () => registered.runtime.getReadiness(id),
+			coverage: (maxLagMilliseconds) =>
+				readDerivedIndexCoverage(
+					auditStore,
+					id,
+					() => indexStore.getSync(DERIVED_INDEX_CURSOR_KEY),
+					maxLagMilliseconds
+				),
+			requestRebuild: () => registered.runtime.requestRebuild(id),
+			waitForCoverage: (since, timeout, signal) => registered.runtime.waitForCoverage(id, since, timeout, signal),
+		});
+		let predecessor = registered.backends.get(id);
+		const inherited = predecessor;
+		const settlePredecessor = () => {
+			const current = predecessor;
+			if (!current) return Promise.resolve();
+			const settled = current.settle();
+			settled.then(
+				() => {
+					if (predecessor === current) predecessor = undefined;
+				},
+				() => {}
+			);
+			return settled;
+		};
+		const predecessorSettled = settlePredecessor();
+		predecessorSettled.catch(() => {});
+		if (registered.backends.get(id) === inherited) registered.backends.delete(id);
+		const release = registered.runtime.register({
+			backend: new HnswDerivedIndexBackend(id, index),
+			projections: new Map([
+				[
+					Table.tableId,
+					(record: any) => {
+						const vector = resolver ? resolver(record) : record[attribute.name];
+						if (vector == null) return undefined;
+						index.assertDerivedValue(vector, label);
+						return vector;
+					},
+				],
+			]),
+			options: { maxLagMilliseconds: attribute.indexed?.maxLagMilliseconds ?? DEFAULT_MAX_LAG_MILLISECONDS },
+		});
+		let settling: Promise<void> | undefined;
+		const registeredBackend: RegisteredBackend = {
+			tableId: Table.tableId,
+			settle: () => {
+				if (settling) return settling;
+				const attempt = Promise.allSettled([settlePredecessor(), release()]).then((results) => {
+					if (results[1].status === 'fulfilled') {
+						if (registered.backends.get(id) === registeredBackend) registered.backends.delete(id);
+						const tableBackends = registered.tableBackends.get(Table.tableId);
+						tableBackends?.delete(registeredBackend);
+						if (tableBackends?.size === 0) registered.tableBackends.delete(Table.tableId);
+					}
+					const failures = results
+						.filter((result): result is PromiseRejectedResult => result.status === 'rejected')
+						.map((result) => result.reason);
+					if (failures.length === 1) throw failures[0];
+					if (failures.length) throw new AggregateError(failures, `Could not settle HNSW backend '${id}'`);
+				});
+				settling = attempt;
+				attempt.catch(() => {
+					if (settling === attempt) settling = undefined;
+				});
+				return attempt;
+			},
+		};
+		let tableBackends = registered.tableBackends.get(Table.tableId);
+		if (!tableBackends) registered.tableBackends.set(Table.tableId, (tableBackends = new Set()));
+		tableBackends.add(registeredBackend);
+		registered.backends.set(id, registeredBackend);
+		registeredBackends.set(id, registeredBackend);
+		const retryUnavailable = consumeUnavailableRetry(registered, auditStore, id);
+		if (retryUnavailable && registered.runtime.getReadiness(id).state === 'unavailable')
+			registered.runtime.requestRebuild(id);
+		backendIds.push(id);
+		releases.push(() => registeredBackend.settle());
+	}
+	return {
+		async close(dropping = false) {
+			if (dropping) registered.droppingTables.add(Table.tableId);
+			const settlements = dropping
+				? [...(registered.tableBackends.get(Table.tableId) ?? [])].map((backend) => backend.settle())
+				: releases.map((release) => release());
+			const tableRegistration = registered.tables.get(Table.tableId);
+			if (tableRegistration) {
+				tableRegistration.owners.delete(installed);
+				if (tableRegistration.owners.size === 0) {
+					registered.tables.delete(Table.tableId);
+				} else if (tableRegistration.current === installed) {
+					tableRegistration.current = tableRegistration.owners.values().next().value;
+				}
+			}
+			if (dropping) {
+				await Promise.all(settlements);
+			} else {
+				for (const result of await Promise.allSettled(settlements)) {
+					if (result.status === 'rejected')
+						logger.error(`Could not settle a superseded HNSW backend for table ${Table.tableId}`, result.reason);
+				}
+			}
+		},
+		restoreAfterFailedDrop() {
+			registered.droppingTables.delete(Table.tableId);
+			for (const [id, registeredBackend] of registeredBackends) {
+				const current = registered.backends.get(id);
+				if (current && current !== registeredBackend) return;
+			}
+			return attachDerivedIndexes(Table);
+		},
+		completeDrop(dropped = true) {
+			registered.droppingTables.delete(Table.tableId);
+			if (dropped) for (const id of backendIds) markUnavailableRetry(registered, auditStore, id);
+		},
+	};
+}
+
+/** Shared readiness of an index on any worker, registered or not. */
+export function derivedIndexReadiness(auditStore: RocksTransactionLogStore, indexStoreName: string) {
+	return readDerivedIndexReadiness(auditStore, `hnsw:${indexStoreName}`);
+}

package/resources/indexes/hnswPlaneBinding.ts

@@ -0,0 +1,174 @@
+import { closeSync, fsyncSync, openSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { loggerWithTag } from '../../utility/logging/logger.ts';
+
+const logger = loggerWithTag('HNSW');
+
+/**
+ * Parallel arrays, ascending by distance: hit i is (ids[i], distances[i]) with its host key at
+ * keys.subarray(keyEnds[i - 1] ?? 0, keyEnds[i]); an empty key means the node vanished mid-query.
+ */
+export interface PlaneSearchHits {
+	ids: Uint32Array;
+	distances: Float32Array;
+	keys: Buffer;
+	keyEnds: Uint32Array;
+}
+
+/** A predicate batch: candidate ids and their host keys, laid out as in PlaneSearchHits. */
+export type PlanePredicate = (ids: number[], keys: Buffer, keyEnds: Uint32Array) => Uint8Array;
+
+/** NAPI surface of the native file-primary HNSW index (`@harperfast/hnsw`). */
+export interface HnswPlane {
+	readonly dims: number;
+	readonly layer0Cap: number;
+	readonly keyCap: number;
+	insert(vector: Float32Array, key?: Buffer): number;
+	remove(id: number): void;
+	writeNodeRaw(
+		id: number,
+		level: number,
+		vector: Buffer,
+		scale: number,
+		invMag: number,
+		neighbors: Uint32Array,
+		upper: Uint32Array[] | null
+	): void;
+	clearNode(id: number): void;
+	setEntryPoint(id: number, level: number): void;
+	getEntryPoint(): number[];
+	search(
+		vector: Float32Array,
+		k: number,
+		ef: number,
+		filter?: Uint8Array | null,
+		filterExpansion?: number | null
+	): Promise<PlaneSearchHits>;
+	searchWithPredicate(
+		vector: Float32Array,
+		k: number,
+		ef: number,
+		predicate: PlanePredicate,
+		filterExpansion?: number | null,
+		visitBudget?: number | null
+	): Promise<PlaneSearchHits>;
+	searchSync(vector: Float32Array, k: number, ef: number): PlaneSearchHits;
+	writeNodeRawIfAbsent(
+		id: number,
+		level: number,
+		vector: Buffer,
+		scale: number,
+		invMag: number,
+		neighbors: Uint32Array,
+		upper: Uint32Array[] | null
+	): boolean;
+	openedClean(): boolean;
+	idHighWater(): number;
+	getWatermark(): number;
+	setWatermark(txn: number): void;
+	flush(watermark?: number): void;
+	flushAsync(watermark?: number): Promise<void>;
+	invalidateFile(): PlaneInvalidationOutcome;
+	invalidated(): boolean;
+}
+
+export interface HnswPlaneConstructor {
+	readonly prototype: HnswPlane;
+	create(path: string, dims: number, layer0Cap: number, maxNodes: number, keyCap?: number): HnswPlane;
+	open(path: string): HnswPlane;
+}
+
+export interface PlaneInvalidationOutcome {
+	inBand: boolean;
+	sidecar: boolean;
+	inBandError?: string;
+	sidecarError?: string;
+}
+
+interface HnswPlanePackage {
+	Plane: HnswPlaneConstructor;
+	invalidatePlane(path: string): PlaneInvalidationOutcome;
+	stalePathFor(path: string): string;
+}
+
+/** Entry-point id meaning "none" (u32::MAX in the plane header). */
+export const PLANE_NO_ID = 0xffffffff;
+
+/**
+ * Where an index's plane file lives: next to its store, named by the dbiKey
+ * (`table/attribute`, flattened to a single file name). Exposed separately from the index
+ * instance so crash-recovery drop paths can remove the file without opening the index.
+ */
+export function planeFilePathFor(storePath: string, storeName: string): string {
+	// encodeURIComponent is injective and never emits a path separator: table `a` attribute
+	// `b.c` and table `a.b` attribute `c` must not share a plane file (dot-flattening let two
+	// indexes serve each other's node ids as their own primary keys)
+	return join(storePath, `${encodeURIComponent(storeName)}.hnsw`);
+}
+
+/**
+ * Tombstone marking a plane file that could not be deleted (e.g. Windows EBUSY while still
+ * mapped). Its presence means the plane file is STALE: never open it — delete both when
+ * possible and rebuild.
+ */
+export function planeStalePathFor(planePath: string): string {
+	// the package owns the convention; the literal covers the calls that precede its load
+	return binding?.stalePathFor(planePath) ?? `${planePath}.stale`;
+}
+
+let binding: HnswPlanePackage | null | undefined;
+let bindingLoadError: Error | undefined;
+let bindingWarningLogged = false;
+
+function getHnswPackage(warn = true): HnswPlanePackage | null {
+	if (binding === undefined) {
+		try {
+			binding = require('@harperfast/hnsw') as HnswPlanePackage;
+		} catch (error) {
+			binding = null;
+			bindingLoadError = error as Error;
+		}
+	}
+	if (binding === null && warn && !bindingWarningLogged) {
+		bindingWarningLogged = true;
+		logger.warn?.(
+			`The @harperfast/hnsw native module is not available (${bindingLoadError?.message}); ` +
+				'indexes with nativePlane enabled will remain unavailable until the module can load'
+		);
+	}
+	return binding;
+}
+
+/** The native plane constructor, or null when the compiled artifact is unavailable (warns once by default). */
+export function getPlaneBinding(warn = true): HnswPlaneConstructor | null {
+	return getHnswPackage(warn)?.Plane ?? null;
+}
+
+/** Make a derived plane unadoptable before it is replaced or removed. */
+export function invalidatePlaneFile(filePath: string, attached?: HnswPlane | null): PlaneInvalidationOutcome {
+	if (attached) return attached.invalidateFile();
+	const hnswPackage = getHnswPackage();
+	if (hnswPackage) return hnswPackage.invalidatePlane(filePath);
+	// A plane can outlive the package that made it (uninstall, or a prebuild that stopped
+	// loading), and a reinstall would then adopt it. Only the sidecar is reachable without the
+	// package, and it has to survive a power loss to be worth writing, so fsync it and the
+	// directory entry that names it — the same durability the package's own sidecar has.
+	const fd = openSync(planeStalePathFor(filePath), 'w');
+	try {
+		fsyncSync(fd);
+	} finally {
+		closeSync(fd);
+	}
+	try {
+		const dirFd = openSync(dirname(filePath), 'r');
+		try {
+			fsyncSync(dirFd);
+		} finally {
+			closeSync(dirFd);
+		}
+	} catch {
+		// Windows cannot open a directory as a file; it also does not need this — the metadata
+		// journal already orders the create ahead of anything that could read the sidecar
+	}
+	return { inBand: false, sidecar: true };
+}