@harperfast/harper 5.3.0-alpha.1 → 5.3.0-beta.2

package/resources/indexes/hnswPlaneBinding.ts

@@ -0,0 +1,174 @@
+import { closeSync, fsyncSync, openSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { loggerWithTag } from '../../utility/logging/logger.ts';
+
+const logger = loggerWithTag('HNSW');
+
+/**
+ * Parallel arrays, ascending by distance: hit i is (ids[i], distances[i]) with its host key at
+ * keys.subarray(keyEnds[i - 1] ?? 0, keyEnds[i]); an empty key means the node vanished mid-query.
+ */
+export interface PlaneSearchHits {
+	ids: Uint32Array;
+	distances: Float32Array;
+	keys: Buffer;
+	keyEnds: Uint32Array;
+}
+
+/** A predicate batch: candidate ids and their host keys, laid out as in PlaneSearchHits. */
+export type PlanePredicate = (ids: number[], keys: Buffer, keyEnds: Uint32Array) => Uint8Array;
+
+/** NAPI surface of the native file-primary HNSW index (`@harperfast/hnsw`). */
+export interface HnswPlane {
+	readonly dims: number;
+	readonly layer0Cap: number;
+	readonly keyCap: number;
+	insert(vector: Float32Array, key?: Buffer): number;
+	remove(id: number): void;
+	writeNodeRaw(
+		id: number,
+		level: number,
+		vector: Buffer,
+		scale: number,
+		invMag: number,
+		neighbors: Uint32Array,
+		upper: Uint32Array[] | null
+	): void;
+	clearNode(id: number): void;
+	setEntryPoint(id: number, level: number): void;
+	getEntryPoint(): number[];
+	search(
+		vector: Float32Array,
+		k: number,
+		ef: number,
+		filter?: Uint8Array | null,
+		filterExpansion?: number | null
+	): Promise<PlaneSearchHits>;
+	searchWithPredicate(
+		vector: Float32Array,
+		k: number,
+		ef: number,
+		predicate: PlanePredicate,
+		filterExpansion?: number | null,
+		visitBudget?: number | null
+	): Promise<PlaneSearchHits>;
+	searchSync(vector: Float32Array, k: number, ef: number): PlaneSearchHits;
+	writeNodeRawIfAbsent(
+		id: number,
+		level: number,
+		vector: Buffer,
+		scale: number,
+		invMag: number,
+		neighbors: Uint32Array,
+		upper: Uint32Array[] | null
+	): boolean;
+	openedClean(): boolean;
+	idHighWater(): number;
+	getWatermark(): number;
+	setWatermark(txn: number): void;
+	flush(watermark?: number): void;
+	flushAsync(watermark?: number): Promise<void>;
+	invalidateFile(): PlaneInvalidationOutcome;
+	invalidated(): boolean;
+}
+
+export interface HnswPlaneConstructor {
+	readonly prototype: HnswPlane;
+	create(path: string, dims: number, layer0Cap: number, maxNodes: number, keyCap?: number): HnswPlane;
+	open(path: string): HnswPlane;
+}
+
+export interface PlaneInvalidationOutcome {
+	inBand: boolean;
+	sidecar: boolean;
+	inBandError?: string;
+	sidecarError?: string;
+}
+
+interface HnswPlanePackage {
+	Plane: HnswPlaneConstructor;
+	invalidatePlane(path: string): PlaneInvalidationOutcome;
+	stalePathFor(path: string): string;
+}
+
+/** Entry-point id meaning "none" (u32::MAX in the plane header). */
+export const PLANE_NO_ID = 0xffffffff;
+
+/**
+ * Where an index's plane file lives: next to its store, named by the dbiKey
+ * (`table/attribute`, flattened to a single file name). Exposed separately from the index
+ * instance so crash-recovery drop paths can remove the file without opening the index.
+ */
+export function planeFilePathFor(storePath: string, storeName: string): string {
+	// encodeURIComponent is injective and never emits a path separator: table `a` attribute
+	// `b.c` and table `a.b` attribute `c` must not share a plane file (dot-flattening let two
+	// indexes serve each other's node ids as their own primary keys)
+	return join(storePath, `${encodeURIComponent(storeName)}.hnsw`);
+}
+
+/**
+ * Tombstone marking a plane file that could not be deleted (e.g. Windows EBUSY while still
+ * mapped). Its presence means the plane file is STALE: never open it — delete both when
+ * possible and rebuild.
+ */
+export function planeStalePathFor(planePath: string): string {
+	// the package owns the convention; the literal covers the calls that precede its load
+	return binding?.stalePathFor(planePath) ?? `${planePath}.stale`;
+}
+
+let binding: HnswPlanePackage | null | undefined;
+let bindingLoadError: Error | undefined;
+let bindingWarningLogged = false;
+
+function getHnswPackage(warn = true): HnswPlanePackage | null {
+	if (binding === undefined) {
+		try {
+			binding = require('@harperfast/hnsw') as HnswPlanePackage;
+		} catch (error) {
+			binding = null;
+			bindingLoadError = error as Error;
+		}
+	}
+	if (binding === null && warn && !bindingWarningLogged) {
+		bindingWarningLogged = true;
+		logger.warn?.(
+			`The @harperfast/hnsw native module is not available (${bindingLoadError?.message}); ` +
+				'indexes with nativePlane enabled will remain unavailable until the module can load'
+		);
+	}
+	return binding;
+}
+
+/** The native plane constructor, or null when the compiled artifact is unavailable (warns once by default). */
+export function getPlaneBinding(warn = true): HnswPlaneConstructor | null {
+	return getHnswPackage(warn)?.Plane ?? null;
+}
+
+/** Make a derived plane unadoptable before it is replaced or removed. */
+export function invalidatePlaneFile(filePath: string, attached?: HnswPlane | null): PlaneInvalidationOutcome {
+	if (attached) return attached.invalidateFile();
+	const hnswPackage = getHnswPackage();
+	if (hnswPackage) return hnswPackage.invalidatePlane(filePath);
+	// A plane can outlive the package that made it (uninstall, or a prebuild that stopped
+	// loading), and a reinstall would then adopt it. Only the sidecar is reachable without the
+	// package, and it has to survive a power loss to be worth writing, so fsync it and the
+	// directory entry that names it — the same durability the package's own sidecar has.
+	const fd = openSync(planeStalePathFor(filePath), 'w');
+	try {
+		fsyncSync(fd);
+	} finally {
+		closeSync(fd);
+	}
+	try {
+		const dirFd = openSync(dirname(filePath), 'r');
+		try {
+			fsyncSync(dirFd);
+		} finally {
+			closeSync(dirFd);
+		}
+	} catch {
+		// Windows cannot open a directory as a file; it also does not need this — the metadata
+		// journal already orders the create ahead of anything that could read the sidecar
+	}
+	return { inBand: false, sidecar: true };
+}

package/resources/indexes/nativeFullTextDerivedIndexLifecycle.ts

@@ -0,0 +1,181 @@
+import { createHash } from 'node:crypto';
+import { isAbsolute, join, resolve } from 'node:path';
+import { loggerWithTag } from '../../utility/logging/logger.ts';
+import {
+	FullTextDerivedIndexBackend,
+	HARPER_FULLTEXT_MAX_CURSOR_PAYLOAD_BYTES,
+	type FullTextDerivedIndexBackendOptions,
+	type FullTextDerivedIndexEngine,
+	type FullTextDerivedIndexInspection,
+} from './fullTextDerivedIndex.ts';
+import {
+	getValidatedFullTextRuntimeInfo,
+	loadFullTextNativeBinding,
+	type NativeFullTextIndexConfiguration,
+	type NativeFullTextModule,
+	validateFullTextNativeBinding,
+} from './fullTextNativeBinding.ts';
+
+const logger = loggerWithTag('fulltext-derived-index');
+
+export type NativeFullTextDerivedIndexLifecycleOptions = NativeFullTextIndexConfiguration & {
+	storePath: string;
+	storeName: string;
+	indexId: string;
+	sourceGeneration: string;
+	binding?: NativeFullTextModule | (() => Promise<NativeFullTextModule>);
+};
+
+export type NativeFullTextDerivedIndexBackendOptions = Omit<FullTextDerivedIndexBackendOptions, 'lifecycle'> &
+	Omit<NativeFullTextDerivedIndexLifecycleOptions, 'indexId'>;
+
+export class NativeFullTextDerivedIndexLifecycle {
+	readonly #options: NativeFullTextDerivedIndexLifecycleOptions;
+	readonly #path: string;
+	readonly #generation: string;
+	#binding?: NativeFullTextModule;
+	#maxCommitPayloadBytes?: number;
+	#reclaimOperation = Promise.resolve();
+
+	constructor(options: NativeFullTextDerivedIndexLifecycleOptions) {
+		if (!isAbsolute(options.storePath)) throw new TypeError('Full-text storePath must be absolute');
+		if (!options.storeName) throw new TypeError('Full-text storeName is required');
+		if (!options.indexId) throw new TypeError('Full-text indexId is required');
+		if (!options.sourceGeneration) throw new TypeError('Full-text sourceGeneration is required');
+		this.#options = {
+			...options,
+			fields: options.fields.map((field) => ({ ...field })),
+			limits: { ...options.limits },
+		};
+		// Hashing bounds filenames for arbitrary store names; the index id remains in native metadata and logs.
+		this.#path = join(resolve(options.storePath), `${digest(options.storeName)}.fulltext`);
+		this.#generation = digest(options.sourceGeneration);
+	}
+
+	get path(): string {
+		return this.#path;
+	}
+
+	async initialize(options: { reclaimRetired?: boolean } = {}): Promise<void> {
+		const binding = await this.#getBinding();
+		binding.validateNativeFullTextIndexOptions({
+			...this.#nativeOptions(),
+			limits: this.#options.limits,
+		});
+		this.#maxCommitPayloadBytes = getValidatedFullTextRuntimeInfo(binding).limits.maxCommitPayloadBytes;
+		if (options.reclaimRetired !== false) this.#queueReclaimRetired();
+	}
+
+	startRetiredStorageReclamation(): void {
+		this.#requireBinding();
+		// Fulltext publishes unique retired paths and removes them with force, so workers may sweep concurrently.
+		this.#queueReclaimRetired();
+	}
+
+	get maxCommitPayloadBytes(): number {
+		if (this.#maxCommitPayloadBytes === undefined) throw new Error('Full-text lifecycle has not been initialized');
+		return this.#maxCommitPayloadBytes;
+	}
+
+	inspect(): FullTextDerivedIndexInspection {
+		return this.#requireBinding().inspectNativeFullTextIndex(this.#nativeOptions());
+	}
+
+	async open(): Promise<FullTextDerivedIndexEngine> {
+		return this.#requireBinding().openNativeFullTextIndex({
+			...this.#nativeOptions(),
+			limits: this.#options.limits,
+		});
+	}
+
+	async reset(): Promise<void> {
+		const result = await this.#requireBinding().resetNativeFullTextIndex({
+			path: this.#path,
+			indexId: this.#options.indexId,
+		});
+		this.#queueReclaimRetired(result.state === 'reset' ? result.retiredPath : undefined);
+	}
+
+	#queueReclaimRetired(retiredPath?: string): void {
+		const reclaim = () => this.#reclaimRetired(retiredPath);
+		this.#reclaimOperation = this.#reclaimOperation.then(reclaim, reclaim).catch((error) => {
+			logWarning(`Could not complete reclamation for full-text index '${this.#options.indexId}'`, error);
+		});
+	}
+
+	async #reclaimRetired(retiredPath?: string): Promise<void> {
+		try {
+			const result = await this.#requireBinding().reclaimRetiredNativeFullTextIndexes({
+				path: this.#path,
+				retiredPath,
+			});
+			if (result.failed > 0)
+				logWarning(`Could not remove ${result.failed} retired paths for full-text index '${this.#options.indexId}'`);
+		} catch (error) {
+			logWarning(`Could not reclaim retired paths for full-text index '${this.#options.indexId}'`, error);
+		}
+	}
+
+	#nativeOptions(): Omit<NativeFullTextIndexConfiguration, 'limits'> & {
+		path: string;
+		indexId: string;
+		generation: string;
+	} {
+		return {
+			path: this.#path,
+			indexId: this.#options.indexId,
+			generation: this.#generation,
+			fields: this.#options.fields,
+			analyzer: this.#options.analyzer,
+			stopWords: this.#options.stopWords,
+			positions: this.#options.positions,
+			surfaceTerms: this.#options.surfaceTerms,
+		};
+	}
+
+	async #getBinding(): Promise<NativeFullTextModule> {
+		if (this.#binding) return this.#binding;
+		const configured = this.#options.binding;
+		this.#binding = configured
+			? await validateFullTextNativeBinding(typeof configured === 'function' ? await configured() : configured)
+			: await loadFullTextNativeBinding();
+		return this.#binding;
+	}
+
+	#requireBinding(): NativeFullTextModule {
+		if (!this.#binding) throw new Error('Full-text native binding has not been initialized');
+		return this.#binding;
+	}
+}
+
+export async function createNativeFullTextDerivedIndexBackend(
+	options: NativeFullTextDerivedIndexBackendOptions
+): Promise<FullTextDerivedIndexBackend> {
+	const maxCursorPayloadBytes = options.maxCursorPayloadBytes ?? HARPER_FULLTEXT_MAX_CURSOR_PAYLOAD_BYTES;
+	if (maxCursorPayloadBytes > HARPER_FULLTEXT_MAX_CURSOR_PAYLOAD_BYTES)
+		throw new RangeError(`Full-text maxCursorPayloadBytes must not exceed ${HARPER_FULLTEXT_MAX_CURSOR_PAYLOAD_BYTES}`);
+	const lifecycle = new NativeFullTextDerivedIndexLifecycle({ ...options, indexId: options.id });
+	await lifecycle.initialize({ reclaimRetired: false });
+	if (maxCursorPayloadBytes > lifecycle.maxCommitPayloadBytes)
+		throw new RangeError('Full-text maxCursorPayloadBytes exceeds the native commit payload capacity');
+	const backend = new FullTextDerivedIndexBackend({
+		...options,
+		id: options.id,
+		lifecycle,
+		maxCursorPayloadBytes,
+	});
+	lifecycle.startRetiredStorageReclamation();
+	return backend;
+}
+
+function digest(value: string): string {
+	return createHash('sha256').update(value).digest('hex');
+}
+
+function logWarning(message: string, error?: unknown): void {
+	const code =
+		error && typeof error === 'object' && 'code' in error && typeof error.code === 'string' ? error.code : undefined;
+	try {
+		logger.warn?.(`${message}${code ? ` (${code})` : ''}`);
+	} catch {}
+}

package/resources/nodeIdMapping.ts

@@ -50,6 +50,7 @@ function getIdMappingRecord(auditStore) {
 		// now we can take over the local node id
 		nameToId[node_name] = 0;
 		auditStore.putSync(REMOTE_NODE_IDS, pack(idMappingRecord));
+		invalidateNodeNames(auditStore);
 	}
 	return idMappingRecord;
 }
@@ -84,6 +85,7 @@ export function remoteToLocalNodeId(remoteMapping: any, auditStore: any) {
 	}
 	if (hasChanges) {
 		auditStore.putSync(REMOTE_NODE_IDS, pack(idMappingRecord));
+		invalidateNodeNames(auditStore);
 	}
 	return remoteToLocalId;
 }
@@ -103,6 +105,7 @@ export function getIdOfRemoteNode(remoteNodeName, auditStore) {
 		id = lastId + 1;
 		nameToId[remoteNodeName] = id;
 		auditStore.putSync(REMOTE_NODE_IDS, pack(idMappingRecord));
+		invalidateNodeNames(auditStore);
 	}
 	logger.trace?.('The remote node name map', remoteNodeName, nameToId, id);
 	return id;
@@ -123,3 +126,50 @@ export function lastTimeInAuditStore(auditStore: Database) {
 export function getThisNodeId(auditStore: any) {
 	return exportIdMapping(auditStore)?.[server.hostname];
 }
+
+// Inverted id -> name map. exportIdMapping() re-reads and unpacks the mapping record on every call,
+// which is far too much per replicated entry on the apply thread; ids are stable once minted, so the
+// inversion is cached and dropped wherever nameToId is written. Keyed per audit store because short
+// ids are minted per database in first-seen order — id 1 names a different node in each one.
+// Refresh window for a MISS. A hit is always trusted, but a miss cannot be: ids are minted by
+// whichever worker first talks to a peer, and invalidation only reaches that worker's own copy — so
+// a newly admitted node is absent from every other worker's map until it is rebuilt. Rebuilding on
+// every miss would put a store read and unpack back on the apply thread for each unmapped entry, so
+// misses re-read at most this often — short enough that a joining node loses at most a stray entry,
+// long enough that a burst of unmapped ids cannot drive the store. `onMiss` records whether the
+// cached map was built by a `rebuildOnMiss` caller, which is what bounds that caller to one read per
+// window too rather than exempting it from the interval entirely.
+const NODE_NAME_REFRESH_MS = 50;
+const idToNodeName = new WeakMap<object, { names: Map<number, string>; refreshedAt: number; onMiss: boolean }>();
+function invalidateNodeNames(auditStore: any) {
+	idToNodeName.delete(auditStore);
+}
+
+/**
+ * The node name a local short id refers to. Callers that must attribute a replicated entry to its
+ * origin node need the globally stable name, not the id, which is assigned per node.
+ */
+export function getNodeNameForId(
+	auditStore: any,
+	nodeId: number | undefined,
+	rebuildOnMiss = false
+): string | undefined {
+	if (typeof nodeId !== 'number' || !auditStore) return undefined;
+	const cached = idToNodeName.get(auditStore);
+	const hit = cached?.names.get(nodeId);
+	if (hit !== undefined) return hit;
+	const now = Date.now();
+	// `rebuildOnMiss` is for entry classes whose loss costs more than the read: a dropped record lock
+	// release leaves the home holding a grant until its own deadline, so a just-minted id is worth one
+	// store read rather than up to `NODE_NAME_REFRESH_MS` of dropped entries. ONE read, though, not one
+	// per entry — an id this map will never resolve (a node whose mapping was purged, an origin
+	// relayed from a database that never minted one) is exactly the burst the interval exists to keep
+	// off the apply thread, and a rebuild inside the window is not stale: a write to the mapping drops
+	// this entry outright (`invalidateNodeNames`).
+	if (cached && now - cached.refreshedAt < NODE_NAME_REFRESH_MS && (!rebuildOnMiss || cached.onMiss)) return undefined;
+	const nameToId = exportIdMapping(auditStore);
+	const names = new Map<number, string>();
+	for (const name in nameToId) names.set(nameToId[name], name);
+	idToNodeName.set(auditStore, { names, refreshedAt: now, onMiss: rebuildOnMiss });
+	return names.get(nodeId);
+}