@harperfast/harper 5.0.25 → 5.1.0-beta.1

package/resources/indexes/HierarchicalNavigableSmallWorld.ts

@@ -1,11 +1,113 @@
-import { cosineDistance, euclideanDistance } from './vector.ts';
+import { cosineDistance, euclideanDistance, dotProductDistance } from './vector.ts';
 import { FLOAT32_OPTIONS } from 'msgpackr';
 import { loggerWithTag } from '../../utility/logging/logger.ts';
-import { ClientError } from '../../utility/errors/hdbError.js';
+import { ClientError } from '../../utility/errors/hdbError.ts';
 import type { Id } from '../../resources/ResourceInterface.ts';
 import { RocksDatabase } from '@harperfast/rocksdb-js';
 
 const logger = loggerWithTag('HNSW');
+
+// Optional int8 scalar quantization of stored vectors, enabled per-index via the
+// schema directive: `@indexed(type: "HNSW", quantization: "int8")`. The stored
+// graph node holds the vector as a compact int8 `bin` plus a per-vector `scale`,
+// roughly a 5x size reduction over the float32 array and ~10x cheaper to decode
+// (a single typed-array view instead of decoding 768 individually-tagged floats
+// into a boxed Array). The full-precision vector still lives on the record, so
+// only graph navigation is approximate; quantization recall loss is ~1%.
+//
+// Decode auto-detects the stored format (number[] = float, bin = int8), so an
+// int8-enabled index transparently reads legacy float nodes written before the
+// option was set.
+
+/** Symmetric int8 scalar-quantize a float vector. scale = max|component| / 127. */
+function quantizeInt8(vector: number[]): { bytes: Buffer; scale: number } {
+	let max = 0;
+	for (let i = 0; i < vector.length; i++) {
+		const a = vector[i] < 0 ? -vector[i] : vector[i];
+		if (a > max) max = a;
+	}
+	const scale = max / 127 || 1;
+	const inv = 1 / scale;
+	const q = new Int8Array(vector.length);
+	// clamp guards against a float-rounding edge landing on 128 (which Int8Array would wrap to -128)
+	for (let i = 0; i < vector.length; i++) q[i] = Math.max(-127, Math.min(127, Math.round(vector[i] * inv)));
+	return { bytes: Buffer.from(q.buffer, q.byteOffset, q.byteLength), scale };
+}
+
+/** Reconstruct an approximate float array from an int8 vector + scale. */
+function dequantizeInt8(q: Int8Array, scale: number): number[] {
+	const out = new Array(q.length);
+	for (let i = 0; i < q.length; i++) out[i] = q[i] * scale;
+	return out;
+}
+
+// Auto-scaled search ef, used only when an index does not explicitly configure efConstructionSearch
+// and a query does not pass its own ef. A fixed ef makes recall decay as the graph grows (it explores
+// a shrinking fraction of the graph), so ef grows with sqrt(node count), capped to bound search cost.
+// Constants from a recall/latency-vs-N sweep (768-dim cosine, int8): ef≈400 holds ~0.8 recall@10 from
+// 5K–30K, and the recall/latency tradeoff is steep (ef 800 at 30K ≈ 0.92 recall but ~2s p50), so the
+// cap deliberately favors latency — apps wanting higher recall set efConstructionSearch or a per-query
+// ef. Tune as graph build quality / larger-N data improves.
+const AUTO_EF_BASE = 100;
+const AUTO_EF_REF = 1000;
+const AUTO_EF_MAX = 512;
+function autoScaleEf(nodeCount: number): number {
+	const scaled = Math.round(AUTO_EF_BASE * Math.sqrt(Math.max(1, nodeCount / AUTO_EF_REF)));
+	return Math.min(AUTO_EF_MAX, Math.max(AUTO_EF_BASE, scaled));
+}
+
+class MinHeap {
+	private data: Candidate[] = [];
+	get size() {
+		return this.data.length;
+	}
+	push(item: Candidate) {
+		this.data.push(item);
+		let i = this.data.length - 1;
+		while (i > 0) {
+			const p = (i - 1) >> 1;
+			if (this.data[p].distance <= this.data[i].distance) break;
+			const tmp = this.data[p];
+			this.data[p] = this.data[i];
+			this.data[i] = tmp;
+			i = p;
+		}
+	}
+	pop(): Candidate | undefined {
+		if (this.data.length === 0) return undefined;
+		const top = this.data[0];
+		const last = this.data.pop()!;
+		if (this.data.length > 0) {
+			this.data[0] = last;
+			let i = 0;
+			for (;;) {
+				const l = 2 * i + 1,
+					r = l + 1;
+				let min = i;
+				if (l < this.data.length && this.data[l].distance < this.data[min].distance) min = l;
+				if (r < this.data.length && this.data[r].distance < this.data[min].distance) min = r;
+				if (min === i) break;
+				const tmp = this.data[min];
+				this.data[min] = this.data[i];
+				this.data[i] = tmp;
+				i = min;
+			}
+		}
+		return top;
+	}
+}
+
+function bisectInsert(arr: Candidate[], distance: number): number {
+	let lo = 0,
+		hi = arr.length;
+	while (lo < hi) {
+		const mid = (lo + hi) >> 1;
+		if (arr[mid].distance <= distance) lo = mid + 1;
+		else hi = mid;
+	}
+	return lo;
+}
+
 /**
  * Implementation of a vector index for Harper, using hierarchical navigable small world graphs.
  */
@@ -17,7 +119,9 @@ type Connection = {
 	distance: number;
 };
 type Node = {
-	vector: number[];
+	vector: number[] | Int8Array; // float nodes: number[]; quantized nodes: Int8Array (decoded from a bin)
+	scale?: number; // int8 dequantization scale; undefined on float nodes
+	invMag?: number; // cached 1/|vector| for cosine distance; undefined on legacy nodes
 	level?: number;
 	primaryKey: string;
 	[level: number]: Connection[];
@@ -34,6 +138,10 @@ type Node = {
  */
 export class HierarchicalNavigableSmallWorld {
 	static useObjectStore = true;
+	// Index options that only affect search, not the stored graph — changing them must not trigger a
+	// reindex (databases.ts persists the new value but skips rebuilding). efConstructionSearch is the
+	// search-time candidate-list size; the build uses efConstruction/M/distance, which are structural.
+	static searchOnlyOptions = ['efConstructionSearch'];
 	indexStore: any;
 	M: number = 16; // max number of connections per layer
 	efConstruction: number = 100; // size of dynamic candidate list
@@ -46,6 +154,8 @@ export class HierarchicalNavigableSmallWorld {
 
 	idIncrementer: BigInt64Array | undefined;
 	distance: (a: number[], b: number[]) => number;
+	int8 = false; // store vectors as int8-quantized bins (set via the `quantization` index option)
+	efSearchConfigured = false; // whether the schema set an explicit search ef; if not, search ef auto-scales with N
 	constructor(indexStore: any, options: any) {
 		this.indexStore = indexStore;
 		if (indexStore) {
@@ -53,7 +163,15 @@ export class HierarchicalNavigableSmallWorld {
 			// (we would actually like to use float16 if it were available)
 			this.indexStore.encoder.useFloat32 = FLOAT32_OPTIONS.ALWAYS;
 		}
-		this.distance = options?.distance === 'euclidean' ? euclideanDistance : cosineDistance;
+		this.int8 = options?.quantization === 'int8';
+		// Respect an explicitly-configured search ef (or efConstruction, which seeds it); otherwise auto-scale.
+		this.efSearchConfigured = options?.efConstructionSearch !== undefined || options?.efConstruction !== undefined;
+		this.distance =
+			options?.distance === 'euclidean'
+				? euclideanDistance
+				: options?.distance === 'dotProduct'
+					? dotProductDistance
+					: cosineDistance;
 		if (options) {
 			// allow all the HNSW parameters to be configured/tuned
 			if (options.M !== undefined) {
@@ -105,14 +223,29 @@ export class HierarchicalNavigableSmallWorld {
 		if (existingVector) {
 			// If we are updating an existing entry, we need to update the entry point
 			// if the new entry is closer to the entry point than the old one
-			oldNode = { ...this.indexStore.getSync(nodeId, options) };
+			oldNode = { ...this.safeGetSync(nodeId, options) };
 		} else oldNode = {} as Node;
 		if (vector) {
-			let entryPoint = entryPointId && this.indexStore.getSync(entryPointId, options);
+			// Pre-compute 1/|vector| for cosine distance so searchLayer can skip sqrt per neighbor
+			let invMag: number | undefined;
+			if (this.distance === cosineDistance) {
+				let magSq = 0;
+				for (const v of vector) magSq += v * v;
+				invMag = 1 / (Math.sqrt(magSq) || 1);
+			}
+			// Quantized storage form. The float `vector` is still used as the query for every
+			// searchLayer call below (asymmetric distance: float query x int8 stored); only what
+			// we PUT to the store is quantized.
+			const q = this.int8 ? quantizeInt8(vector) : undefined;
+			const storedVector: number[] | Buffer = q ? q.bytes : vector;
+			const storedScale = q ? q.scale : undefined;
+			let entryPoint = entryPointId && this.safeGetSync(entryPointId, options);
 			if (entryPoint == null) {
 				const level = Math.floor(-Math.log(Math.random()) * this.mL);
 				const node = {
-					vector,
+					vector: storedVector,
+					scale: storedScale,
+					invMag,
 					level,
 					primaryKey,
 				};
@@ -214,8 +347,11 @@ export class HierarchicalNavigableSmallWorld {
 
 					for (const { fromId, toId } of connectionsToBeReplaced) {
 						let from = updateNode(fromId);
-						if (!from) from = updateNode(fromId, this.indexStore.getSync(fromId, options));
-						for (let i = 0; i < from[l].length; i++) {
+						if (!from) from = updateNode(fromId, this.safeGetSync(fromId, options));
+						if (!from) continue;
+						const fromAtLevel = from[l];
+						if (!fromAtLevel) continue;
+						for (let i = 0; i < fromAtLevel.length; i++) {
 							if (from[l][i].id === toId) {
 								if (Object.isFrozen(from[l])) {
 									from[l] = from[l].slice();
@@ -264,7 +400,9 @@ export class HierarchicalNavigableSmallWorld {
 			this.indexStore.put(
 				nodeId,
 				{
-					vector,
+					vector: storedVector,
+					scale: storedScale,
+					invMag,
 					level,
 					primaryKey,
 					...connections,
@@ -315,7 +453,7 @@ export class HierarchicalNavigableSmallWorld {
 				const oldConnections = oldNode[l];
 				for (const { id: neighborId } of oldConnections) {
 					// get and copy the neighbor node so we can modify it
-					const neighborNode = updateNode(neighborId, this.indexStore.getSync(neighborId, options));
+					const neighborNode = updateNode(neighborId, this.safeGetSync(neighborId, options));
 					if (!neighborNode) continue;
 					for (let l2 = 0; l2 <= l; l2++) {
 						// remove the connection to this node from the neighbor node
@@ -324,7 +462,13 @@ export class HierarchicalNavigableSmallWorld {
 						});
 						if (neighborNode[l2]?.length === 0) {
 							logger.trace?.('node was left orphaned, will reindex', neighborId);
-							needsReindexing.set(neighborNode.primaryKey, neighborNode.vector);
+							// reindex re-feeds this vector into index() as a float query, so dequantize int8 back to float
+							needsReindexing.set(
+								neighborNode.primaryKey,
+								neighborNode.scale !== undefined
+									? dequantizeInt8(neighborNode.vector as Int8Array, neighborNode.scale)
+									: neighborNode.vector
+							);
 						}
 					}
 				}
@@ -346,14 +490,34 @@ export class HierarchicalNavigableSmallWorld {
 		for (const [key, vector] of needsReindexing) {
 			this.index(key, vector, vector, options);
 		}
-		this.checkSymmetry(nodeId, this.indexStore.getSync(nodeId, options), options);
+		this.checkSymmetry(nodeId, this.safeGetSync(nodeId, options), options);
+	}
+
+	private safeGetSync(key: any, options?: any): any {
+		try {
+			const node = this.indexStore.getSync(key, options);
+			// A quantized vector decodes as a bin (Uint8Array/Buffer) that is a view into the
+			// store's read buffer, which may be reused on the next getSync — so copy the bytes
+			// into a retained Int8Array (raw two's-complement reinterpret). The Int8Array guard
+			// skips re-conversion when the object store (useObjectStore) hands back an
+			// already-converted cached node. Float nodes (vector is a number[]) pass through.
+			if (node && node.vector && !Array.isArray(node.vector) && !(node.vector instanceof Int8Array)) {
+				const u8 = node.vector as Uint8Array;
+				node.vector = new Int8Array(u8.buffer, u8.byteOffset, u8.byteLength).slice();
+			}
+			return node;
+		} catch {
+			logger.warn?.('Failed to decode HNSW node, skipping', key);
+			return undefined;
+		}
 	}
 
 	private getEntryPoint(options: { transaction?: any } = {}) {
 		// Get entry point
 		const entryPointId = this.indexStore.getSync(ENTRY_POINT, options);
 		if (entryPointId === undefined) return;
-		const node = this.indexStore.getSync(entryPointId, options);
+		const node = this.safeGetSync(entryPointId, options);
+		if (!node) return;
 		return { id: entryPointId, ...node };
 	}
 
@@ -379,50 +543,82 @@ export class HierarchicalNavigableSmallWorld {
 		options: { transaction?: any } = {},
 		distanceFunction = this.distance
 	): SearchResults {
+		// Pre-compute query magnitude for cosine; use cached invMag on stored nodes to skip sqrt per neighbor.
+		// Asymmetric distance: the query stays full-precision float; a stored neighbor may be int8
+		// (with per-vector `scaleB`) or float (`scaleB` undefined).
+		let computeDistance: (b: number[] | Int8Array, invMagB?: number, scaleB?: number) => number;
+		if (distanceFunction === cosineDistance) {
+			let magASq = 0;
+			for (const v of queryVector) magASq += v * v;
+			const invMagA = 1 / (Math.sqrt(magASq) || 1);
+			computeDistance = (b: number[] | Int8Array, invMagB?: number, scaleB?: number) => {
+				let dot = 0;
+				for (let i = 0; i < b.length; i++) dot += queryVector[i] * (b[i] as number);
+				if (scaleB !== undefined) dot *= scaleB; // dequantize the int8 dot product
+				if (invMagB !== undefined) return 1 - dot * invMagA * invMagB;
+				// Fallback when the stored node has no cached invMag (a non-cosine index queried as
+				// cosine). Compute the stored magnitude and dequantize it by scaleB so it matches the
+				// already-dequantized dot product.
+				let magBSq = 0;
+				for (let i = 0; i < b.length; i++) magBSq += (b[i] as number) * (b[i] as number);
+				let magB = Math.sqrt(magBSq) || 1;
+				if (scaleB !== undefined) magB *= scaleB;
+				return 1 - (dot * invMagA) / magB;
+			};
+		} else if (distanceFunction === euclideanDistance) {
+			// Asymmetric squared-euclidean, dequantizing each int8 component inline (no allocation).
+			computeDistance = (b: number[] | Int8Array, _invMagB?: number, scaleB?: number) => {
+				if (scaleB === undefined) return distanceFunction(queryVector, b as number[]);
+				let distanceSquared = 0;
+				for (let i = 0; i < b.length; i++) {
+					const diff = queryVector[i] - (b[i] as number) * scaleB;
+					distanceSquared += diff * diff;
+				}
+				return distanceSquared;
+			};
+		} else {
+			// Negated inner product, dequantizing the int8 dot product inline (no allocation).
+			computeDistance = (b: number[] | Int8Array, _invMagB?: number, scaleB?: number) => {
+				if (scaleB === undefined) return distanceFunction(queryVector, b as number[]);
+				let dot = 0;
+				for (let i = 0; i < b.length; i++) dot += queryVector[i] * (b[i] as number);
+				return -(dot * scaleB);
+			};
+		}
+
 		const visited = new Set([entryPointId]);
-		const candidates = [
-			{
-				id: entryPointId,
-				distance: distanceFunction(queryVector, entryPoint.vector),
-				node: entryPoint,
-			},
-		];
-		const results = [...candidates] as SearchResults;
-
-		while (candidates.length > 0) {
-			// Get closest unvisited element
-			candidates.sort((a, b) => a.distance - b.distance);
-			const current = candidates.shift();
-
-			// Get least result distance
+		const initialCandidate: Candidate = {
+			id: entryPointId,
+			distance: computeDistance(entryPoint.vector, entryPoint.invMag, entryPoint.scale),
+			node: entryPoint,
+		};
+
+		const candidates = new MinHeap();
+		candidates.push(initialCandidate);
+		const results = [initialCandidate] as SearchResults;
+
+		while (candidates.size > 0) {
+			const current = candidates.pop()!;
 			const furthestDistance = results[results.length - 1].distance;
 
-			// If current candidate is less similar than our worst result, we're done
 			if (current.distance > furthestDistance) break;
 
-			// Check neighbors of current point
-			const currentNode = current.node;
-			for (const { id: neighborId } of currentNode[level] || []) {
+			for (const { id: neighborId } of current.node[level] || []) {
 				if (visited.has(neighborId) || neighborId === undefined) continue;
 				visited.add(neighborId);
 
-				const neighbor = this.indexStore.getSync(neighborId, options);
+				const neighbor = this.safeGetSync(neighborId, options);
 				if (!neighbor) continue;
 				this.nodesVisitedCount++;
-				const distance = distanceFunction(queryVector, neighbor.vector);
+				const distance = computeDistance(neighbor.vector, neighbor.invMag, neighbor.scale);
 
 				if (distance < furthestDistance || results.length < ef) {
-					const candidate = {
-						id: neighborId,
-						distance,
-						node: neighbor,
-					};
+					const candidate: Candidate = { id: neighborId, distance, node: neighbor };
 					candidates.push(candidate);
-					results.push(candidate);
+					results.splice(bisectInsert(results, distance), 0, candidate);
+					if (results.length > ef) results.pop();
 				}
 			}
-			results.sort((a, b) => a.distance - b.distance);
-			if (results.length > ef) results.splice(ef, results.length - ef);
 		}
 		results.visited = visited.size;
 		return results;
@@ -446,12 +642,14 @@ export class HierarchicalNavigableSmallWorld {
 			descending,
 			distance,
 			comparator,
+			ef,
 		}: {
 			target: number[];
 			value: number;
 			descending: boolean;
 			distance: string;
 			comparator: string;
+			ef?: number;
 		},
 		context: any
 	) {
@@ -470,12 +668,23 @@ export class HierarchicalNavigableSmallWorld {
 		let distanceFunction: (a: number[], b: number[]) => number;
 		if (distance === 'cosine') distanceFunction = cosineDistance;
 		else if (distance === 'euclidean') distanceFunction = euclideanDistance;
+		else if (distance === 'dotProduct') distanceFunction = dotProductDistance;
 		else if (distance) throw new ClientError('Unknown distance function');
 		else distanceFunction = this.distance;
 		if (!target) throw new ClientError('A target vector must be provided for an HNSW query');
 		if (!Array.isArray(target)) throw new ClientError('The target vector must be an array');
 
 		const options = context.transaction; // should have a nested RocksDB transaction
+		// Resolve search ef: per-query ef wins; else an explicitly-configured efConstructionSearch;
+		// else auto-scale with the graph size so recall holds as the table grows.
+		let effectiveEf = this.efConstructionSearch;
+		if (ef !== undefined && ef > 0) effectiveEf = ef;
+		else if (!this.efSearchConfigured) {
+			const nodeCount = this.indexStore.getKeysCount
+				? this.indexStore.getKeysCount()
+				: (this.indexStore.getStats?.()?.entryCount ?? 0);
+			effectiveEf = autoScaleEf(nodeCount);
+		}
 		let entryPoint = this.getEntryPoint(options);
 		if (!entryPoint) return [];
 		let entryPointId = entryPoint.id;
@@ -483,15 +692,7 @@ export class HierarchicalNavigableSmallWorld {
 		// For each level from top to bottom
 		for (let l = entryPoint.level; l >= 0; l--) {
 			// Search for closest neighbors at current level
-			results = this.searchLayer(
-				target,
-				entryPointId,
-				entryPoint,
-				this.efConstructionSearch,
-				l,
-				options,
-				distanceFunction
-			);
+			results = this.searchLayer(target, entryPointId, entryPoint, effectiveEf, l, options, distanceFunction);
 
 			if (results.length > 0) {
 				const neighbor = results[0]; // closest neighbor becomes new entry point
@@ -506,6 +707,27 @@ export class HierarchicalNavigableSmallWorld {
 			distance: candidate.distance,
 		}));
 	}
+	/**
+	 * Exact distance between a query and a record's FULL-precision vector, mirroring search()'s metric
+	 * selection. Used to rerank quantized (int8) results: graph traversal navigates on approximate
+	 * (quantized) distances, but the caller has the exact record vector and can restore exact ordering
+	 * and $distance.
+	 */
+	exactDistance(searchCondition: { target: number[]; distance?: string }, recordVector: number[] | Int8Array): number {
+		if (recordVector == null) return Infinity; // missing vector sorts last
+		// distance fns require a plain Array (they guard on Array.isArray); records normally store a
+		// float[] vector, but convert defensively in case a typed array slips through.
+		const vec = Array.isArray(recordVector) ? recordVector : Array.from(recordVector);
+		const fn =
+			searchCondition.distance === 'euclidean'
+				? euclideanDistance
+				: searchCondition.distance === 'dotProduct'
+					? dotProductDistance
+					: searchCondition.distance === 'cosine'
+						? cosineDistance
+						: this.distance;
+		return fn(searchCondition.target, vec);
+	}
 	private checkSymmetry(id, node, options) {
 		if (!node) return;
 		let l = 0;
@@ -514,9 +736,9 @@ export class HierarchicalNavigableSmallWorld {
 			// verify that the level is not empty, otherwise this means we have an orphaned node
 			if (connections.length === 0) break;
 			for (const { id: neighbor } of connections) {
-				const neighborNode = this.indexStore.getSync(neighbor, options);
+				const neighborNode = this.safeGetSync(neighbor, options);
 				if (!neighborNode) {
-					logger.info?.('could not find neighbor node', neighborNode);
+					logger.info?.('could not find neighbor node', neighbor);
 					continue;
 				}
 				// verify that the connection is symmetrical
@@ -562,7 +784,7 @@ export class HierarchicalNavigableSmallWorld {
 			node[level] = keptConnections;
 			// For removed connections, ensure there's still a path to them
 			for (const removed of removedConnections) {
-				let removedNode = updateNode(removed.id) ?? this.indexStore.getSync(removed.id, options);
+				let removedNode = updateNode(removed.id) ?? this.safeGetSync(removed.id, options);
 				if (removedNode) {
 					// Remove the reverse connection if it exists
 					if (removedNode[level]) {
@@ -589,22 +811,27 @@ export class HierarchicalNavigableSmallWorld {
 	}
 	validateConnectivity(startLevel: number = 0) {
 		const entryPoint = this.getEntryPoint();
+		if (!entryPoint) return;
 		const visited = new Set<number>();
 
-		// BFS from entry point to ensure all nodes are reachable
-		const queue = [entryPoint.id];
-		visited.add(entryPoint.id);
+		// BFS from entry point to ensure all nodes are reachable. Asymmetric stale neighbor
+		// references can survive deletes, so a referenced node may not actually exist anymore;
+		// only count a node as visited once we confirm the underlying record is present.
+		const queue: number[] = [entryPoint.id];
+		const enqueued = new Set<number>([entryPoint.id]);
 		let connections = 0;
 
 		while (queue.length > 0) {
 			const currentId = queue.shift()!;
-			const current = this.indexStore.getSync(currentId);
+			const current = this.safeGetSync(currentId);
+			if (!current) continue;
+			visited.add(currentId);
 
 			for (let level = startLevel; level <= current.level; level++) {
 				for (const { id: neighborId } of current[level] || []) {
 					connections++;
-					if (!visited.has(neighborId)) {
-						visited.add(neighborId);
+					if (!enqueued.has(neighborId)) {
+						enqueued.add(neighborId);
 						queue.push(neighborId);
 					}
 				}
@@ -613,9 +840,6 @@ export class HierarchicalNavigableSmallWorld {
 
 		// Check if all nodes are reachable
 		// This would require maintaining a separate set/count of all nodes
-		if (visited.size !== this.totalNodes) {
-			console.log('visited', visited.size, 'total', this.totalNodes);
-		}
 		return {
 			isFullyConnected: visited.size === this.totalNodes,
 			averageConnections: connections / visited.size,
@@ -661,7 +885,12 @@ export class HierarchicalNavigableSmallWorld {
 
 			let distanceFunction = this.distance;
 			if (sortDefinition.type)
-				distanceFunction = sortDefinition.distance === 'euclidean' ? euclideanDistance : cosineDistance;
+				distanceFunction =
+					sortDefinition.distance === 'euclidean'
+						? euclideanDistance
+						: sortDefinition.distance === 'dotProduct'
+							? dotProductDistance
+							: cosineDistance;
 			const distance = distanceFunction(sortDefinition.target, vector);
 			vectorDistances.set(entry, distance);
 			return distance;

package/resources/indexes/vector.ts

@@ -36,3 +36,20 @@ export function cosineDistance(a: number[], b: number[]): number {
 
 	return 1 - dotProduct / (magnitudeA * magnitudeB || 1);
 }
+
+export function dotProductDistance(a: number[], b: number[]): number {
+	if (!Array.isArray(a) || !Array.isArray(b)) {
+		throw new Error('Inner product comparison requires an array');
+	}
+
+	let dotProduct = 0;
+	const length = Math.max(a.length, b.length);
+
+	for (let i = 0; i < length; i++) {
+		const va = a[i] || 0;
+		const vb = b[i] || 0;
+		dotProduct += va * vb;
+	}
+
+	return -dotProduct;
+}

package/resources/loadEnv.ts

@@ -1,22 +1,26 @@
 import { parse } from 'dotenv';
-import logger from '../utility/logging/harper_logger.js';
+import logger from '../utility/logging/harper_logger.ts';
+import { Scope } from '../components/Scope.ts';
 
-export function start({ override }: { override: boolean }) {
-	return {
-		handleFile: (contents, _, filePath) => {
-			logger.debug(`Loading env file: ${filePath}`);
-			for (const [key, value] of Object.entries(parse(contents))) {
-				if (process.env[key] !== undefined) {
-					logger.warn(`Environment variable conflict: ${key} from ${filePath} is already set on process.env`);
-					if (override) {
-						logger.debug(`override option enabled. overriding environment variable: ${key}`);
-					} else {
-						continue;
-					}
+export function handleApplication(scope: Scope) {
+	const override = (scope.options.getAll() as { override?: boolean }).override ?? false;
+	scope.handleEntry((entry) => {
+		if (entry.eventType !== 'add') {
+			scope.requestRestart();
+			return;
+		}
+		logger.debug(`Loading env file: ${entry.absolutePath}`);
+		for (const [key, value] of Object.entries(parse(entry.contents))) {
+			if (process.env[key] !== undefined) {
+				logger.warn(`Environment variable conflict: ${key} from ${entry.absolutePath} is already set on process.env`);
+				if (override) {
+					logger.debug(`override option enabled. overriding environment variable: ${key}`);
+				} else {
+					continue;
 				}
-
-				process.env[key] = value;
 			}
-		},
-	};
+
+			process.env[key] = value;
+		}
+	});
 }

package/resources/login.ts

@@ -1,10 +1,12 @@
 import { Resource } from './Resource.ts';
-export function start({ resources }) {
-	resources.set('login', Login);
-	resources.loginPath = (request) => {
+import { Scope } from '../components/Scope.ts';
+export function handleApplication(scope: Scope) {
+	scope.resources.set('login', Login);
+	scope.resources.loginPath = (request) => {
 		return '/login?redirect=' + encodeURIComponent(request.url);
 	};
 }
+// @ts-ignore
 class Login extends Resource {
 	static async get(_id, _body, _request) {
 		// TODO: Return a login page