@harperfast/harper 5.0.30 → 5.1.0-beta.2

package/dist/resources/indexes/HierarchicalNavigableSmallWorld.js

@@ -4,9 +4,112 @@ exports.HierarchicalNavigableSmallWorld = void 0;
 const vector_ts_1 = require("./vector.js");
 const msgpackr_1 = require("msgpackr");
 const logger_ts_1 = require("../../utility/logging/logger.js");
-const hdbError_js_1 = require("../../utility/errors/hdbError.js");
+const hdbError_ts_1 = require("../../utility/errors/hdbError.js");
 const rocksdb_js_1 = require("@harperfast/rocksdb-js");
 const logger = (0, logger_ts_1.loggerWithTag)('HNSW');
+// Optional int8 scalar quantization of stored vectors, enabled per-index via the
+// schema directive: `@indexed(type: "HNSW", quantization: "int8")`. The stored
+// graph node holds the vector as a compact int8 `bin` plus a per-vector `scale`,
+// roughly a 5x size reduction over the float32 array and ~10x cheaper to decode
+// (a single typed-array view instead of decoding 768 individually-tagged floats
+// into a boxed Array). The full-precision vector still lives on the record, so
+// only graph navigation is approximate; quantization recall loss is ~1%.
+//
+// Decode auto-detects the stored format (number[] = float, bin = int8), so an
+// int8-enabled index transparently reads legacy float nodes written before the
+// option was set.
+/** Symmetric int8 scalar-quantize a float vector. scale = max|component| / 127. */
+function quantizeInt8(vector) {
+    let max = 0;
+    for (let i = 0; i < vector.length; i++) {
+        const a = vector[i] < 0 ? -vector[i] : vector[i];
+        if (a > max)
+            max = a;
+    }
+    const scale = max / 127 || 1;
+    const inv = 1 / scale;
+    const q = new Int8Array(vector.length);
+    // clamp guards against a float-rounding edge landing on 128 (which Int8Array would wrap to -128)
+    for (let i = 0; i < vector.length; i++)
+        q[i] = Math.max(-127, Math.min(127, Math.round(vector[i] * inv)));
+    return { bytes: Buffer.from(q.buffer, q.byteOffset, q.byteLength), scale };
+}
+/** Reconstruct an approximate float array from an int8 vector + scale. */
+function dequantizeInt8(q, scale) {
+    const out = new Array(q.length);
+    for (let i = 0; i < q.length; i++)
+        out[i] = q[i] * scale;
+    return out;
+}
+// Auto-scaled search ef, used only when an index does not explicitly configure efConstructionSearch
+// and a query does not pass its own ef. A fixed ef makes recall decay as the graph grows (it explores
+// a shrinking fraction of the graph), so ef grows with sqrt(node count), capped to bound search cost.
+// Constants from a recall/latency-vs-N sweep (768-dim cosine, int8): ef≈400 holds ~0.8 recall@10 from
+// 5K–30K, and the recall/latency tradeoff is steep (ef 800 at 30K ≈ 0.92 recall but ~2s p50), so the
+// cap deliberately favors latency — apps wanting higher recall set efConstructionSearch or a per-query
+// ef. Tune as graph build quality / larger-N data improves.
+const AUTO_EF_BASE = 100;
+const AUTO_EF_REF = 1000;
+const AUTO_EF_MAX = 512;
+function autoScaleEf(nodeCount) {
+    const scaled = Math.round(AUTO_EF_BASE * Math.sqrt(Math.max(1, nodeCount / AUTO_EF_REF)));
+    return Math.min(AUTO_EF_MAX, Math.max(AUTO_EF_BASE, scaled));
+}
+class MinHeap {
+    data = [];
+    get size() {
+        return this.data.length;
+    }
+    push(item) {
+        this.data.push(item);
+        let i = this.data.length - 1;
+        while (i > 0) {
+            const p = (i - 1) >> 1;
+            if (this.data[p].distance <= this.data[i].distance)
+                break;
+            const tmp = this.data[p];
+            this.data[p] = this.data[i];
+            this.data[i] = tmp;
+            i = p;
+        }
+    }
+    pop() {
+        if (this.data.length === 0)
+            return undefined;
+        const top = this.data[0];
+        const last = this.data.pop();
+        if (this.data.length > 0) {
+            this.data[0] = last;
+            let i = 0;
+            for (;;) {
+                const l = 2 * i + 1, r = l + 1;
+                let min = i;
+                if (l < this.data.length && this.data[l].distance < this.data[min].distance)
+                    min = l;
+                if (r < this.data.length && this.data[r].distance < this.data[min].distance)
+                    min = r;
+                if (min === i)
+                    break;
+                const tmp = this.data[min];
+                this.data[min] = this.data[i];
+                this.data[i] = tmp;
+                i = min;
+            }
+        }
+        return top;
+    }
+}
+function bisectInsert(arr, distance) {
+    let lo = 0, hi = arr.length;
+    while (lo < hi) {
+        const mid = (lo + hi) >> 1;
+        if (arr[mid].distance <= distance)
+            lo = mid + 1;
+        else
+            hi = mid;
+    }
+    return lo;
+}
 /**
  * Implementation of a vector index for Harper, using hierarchical navigable small world graphs.
  */
@@ -25,6 +128,10 @@ const MAX_LEVEL = 10; // should give good high-level skip list performance up to
  */
 class HierarchicalNavigableSmallWorld {
     static useObjectStore = true;
+    // Index options that only affect search, not the stored graph — changing them must not trigger a
+    // reindex (databases.ts persists the new value but skips rebuilding). efConstructionSearch is the
+    // search-time candidate-list size; the build uses efConstruction/M/distance, which are structural.
+    static searchOnlyOptions = ['efConstructionSearch'];
     indexStore;
     M = 16; // max number of connections per layer
     efConstruction = 100; // size of dynamic candidate list
@@ -36,6 +143,12 @@ class HierarchicalNavigableSmallWorld {
     nodesVisitedCount = 0;
     idIncrementer;
     distance;
+    int8 = false; // store vectors as int8-quantized bins (set via the `quantization` index option)
+    efSearchConfigured = false; // whether the schema set an explicit search ef; if not, search ef auto-scales with N
+    // Caches the Int8Array-converted clone of a frozen (decoded-from-disk) int8 node, keyed by the
+    // frozen node the object store hands back. WeakMap so entries are collected when the store evicts
+    // the frozen node — without it, every cache hit on a frozen node would re-slice and re-clone.
+    convertedNodes = new WeakMap();
     constructor(indexStore, options) {
         this.indexStore = indexStore;
         if (indexStore) {
@@ -43,7 +156,15 @@ class HierarchicalNavigableSmallWorld {
             // (we would actually like to use float16 if it were available)
             this.indexStore.encoder.useFloat32 = msgpackr_1.FLOAT32_OPTIONS.ALWAYS;
         }
-        this.distance = options?.distance === 'euclidean' ? vector_ts_1.euclideanDistance : vector_ts_1.cosineDistance;
+        this.int8 = options?.quantization === 'int8';
+        // Respect an explicitly-configured search ef (or efConstruction, which seeds it); otherwise auto-scale.
+        this.efSearchConfigured = options?.efConstructionSearch !== undefined || options?.efConstruction !== undefined;
+        this.distance =
+            options?.distance === 'euclidean'
+                ? vector_ts_1.euclideanDistance
+                : options?.distance === 'dotProduct'
+                    ? vector_ts_1.dotProductDistance
+                    : vector_ts_1.cosineDistance;
         if (options) {
             // allow all the HNSW parameters to be configured/tuned
             if (options.M !== undefined) {
@@ -97,16 +218,32 @@ class HierarchicalNavigableSmallWorld {
         if (existingVector) {
             // If we are updating an existing entry, we need to update the entry point
             // if the new entry is closer to the entry point than the old one
-            oldNode = { ...this.indexStore.getSync(nodeId, options) };
+            oldNode = { ...this.safeGetSync(nodeId, options) };
         }
         else
             oldNode = {};
         if (vector) {
-            let entryPoint = entryPointId && this.indexStore.getSync(entryPointId, options);
+            // Pre-compute 1/|vector| for cosine distance so searchLayer can skip sqrt per neighbor
+            let invMag;
+            if (this.distance === vector_ts_1.cosineDistance) {
+                let magSq = 0;
+                for (const v of vector)
+                    magSq += v * v;
+                invMag = 1 / (Math.sqrt(magSq) || 1);
+            }
+            // Quantized storage form. The float `vector` is still used as the query for every
+            // searchLayer call below (asymmetric distance: float query x int8 stored); only what
+            // we PUT to the store is quantized.
+            const q = this.int8 ? quantizeInt8(vector) : undefined;
+            const storedVector = q ? q.bytes : vector;
+            const storedScale = q ? q.scale : undefined;
+            let entryPoint = entryPointId && this.safeGetSync(entryPointId, options);
             if (entryPoint == null) {
                 const level = Math.floor(-Math.log(Math.random()) * this.mL);
                 const node = {
-                    vector,
+                    vector: storedVector,
+                    scale: storedScale,
+                    invMag,
                     level,
                     primaryKey,
                 };
@@ -201,8 +338,13 @@ class HierarchicalNavigableSmallWorld {
                     for (const { fromId, toId } of connectionsToBeReplaced) {
                         let from = updateNode(fromId);
                         if (!from)
-                            from = updateNode(fromId, this.indexStore.getSync(fromId, options));
-                        for (let i = 0; i < from[l].length; i++) {
+                            from = updateNode(fromId, this.safeGetSync(fromId, options));
+                        if (!from)
+                            continue;
+                        const fromAtLevel = from[l];
+                        if (!fromAtLevel)
+                            continue;
+                        for (let i = 0; i < fromAtLevel.length; i++) {
                             if (from[l][i].id === toId) {
                                 if (Object.isFrozen(from[l])) {
                                     from[l] = from[l].slice();
@@ -248,7 +390,9 @@ class HierarchicalNavigableSmallWorld {
             }
             // Store the new element
             this.indexStore.put(nodeId, {
-                vector,
+                vector: storedVector,
+                scale: storedScale,
+                invMag,
                 level,
                 primaryKey,
                 ...connections,
@@ -301,7 +445,7 @@ class HierarchicalNavigableSmallWorld {
                 const oldConnections = oldNode[l];
                 for (const { id: neighborId } of oldConnections) {
                     // get and copy the neighbor node so we can modify it
-                    const neighborNode = updateNode(neighborId, this.indexStore.getSync(neighborId, options));
+                    const neighborNode = updateNode(neighborId, this.safeGetSync(neighborId, options));
                     if (!neighborNode)
                         continue;
                     for (let l2 = 0; l2 <= l; l2++) {
@@ -311,7 +455,10 @@ class HierarchicalNavigableSmallWorld {
                         });
                         if (neighborNode[l2]?.length === 0) {
                             logger.trace?.('node was left orphaned, will reindex', neighborId);
-                            needsReindexing.set(neighborNode.primaryKey, neighborNode.vector);
+                            // reindex re-feeds this vector into index() as a float query, so dequantize int8 back to float
+                            needsReindexing.set(neighborNode.primaryKey, neighborNode.scale !== undefined
+                                ? dequantizeInt8(neighborNode.vector, neighborNode.scale)
+                                : neighborNode.vector);
                         }
                     }
                 }
@@ -333,14 +480,50 @@ class HierarchicalNavigableSmallWorld {
         for (const [key, vector] of needsReindexing) {
             this.index(key, vector, vector, options);
         }
-        this.checkSymmetry(nodeId, this.indexStore.getSync(nodeId, options), options);
+        this.checkSymmetry(nodeId, this.safeGetSync(nodeId, options), options);
+    }
+    safeGetSync(key, options) {
+        try {
+            let node = this.indexStore.getSync(key, options);
+            // A quantized vector decodes as a bin (Uint8Array/Buffer) that is a view into the
+            // store's read buffer, which may be reused on the next getSync — so copy the bytes
+            // into a retained Int8Array (raw two's-complement reinterpret). The Int8Array guard
+            // skips re-conversion when the object store (useObjectStore) hands back an
+            // already-converted cached node. Float nodes (vector is a number[]) pass through.
+            if (node && node.vector && !Array.isArray(node.vector) && !(node.vector instanceof Int8Array)) {
+                // A node decoded from disk (a cache miss, common once the table outgrows the object
+                // cache) is frozen — the index store sets freezeData — so assigning node.vector would
+                // throw and the catch below would silently drop the node, fragmenting the graph (#1161).
+                // Clone the frozen node, memoizing the clone against the frozen node so repeated cache
+                // hits skip re-slicing/re-cloning. Mutate in place only the writable just-written object.
+                const cached = this.convertedNodes.get(node);
+                if (cached)
+                    return cached;
+                const u8 = node.vector;
+                const vector = new Int8Array(u8.buffer, u8.byteOffset, u8.byteLength).slice();
+                if (Object.isFrozen(node)) {
+                    const converted = { ...node, vector };
+                    this.convertedNodes.set(node, converted);
+                    node = converted;
+                }
+                else
+                    node.vector = vector;
+            }
+            return node;
+        }
+        catch {
+            logger.warn?.('Failed to decode HNSW node, skipping', key);
+            return undefined;
+        }
     }
     getEntryPoint(options = {}) {
         // Get entry point
         const entryPointId = this.indexStore.getSync(ENTRY_POINT, options);
         if (entryPointId === undefined)
             return;
-        const node = this.indexStore.getSync(entryPointId, options);
+        const node = this.safeGetSync(entryPointId, options);
+        if (!node)
+            return;
         return { id: entryPointId, ...node };
     }
     /**
@@ -357,48 +540,90 @@ class HierarchicalNavigableSmallWorld {
      * @private
      */
     searchLayer(queryVector, entryPointId, entryPoint, ef, level, options = {}, distanceFunction = this.distance) {
+        // Pre-compute query magnitude for cosine; use cached invMag on stored nodes to skip sqrt per neighbor.
+        // Asymmetric distance: the query stays full-precision float; a stored neighbor may be int8
+        // (with per-vector `scaleB`) or float (`scaleB` undefined).
+        let computeDistance;
+        if (distanceFunction === vector_ts_1.cosineDistance) {
+            let magASq = 0;
+            for (const v of queryVector)
+                magASq += v * v;
+            const invMagA = 1 / (Math.sqrt(magASq) || 1);
+            computeDistance = (b, invMagB, scaleB) => {
+                let dot = 0;
+                for (let i = 0; i < b.length; i++)
+                    dot += queryVector[i] * b[i];
+                if (scaleB !== undefined)
+                    dot *= scaleB; // dequantize the int8 dot product
+                if (invMagB !== undefined)
+                    return 1 - dot * invMagA * invMagB;
+                // Fallback when the stored node has no cached invMag (a non-cosine index queried as
+                // cosine). Compute the stored magnitude and dequantize it by scaleB so it matches the
+                // already-dequantized dot product.
+                let magBSq = 0;
+                for (let i = 0; i < b.length; i++)
+                    magBSq += b[i] * b[i];
+                let magB = Math.sqrt(magBSq) || 1;
+                if (scaleB !== undefined)
+                    magB *= scaleB;
+                return 1 - (dot * invMagA) / magB;
+            };
+        }
+        else if (distanceFunction === vector_ts_1.euclideanDistance) {
+            // Asymmetric squared-euclidean, dequantizing each int8 component inline (no allocation).
+            computeDistance = (b, _invMagB, scaleB) => {
+                if (scaleB === undefined)
+                    return distanceFunction(queryVector, b);
+                let distanceSquared = 0;
+                for (let i = 0; i < b.length; i++) {
+                    const diff = queryVector[i] - b[i] * scaleB;
+                    distanceSquared += diff * diff;
+                }
+                return distanceSquared;
+            };
+        }
+        else {
+            // Negated inner product, dequantizing the int8 dot product inline (no allocation).
+            computeDistance = (b, _invMagB, scaleB) => {
+                if (scaleB === undefined)
+                    return distanceFunction(queryVector, b);
+                let dot = 0;
+                for (let i = 0; i < b.length; i++)
+                    dot += queryVector[i] * b[i];
+                return -(dot * scaleB);
+            };
+        }
         const visited = new Set([entryPointId]);
-        const candidates = [
-            {
-                id: entryPointId,
-                distance: distanceFunction(queryVector, entryPoint.vector),
-                node: entryPoint,
-            },
-        ];
-        const results = [...candidates];
-        while (candidates.length > 0) {
-            // Get closest unvisited element
-            candidates.sort((a, b) => a.distance - b.distance);
-            const current = candidates.shift();
-            // Get least result distance
+        const initialCandidate = {
+            id: entryPointId,
+            distance: computeDistance(entryPoint.vector, entryPoint.invMag, entryPoint.scale),
+            node: entryPoint,
+        };
+        const candidates = new MinHeap();
+        candidates.push(initialCandidate);
+        const results = [initialCandidate];
+        while (candidates.size > 0) {
+            const current = candidates.pop();
             const furthestDistance = results[results.length - 1].distance;
-            // If current candidate is less similar than our worst result, we're done
             if (current.distance > furthestDistance)
                 break;
-            // Check neighbors of current point
-            const currentNode = current.node;
-            for (const { id: neighborId } of currentNode[level] || []) {
+            for (const { id: neighborId } of current.node[level] || []) {
                 if (visited.has(neighborId) || neighborId === undefined)
                     continue;
                 visited.add(neighborId);
-                const neighbor = this.indexStore.getSync(neighborId, options);
+                const neighbor = this.safeGetSync(neighborId, options);
                 if (!neighbor)
                     continue;
                 this.nodesVisitedCount++;
-                const distance = distanceFunction(queryVector, neighbor.vector);
+                const distance = computeDistance(neighbor.vector, neighbor.invMag, neighbor.scale);
                 if (distance < furthestDistance || results.length < ef) {
-                    const candidate = {
-                        id: neighborId,
-                        distance,
-                        node: neighbor,
-                    };
+                    const candidate = { id: neighborId, distance, node: neighbor };
                     candidates.push(candidate);
-                    results.push(candidate);
+                    results.splice(bisectInsert(results, distance), 0, candidate);
+                    if (results.length > ef)
+                        results.pop();
                 }
             }
-            results.sort((a, b) => a.distance - b.distance);
-            if (results.length > ef)
-                results.splice(ef, results.length - ef);
         }
         results.visited = visited.size;
         return results;
@@ -414,7 +639,7 @@ class HierarchicalNavigableSmallWorld {
      * @param comparator
      * @param context
      */
-    search({ target, value, descending, distance, comparator, }, context) {
+    search({ target, value, descending, distance, comparator, ef, }, context) {
         let limit = 0; // zero is ignored, only used if set below
         switch (comparator) {
             case 'lt':
@@ -424,24 +649,37 @@ class HierarchicalNavigableSmallWorld {
             case 'sort':
                 break;
             default:
-                throw new hdbError_js_1.ClientError(`Can not use "${comparator}" comparator with HNSW`);
+                throw new hdbError_ts_1.ClientError(`Can not use "${comparator}" comparator with HNSW`);
         }
         if (descending)
-            throw new hdbError_js_1.ClientError(`Can not use descending sort order with HNSW`);
+            throw new hdbError_ts_1.ClientError(`Can not use descending sort order with HNSW`);
         let distanceFunction;
         if (distance === 'cosine')
             distanceFunction = vector_ts_1.cosineDistance;
         else if (distance === 'euclidean')
             distanceFunction = vector_ts_1.euclideanDistance;
+        else if (distance === 'dotProduct')
+            distanceFunction = vector_ts_1.dotProductDistance;
         else if (distance)
-            throw new hdbError_js_1.ClientError('Unknown distance function');
+            throw new hdbError_ts_1.ClientError('Unknown distance function');
         else
             distanceFunction = this.distance;
         if (!target)
-            throw new hdbError_js_1.ClientError('A target vector must be provided for an HNSW query');
+            throw new hdbError_ts_1.ClientError('A target vector must be provided for an HNSW query');
         if (!Array.isArray(target))
-            throw new hdbError_js_1.ClientError('The target vector must be an array');
+            throw new hdbError_ts_1.ClientError('The target vector must be an array');
         const options = context.transaction; // should have a nested RocksDB transaction
+        // Resolve search ef: per-query ef wins; else an explicitly-configured efConstructionSearch;
+        // else auto-scale with the graph size so recall holds as the table grows.
+        let effectiveEf = this.efConstructionSearch;
+        if (ef !== undefined && ef > 0)
+            effectiveEf = ef;
+        else if (!this.efSearchConfigured) {
+            const nodeCount = this.indexStore.getKeysCount
+                ? this.indexStore.getKeysCount()
+                : (this.indexStore.getStats?.()?.entryCount ?? 0);
+            effectiveEf = autoScaleEf(nodeCount);
+        }
         let entryPoint = this.getEntryPoint(options);
         if (!entryPoint)
             return [];
@@ -450,7 +688,7 @@ class HierarchicalNavigableSmallWorld {
         // For each level from top to bottom
         for (let l = entryPoint.level; l >= 0; l--) {
             // Search for closest neighbors at current level
-            results = this.searchLayer(target, entryPointId, entryPoint, this.efConstructionSearch, l, options, distanceFunction);
+            results = this.searchLayer(target, entryPointId, entryPoint, effectiveEf, l, options, distanceFunction);
             if (results.length > 0) {
                 const neighbor = results[0]; // closest neighbor becomes new entry point
                 entryPoint = neighbor.node;
@@ -465,6 +703,27 @@ class HierarchicalNavigableSmallWorld {
             distance: candidate.distance,
         }));
     }
+    /**
+     * Exact distance between a query and a record's FULL-precision vector, mirroring search()'s metric
+     * selection. Used to rerank quantized (int8) results: graph traversal navigates on approximate
+     * (quantized) distances, but the caller has the exact record vector and can restore exact ordering
+     * and $distance.
+     */
+    exactDistance(searchCondition, recordVector) {
+        if (recordVector == null)
+            return Infinity; // missing vector sorts last
+        // distance fns require a plain Array (they guard on Array.isArray); records normally store a
+        // float[] vector, but convert defensively in case a typed array slips through.
+        const vec = Array.isArray(recordVector) ? recordVector : Array.from(recordVector);
+        const fn = searchCondition.distance === 'euclidean'
+            ? vector_ts_1.euclideanDistance
+            : searchCondition.distance === 'dotProduct'
+                ? vector_ts_1.dotProductDistance
+                : searchCondition.distance === 'cosine'
+                    ? vector_ts_1.cosineDistance
+                    : this.distance;
+        return fn(searchCondition.target, vec);
+    }
     checkSymmetry(id, node, options) {
         if (!node)
             return;
@@ -475,9 +734,9 @@ class HierarchicalNavigableSmallWorld {
             if (connections.length === 0)
                 break;
             for (const { id: neighbor } of connections) {
-                const neighborNode = this.indexStore.getSync(neighbor, options);
+                const neighborNode = this.safeGetSync(neighbor, options);
                 if (!neighborNode) {
-                    logger.info?.('could not find neighbor node', neighborNode);
+                    logger.info?.('could not find neighbor node', neighbor);
                     continue;
                 }
                 // verify that the connection is symmetrical
@@ -512,7 +771,7 @@ class HierarchicalNavigableSmallWorld {
             node[level] = keptConnections;
             // For removed connections, ensure there's still a path to them
             for (const removed of removedConnections) {
-                let removedNode = updateNode(removed.id) ?? this.indexStore.getSync(removed.id, options);
+                let removedNode = updateNode(removed.id) ?? this.safeGetSync(removed.id, options);
                 if (removedNode) {
                     // Remove the reverse connection if it exists
                     if (removedNode[level]) {
@@ -540,19 +799,26 @@ class HierarchicalNavigableSmallWorld {
     }
     validateConnectivity(startLevel = 0) {
         const entryPoint = this.getEntryPoint();
+        if (!entryPoint)
+            return;
         const visited = new Set();
-        // BFS from entry point to ensure all nodes are reachable
+        // BFS from entry point to ensure all nodes are reachable. Asymmetric stale neighbor
+        // references can survive deletes, so a referenced node may not actually exist anymore;
+        // only count a node as visited once we confirm the underlying record is present.
         const queue = [entryPoint.id];
-        visited.add(entryPoint.id);
+        const enqueued = new Set([entryPoint.id]);
         let connections = 0;
         while (queue.length > 0) {
             const currentId = queue.shift();
-            const current = this.indexStore.getSync(currentId);
+            const current = this.safeGetSync(currentId);
+            if (!current)
+                continue;
+            visited.add(currentId);
             for (let level = startLevel; level <= current.level; level++) {
                 for (const { id: neighborId } of current[level] || []) {
                     connections++;
-                    if (!visited.has(neighborId)) {
-                        visited.add(neighborId);
+                    if (!enqueued.has(neighborId)) {
+                        enqueued.add(neighborId);
                         queue.push(neighborId);
                     }
                 }
@@ -560,9 +826,6 @@ class HierarchicalNavigableSmallWorld {
         }
         // Check if all nodes are reachable
         // This would require maintaining a separate set/count of all nodes
-        if (visited.size !== this.totalNodes) {
-            console.log('visited', visited.size, 'total', this.totalNodes);
-        }
         return {
             isFullyConnected: visited.size === this.totalNodes,
             averageConnections: connections / visited.size,
@@ -607,7 +870,12 @@ class HierarchicalNavigableSmallWorld {
                 vectorDistances = context.vectorDistances = sortDefinition.vectorDistances = new Map();
             let distanceFunction = this.distance;
             if (sortDefinition.type)
-                distanceFunction = sortDefinition.distance === 'euclidean' ? vector_ts_1.euclideanDistance : vector_ts_1.cosineDistance;
+                distanceFunction =
+                    sortDefinition.distance === 'euclidean'
+                        ? vector_ts_1.euclideanDistance
+                        : sortDefinition.distance === 'dotProduct'
+                            ? vector_ts_1.dotProductDistance
+                            : vector_ts_1.cosineDistance;
             const distance = distanceFunction(sortDefinition.target, vector);
             vectorDistances.set(entry, distance);
             return distance;