family-ai-agent 1.0.6 → 1.0.8

package/src/database/vector/hnsw-index.ts

@@ -0,0 +1,452 @@
+/**
+ * HNSW (Hierarchical Navigable Small World) Index
+ *
+ * In-memory approximate nearest neighbor search for SQLite mode.
+ * Provides fast vector similarity search without pgvector.
+ */
+
+import { cosineSimilarity, euclideanDistance } from './similarity.js';
+import { createLogger } from '../../utils/logger.js';
+
+const logger = createLogger('HNSWIndex');
+
+/**
+ * HNSW node representing a vector in the graph
+ */
+interface HNSWNode {
+  id: string;
+  vector: number[];
+  neighbors: Map<number, Set<string>>; // level -> neighbor IDs
+  level: number;
+}
+
+/**
+ * Search result with similarity score
+ */
+export interface SearchResult {
+  id: string;
+  similarity: number;
+}
+
+/**
+ * HNSW index configuration
+ */
+export interface HNSWConfig {
+  /** Vector dimension */
+  dimension: number;
+  /** Similarity metric: 'cosine' or 'euclidean' */
+  similarity: 'cosine' | 'euclidean';
+  /** Maximum connections per layer */
+  M: number;
+  /** Maximum connections for layer 0 */
+  M0: number;
+  /** Size of dynamic candidate list during construction */
+  efConstruction: number;
+  /** Level generation factor (1/ln(M)) */
+  mL: number;
+}
+
+/**
+ * Default HNSW configuration
+ */
+const DEFAULT_CONFIG: HNSWConfig = {
+  dimension: 1536,
+  similarity: 'cosine',
+  M: 16,
+  M0: 32,
+  efConstruction: 200,
+  mL: 1 / Math.log(16),
+};
+
+/**
+ * HNSW Index for fast approximate nearest neighbor search
+ */
+export class HNSWIndex {
+  private nodes: Map<string, HNSWNode> = new Map();
+  private entryPoint: string | null = null;
+  private maxLevel: number = 0;
+  private config: HNSWConfig;
+
+  constructor(config: Partial<HNSWConfig> = {}) {
+    this.config = { ...DEFAULT_CONFIG, ...config };
+  }
+
+  /**
+   * Get similarity function based on config
+   */
+  private getSimilarity(): (a: number[], b: number[]) => number {
+    if (this.config.similarity === 'euclidean') {
+      // Convert distance to similarity (higher is better)
+      return (a, b) => 1 / (1 + euclideanDistance(a, b));
+    }
+    return cosineSimilarity;
+  }
+
+  /**
+   * Generate random level for new node
+   */
+  private randomLevel(): number {
+    let level = 0;
+    while (Math.random() < this.config.mL && level < 32) {
+      level++;
+    }
+    return level;
+  }
+
+  /**
+   * Get maximum connections for a level
+   */
+  private getMaxConnections(level: number): number {
+    return level === 0 ? this.config.M0 : this.config.M;
+  }
+
+  /**
+   * Add a vector to the index
+   */
+  add(id: string, vector: number[]): void {
+    if (vector.length !== this.config.dimension) {
+      throw new Error(`Expected dimension ${this.config.dimension}, got ${vector.length}`);
+    }
+
+    // Check if already exists
+    if (this.nodes.has(id)) {
+      // Update existing node
+      const existingNode = this.nodes.get(id)!;
+      existingNode.vector = vector;
+      return;
+    }
+
+    const level = this.randomLevel();
+    const node: HNSWNode = {
+      id,
+      vector,
+      neighbors: new Map(),
+      level,
+    };
+
+    // Initialize neighbor sets for each level
+    for (let l = 0; l <= level; l++) {
+      node.neighbors.set(l, new Set());
+    }
+
+    this.nodes.set(id, node);
+
+    // First node becomes entry point
+    if (!this.entryPoint) {
+      this.entryPoint = id;
+      this.maxLevel = level;
+      return;
+    }
+
+    const similarity = this.getSimilarity();
+    let currentNode = this.entryPoint;
+
+    // Traverse from top level down to level + 1
+    for (let l = this.maxLevel; l > level; l--) {
+      currentNode = this.searchLayer(vector, currentNode, 1, l, similarity)[0]?.id ?? currentNode;
+    }
+
+    // Insert at each level from level down to 0
+    for (let l = Math.min(level, this.maxLevel); l >= 0; l--) {
+      const candidates = this.searchLayer(
+        vector,
+        currentNode,
+        this.config.efConstruction,
+        l,
+        similarity
+      );
+
+      const neighbors = this.selectNeighbors(candidates, this.getMaxConnections(l));
+
+      // Connect new node to neighbors
+      for (const neighbor of neighbors) {
+        node.neighbors.get(l)!.add(neighbor.id);
+
+        // Connect neighbors back to new node (bidirectional)
+        const neighborNode = this.nodes.get(neighbor.id);
+        if (neighborNode) {
+          neighborNode.neighbors.get(l)?.add(id);
+
+          // Prune if too many connections
+          const maxConn = this.getMaxConnections(l);
+          if ((neighborNode.neighbors.get(l)?.size ?? 0) > maxConn) {
+            this.pruneConnections(neighborNode, l, maxConn, similarity);
+          }
+        }
+      }
+
+      if (candidates.length > 0) {
+        currentNode = candidates[0]!.id;
+      }
+    }
+
+    // Update entry point if new node has higher level
+    if (level > this.maxLevel) {
+      this.entryPoint = id;
+      this.maxLevel = level;
+    }
+  }
+
+  /**
+   * Search for k nearest neighbors
+   */
+  search(query: number[], k: number, ef?: number): SearchResult[] {
+    if (query.length !== this.config.dimension) {
+      throw new Error(`Expected dimension ${this.config.dimension}, got ${query.length}`);
+    }
+
+    if (!this.entryPoint) {
+      return [];
+    }
+
+    const effectiveEf = ef ?? Math.max(k, 100);
+    const similarity = this.getSimilarity();
+    let currentNode = this.entryPoint;
+
+    // Traverse from top level down to level 1
+    for (let l = this.maxLevel; l > 0; l--) {
+      currentNode = this.searchLayer(query, currentNode, 1, l, similarity)[0]?.id ?? currentNode;
+    }
+
+    // Search at level 0 with ef candidates
+    const candidates = this.searchLayer(query, currentNode, effectiveEf, 0, similarity);
+
+    // Return top k
+    return candidates.slice(0, k);
+  }
+
+  /**
+   * Search within a layer
+   */
+  private searchLayer(
+    query: number[],
+    entryId: string,
+    ef: number,
+    level: number,
+    similarity: (a: number[], b: number[]) => number
+  ): SearchResult[] {
+    const entryNode = this.nodes.get(entryId);
+    if (!entryNode) return [];
+
+    const visited = new Set<string>([entryId]);
+    const candidates: SearchResult[] = [
+      { id: entryId, similarity: similarity(query, entryNode.vector) },
+    ];
+
+    // Priority queue (sorted by similarity descending)
+    const toExplore: SearchResult[] = [...candidates];
+
+    while (toExplore.length > 0) {
+      // Get best candidate
+      toExplore.sort((a, b) => b.similarity - a.similarity);
+      const current = toExplore.shift()!;
+
+      // Get worst in results
+      candidates.sort((a, b) => b.similarity - a.similarity);
+      const worstSimilarity = candidates.length >= ef
+        ? candidates[candidates.length - 1]!.similarity
+        : -Infinity;
+
+      // Stop if current is worse than worst result
+      if (current.similarity < worstSimilarity) {
+        break;
+      }
+
+      // Explore neighbors
+      const currentNode = this.nodes.get(current.id);
+      const neighbors = currentNode?.neighbors.get(level) ?? new Set();
+
+      for (const neighborId of neighbors) {
+        if (visited.has(neighborId)) continue;
+        visited.add(neighborId);
+
+        const neighborNode = this.nodes.get(neighborId);
+        if (!neighborNode) continue;
+
+        const sim = similarity(query, neighborNode.vector);
+        const result = { id: neighborId, similarity: sim };
+
+        if (candidates.length < ef || sim > worstSimilarity) {
+          candidates.push(result);
+          candidates.sort((a, b) => b.similarity - a.similarity);
+          if (candidates.length > ef) {
+            candidates.pop();
+          }
+          toExplore.push(result);
+        }
+      }
+    }
+
+    return candidates;
+  }
+
+  /**
+   * Select best neighbors from candidates
+   */
+  private selectNeighbors(candidates: SearchResult[], maxNeighbors: number): SearchResult[] {
+    // Simple approach: take top maxNeighbors by similarity
+    return candidates
+      .sort((a, b) => b.similarity - a.similarity)
+      .slice(0, maxNeighbors);
+  }
+
+  /**
+   * Prune connections to maintain max connections
+   */
+  private pruneConnections(
+    node: HNSWNode,
+    level: number,
+    maxConnections: number,
+    similarity: (a: number[], b: number[]) => number
+  ): void {
+    const neighbors = node.neighbors.get(level);
+    if (!neighbors || neighbors.size <= maxConnections) return;
+
+    // Calculate similarities and keep best
+    const scored: SearchResult[] = [];
+    for (const neighborId of neighbors) {
+      const neighborNode = this.nodes.get(neighborId);
+      if (neighborNode) {
+        scored.push({
+          id: neighborId,
+          similarity: similarity(node.vector, neighborNode.vector),
+        });
+      }
+    }
+
+    scored.sort((a, b) => b.similarity - a.similarity);
+    const keep = new Set(scored.slice(0, maxConnections).map((s) => s.id));
+
+    // Remove excess connections
+    for (const neighborId of neighbors) {
+      if (!keep.has(neighborId)) {
+        neighbors.delete(neighborId);
+        // Also remove back-link
+        this.nodes.get(neighborId)?.neighbors.get(level)?.delete(node.id);
+      }
+    }
+  }
+
+  /**
+   * Remove a vector from the index
+   */
+  remove(id: string): boolean {
+    const node = this.nodes.get(id);
+    if (!node) return false;
+
+    // Remove from all neighbors
+    for (const [level, neighbors] of node.neighbors) {
+      for (const neighborId of neighbors) {
+        this.nodes.get(neighborId)?.neighbors.get(level)?.delete(id);
+      }
+    }
+
+    this.nodes.delete(id);
+
+    // Update entry point if needed
+    if (this.entryPoint === id) {
+      if (this.nodes.size === 0) {
+        this.entryPoint = null;
+        this.maxLevel = 0;
+      } else {
+        // Find new entry point with highest level
+        let maxLvl = 0;
+        let newEntry: string | null = null;
+        for (const [nodeId, n] of this.nodes) {
+          if (n.level >= maxLvl) {
+            maxLvl = n.level;
+            newEntry = nodeId;
+          }
+        }
+        this.entryPoint = newEntry;
+        this.maxLevel = maxLvl;
+      }
+    }
+
+    return true;
+  }
+
+  /**
+   * Check if an ID exists in the index
+   */
+  has(id: string): boolean {
+    return this.nodes.has(id);
+  }
+
+  /**
+   * Get vector by ID
+   */
+  get(id: string): number[] | undefined {
+    return this.nodes.get(id)?.vector;
+  }
+
+  /**
+   * Get index size
+   */
+  get size(): number {
+    return this.nodes.size;
+  }
+
+  /**
+   * Get index statistics
+   */
+  getStats(): {
+    size: number;
+    dimension: number;
+    maxLevel: number;
+    avgConnections: number;
+  } {
+    let totalConnections = 0;
+    let connectionCount = 0;
+
+    for (const node of this.nodes.values()) {
+      for (const neighbors of node.neighbors.values()) {
+        totalConnections += neighbors.size;
+        connectionCount++;
+      }
+    }
+
+    return {
+      size: this.nodes.size,
+      dimension: this.config.dimension,
+      maxLevel: this.maxLevel,
+      avgConnections: connectionCount > 0 ? totalConnections / connectionCount : 0,
+    };
+  }
+
+  /**
+   * Clear the index
+   */
+  clear(): void {
+    this.nodes.clear();
+    this.entryPoint = null;
+    this.maxLevel = 0;
+  }
+
+  /**
+   * Get all IDs in the index
+   */
+  getAllIds(): string[] {
+    return Array.from(this.nodes.keys());
+  }
+
+  /**
+   * Rebuild index from scratch (for optimization)
+   */
+  rebuild(): void {
+    const vectors = Array.from(this.nodes.entries()).map(([id, node]) => ({
+      id,
+      vector: node.vector,
+    }));
+
+    this.clear();
+
+    for (const { id, vector } of vectors) {
+      this.add(id, vector);
+    }
+
+    logger.debug('Index rebuilt', { size: this.size });
+  }
+}
+
+export default HNSWIndex;