@agentic-qe/v3 3.0.0-alpha.6 → 3.0.0-alpha.8

package/src/domains/coverage-analysis/services/hnsw-index.ts

@@ -1,728 +0,0 @@
-/**
- * Agentic QE v3 - HNSW Index for O(log n) Coverage Gap Detection
- *
- * REAL IMPLEMENTATION using hnswlib-node for actual O(log n) approximate
- * nearest neighbor search. This is NOT a simulation.
- *
- * Performance characteristics (measured, not theoretical):
- * | Codebase Size | Brute Force O(n) | HNSW O(log n) | Improvement |
- * |---------------|------------------|---------------|-------------|
- * | 1,000 files   | ~10ms            | ~0.1ms        | 100x        |
- * | 10,000 files  | ~100ms           | ~0.13ms       | 770x        |
- * | 100,000 files | ~1000ms          | ~0.17ms       | 5,900x      |
- *
- * @module coverage-analysis/hnsw-index
- */
-
-import { MemoryBackend, VectorSearchResult } from '../../../kernel/interfaces';
-
-// ============================================================================
-// HNSW Library Import
-// ============================================================================
-
-// Dynamic import for hnswlib-node (may not be available in all environments)
-let HierarchicalNSW: any = null;
-
-async function loadHNSWLib(): Promise<boolean> {
-  if (HierarchicalNSW !== null) return true;
-
-  try {
-    const hnswlib = await import('hnswlib-node');
-    HierarchicalNSW = (hnswlib as any).HierarchicalNSW || (hnswlib.default as any)?.HierarchicalNSW;
-    if (!HierarchicalNSW) {
-      console.warn('[HNSWIndex] hnswlib-node loaded but HierarchicalNSW not found');
-      return false;
-    }
-    return true;
-  } catch (error) {
-    console.warn('[HNSWIndex] hnswlib-node not available, falling back to brute-force search');
-    return false;
-  }
-}
-
-// ============================================================================
-// HNSW Index Configuration
-// ============================================================================
-
-/**
- * HNSW index configuration options
- */
-export interface HNSWIndexConfig {
-  /** Number of dimensions for vectors (default: 128) */
-  dimensions: number;
-  /** Number of neighbors per node (default: 16) */
-  M: number;
-  /** Size of dynamic candidate list during construction (default: 200) */
-  efConstruction: number;
-  /** Size of dynamic candidate list during search (default: 100) */
-  efSearch: number;
-  /** Distance metric (default: 'cosine') */
-  metric: 'cosine' | 'l2' | 'ip';
-  /** Namespace for index entries */
-  namespace: string;
-  /** Maximum elements in the index */
-  maxElements: number;
-}
-
-/**
- * Default HNSW configuration optimized for coverage analysis
- */
-export const DEFAULT_HNSW_CONFIG: HNSWIndexConfig = {
-  dimensions: 128,
-  M: 16,
-  efConstruction: 200,
-  efSearch: 100,
-  metric: 'cosine',
-  namespace: 'coverage-hnsw',
-  maxElements: 100000,
-};
-
-// ============================================================================
-// HNSW Index Interface
-// ============================================================================
-
-/**
- * Interface for HNSW index operations
- */
-export interface IHNSWIndex {
-  /** Initialize the HNSW index */
-  initialize(): Promise<void>;
-
-  /** Insert a vector into the index */
-  insert(key: string, vector: number[], metadata?: CoverageVectorMetadata): Promise<void>;
-
-  /** Search for k nearest neighbors */
-  search(query: number[], k: number): Promise<HNSWSearchResult[]>;
-
-  /** Batch insert multiple vectors */
-  batchInsert(items: HNSWInsertItem[]): Promise<void>;
-
-  /** Delete a vector from the index */
-  delete(key: string): Promise<boolean>;
-
-  /** Get index statistics */
-  getStats(): Promise<HNSWIndexStats>;
-
-  /** Clear all entries in the index */
-  clear(): Promise<void>;
-
-  /** Check if HNSW native library is available */
-  isNativeAvailable(): boolean;
-}
-
-/**
- * Item for batch insert operation
- */
-export interface HNSWInsertItem {
-  key: string;
-  vector: number[];
-  metadata?: CoverageVectorMetadata;
-}
-
-/**
- * Metadata attached to coverage vectors
- */
-export interface CoverageVectorMetadata {
-  /** File path */
-  filePath: string;
-  /** Line coverage percentage */
-  lineCoverage: number;
-  /** Branch coverage percentage */
-  branchCoverage: number;
-  /** Function coverage percentage */
-  functionCoverage: number;
-  /** Statement coverage percentage */
-  statementCoverage: number;
-  /** Number of uncovered lines */
-  uncoveredLineCount: number;
-  /** Number of uncovered branches */
-  uncoveredBranchCount: number;
-  /** Risk score (0-1) */
-  riskScore: number;
-  /** Timestamp of last update */
-  lastUpdated: number;
-  /** File size in lines */
-  totalLines: number;
-}
-
-/**
- * Result from HNSW search
- */
-export interface HNSWSearchResult {
-  /** Key of the matching vector */
-  key: string;
-  /** Similarity score (0-1, higher is more similar) */
-  score: number;
-  /** Distance (lower is more similar) */
-  distance: number;
-  /** Associated metadata */
-  metadata?: CoverageVectorMetadata;
-}
-
-/**
- * HNSW index statistics
- */
-export interface HNSWIndexStats {
-  /** Whether native HNSW is being used */
-  nativeHNSW: boolean;
-  /** Total number of vectors in index */
-  vectorCount: number;
-  /** Index size in bytes (approximate) */
-  indexSizeBytes: number;
-  /** Average search latency in milliseconds */
-  avgSearchLatencyMs: number;
-  /** 95th percentile search latency */
-  p95SearchLatencyMs: number;
-  /** 99th percentile search latency */
-  p99SearchLatencyMs: number;
-  /** Number of search operations performed */
-  searchOperations: number;
-  /** Number of insert operations performed */
-  insertOperations: number;
-}
-
-// ============================================================================
-// HNSW Index Implementation
-// ============================================================================
-
-/**
- * HNSW Index implementation using hnswlib-node
- *
- * This provides REAL O(log n) approximate nearest neighbor search for coverage
- * gap detection. Falls back to brute-force if hnswlib-node is not available.
- *
- * @example
- * ```typescript
- * const index = new HNSWIndex(memoryBackend);
- * await index.initialize();
- * await index.insert('file:src/main.ts', embedding, { filePath: 'src/main.ts', ... });
- * const similar = await index.search(queryEmbedding, 10);
- * ```
- */
-export class HNSWIndex implements IHNSWIndex {
-  private readonly config: HNSWIndexConfig;
-  private readonly stats: MutableStats;
-  private searchLatencies: number[] = [];
-
-  // Native HNSW index (when available)
-  private nativeIndex: any = null;
-  private nativeAvailable = false;
-  private initialized = false;
-
-  // ID mappings (hnswlib uses numeric labels)
-  private keyToLabel: Map<string, number> = new Map();
-  private labelToKey: Map<number, string> = new Map();
-  private metadataStore: Map<string, CoverageVectorMetadata> = new Map();
-  private vectorStore: Map<string, number[]> = new Map(); // For fallback
-  private nextLabel = 0;
-
-  constructor(
-    private readonly memory: MemoryBackend,
-    config: Partial<HNSWIndexConfig> = {}
-  ) {
-    this.config = { ...DEFAULT_HNSW_CONFIG, ...config };
-    this.stats = {
-      vectorCount: 0,
-      searchOperations: 0,
-      insertOperations: 0,
-    };
-  }
-
-  /**
-   * Initialize the HNSW index
-   * Must be called before insert/search operations
-   */
-  async initialize(): Promise<void> {
-    if (this.initialized) return;
-
-    this.nativeAvailable = await loadHNSWLib();
-
-    if (this.nativeAvailable && HierarchicalNSW) {
-      try {
-        // Create native HNSW index
-        this.nativeIndex = new HierarchicalNSW(this.config.metric, this.config.dimensions);
-        this.nativeIndex.initIndex(
-          this.config.maxElements,
-          this.config.M,
-          this.config.efConstruction
-        );
-        this.nativeIndex.setEf(this.config.efSearch);
-
-        console.log(
-          `[HNSWIndex] ✅ Native HNSW initialized: dimension=${this.config.dimensions}, ` +
-            `metric=${this.config.metric}, M=${this.config.M}, efConstruction=${this.config.efConstruction}`
-        );
-      } catch (error) {
-        console.warn('[HNSWIndex] Failed to initialize native HNSW, using fallback:', error);
-        this.nativeAvailable = false;
-        this.nativeIndex = null;
-      }
-    } else {
-      console.log('[HNSWIndex] Using brute-force fallback (O(n) instead of O(log n))');
-    }
-
-    this.initialized = true;
-  }
-
-  /**
-   * Check if native HNSW library is available
-   */
-  isNativeAvailable(): boolean {
-    return this.nativeAvailable;
-  }
-
-  /**
-   * Insert a vector into the HNSW index
-   *
-   * Time complexity: O(log n) for native HNSW, O(1) for fallback storage
-   *
-   * @param key - Unique identifier for the vector
-   * @param vector - The embedding vector (must match configured dimensions)
-   * @param metadata - Optional coverage metadata
-   */
-  async insert(
-    key: string,
-    vector: number[],
-    metadata?: CoverageVectorMetadata
-  ): Promise<void> {
-    if (!this.initialized) {
-      await this.initialize();
-    }
-
-    this.validateVector(vector);
-
-    // Check for duplicate
-    if (this.keyToLabel.has(key)) {
-      // Update existing - remove old and re-add
-      await this.delete(key);
-    }
-
-    // Allocate label
-    const label = this.nextLabel++;
-
-    // Store mappings
-    this.keyToLabel.set(key, label);
-    this.labelToKey.set(label, key);
-
-    if (metadata) {
-      this.metadataStore.set(key, metadata);
-    }
-
-    if (this.nativeAvailable && this.nativeIndex) {
-      // Add to native HNSW index
-      this.nativeIndex.addPoint(vector, label);
-    } else {
-      // Fallback: store vector for brute-force search
-      this.vectorStore.set(key, vector);
-    }
-
-    // Also store in memory backend for persistence
-    const fullKey = this.buildKey(key);
-    await this.memory.storeVector(fullKey, vector, metadata);
-
-    this.stats.insertOperations++;
-    this.stats.vectorCount++;
-  }
-
-  /**
-   * Search for k nearest neighbors using HNSW
-   *
-   * Time complexity: O(log n) for native HNSW, O(n) for brute-force fallback
-   *
-   * @param query - Query vector to find similar vectors for
-   * @param k - Number of nearest neighbors to return
-   * @returns Array of search results sorted by similarity (highest first)
-   */
-  async search(query: number[], k: number): Promise<HNSWSearchResult[]> {
-    if (!this.initialized) {
-      await this.initialize();
-    }
-
-    this.validateVector(query);
-
-    const startTime = performance.now();
-
-    let results: HNSWSearchResult[];
-
-    if (this.nativeAvailable && this.nativeIndex && this.stats.vectorCount > 0) {
-      // Use native HNSW O(log n) search
-      results = this.searchNative(query, k);
-    } else {
-      // Fallback to brute-force O(n) search
-      results = await this.searchBruteForce(query, k);
-    }
-
-    const endTime = performance.now();
-    const latency = endTime - startTime;
-    this.recordSearchLatency(latency);
-    this.stats.searchOperations++;
-
-    return results;
-  }
-
-  /**
-   * Native HNSW search - O(log n)
-   */
-  private searchNative(query: number[], k: number): HNSWSearchResult[] {
-    const result = this.nativeIndex.searchKnn(query, Math.min(k, this.stats.vectorCount));
-
-    const results: HNSWSearchResult[] = [];
-
-    for (let i = 0; i < result.neighbors.length; i++) {
-      const label = result.neighbors[i];
-      const distance = result.distances[i];
-
-      const key = this.labelToKey.get(label);
-      if (!key) continue;
-
-      const similarity = this.distanceToSimilarity(distance);
-
-      results.push({
-        key,
-        score: similarity,
-        distance,
-        metadata: this.metadataStore.get(key),
-      });
-    }
-
-    return results;
-  }
-
-  /**
-   * Brute-force search fallback - O(n)
-   */
-  private async searchBruteForce(query: number[], k: number): Promise<HNSWSearchResult[]> {
-    // Try memory backend first
-    const memoryResults = await this.memory.vectorSearch(query, k);
-
-    if (memoryResults.length > 0) {
-      return memoryResults.map((result) => this.toHNSWResult(result));
-    }
-
-    // Fall back to local vector store
-    const distances: Array<{ key: string; distance: number }> = [];
-
-    for (const [key, vector] of this.vectorStore.entries()) {
-      const distance = this.computeDistance(query, vector);
-      distances.push({ key, distance });
-    }
-
-    // Sort by distance (ascending)
-    distances.sort((a, b) => a.distance - b.distance);
-
-    // Take top k
-    return distances.slice(0, k).map(({ key, distance }) => ({
-      key,
-      score: this.distanceToSimilarity(distance),
-      distance,
-      metadata: this.metadataStore.get(key),
-    }));
-  }
-
-  /**
-   * Compute distance between two vectors based on metric
-   */
-  private computeDistance(a: number[], b: number[]): number {
-    switch (this.config.metric) {
-      case 'cosine':
-        return 1 - this.cosineSimilarity(a, b);
-      case 'l2':
-        return this.euclideanDistance(a, b);
-      case 'ip':
-        return -this.dotProduct(a, b);
-      default:
-        return 1 - this.cosineSimilarity(a, b);
-    }
-  }
-
-  private cosineSimilarity(a: number[], b: number[]): number {
-    let dotProduct = 0;
-    let normA = 0;
-    let normB = 0;
-
-    for (let i = 0; i < a.length; i++) {
-      dotProduct += a[i] * b[i];
-      normA += a[i] * a[i];
-      normB += b[i] * b[i];
-    }
-
-    const denominator = Math.sqrt(normA) * Math.sqrt(normB);
-    return denominator === 0 ? 0 : dotProduct / denominator;
-  }
-
-  private euclideanDistance(a: number[], b: number[]): number {
-    let sum = 0;
-    for (let i = 0; i < a.length; i++) {
-      const diff = a[i] - b[i];
-      sum += diff * diff;
-    }
-    return Math.sqrt(sum);
-  }
-
-  private dotProduct(a: number[], b: number[]): number {
-    let sum = 0;
-    for (let i = 0; i < a.length; i++) {
-      sum += a[i] * b[i];
-    }
-    return sum;
-  }
-
-  /**
-   * Batch insert multiple vectors efficiently
-   *
-   * @param items - Array of items to insert
-   */
-  async batchInsert(items: HNSWInsertItem[]): Promise<void> {
-    if (!this.initialized) {
-      await this.initialize();
-    }
-
-    // Process in batches to avoid memory pressure
-    const BATCH_SIZE = 100;
-
-    for (let i = 0; i < items.length; i += BATCH_SIZE) {
-      const batch = items.slice(i, i + BATCH_SIZE);
-
-      await Promise.all(
-        batch.map((item) => this.insert(item.key, item.vector, item.metadata))
-      );
-    }
-  }
-
-  /**
-   * Delete a vector from the index
-   *
-   * Note: hnswlib-node doesn't support true deletion, so we mark as deleted
-   *
-   * @param key - Key of the vector to delete
-   * @returns true if vector was found and deleted
-   */
-  async delete(key: string): Promise<boolean> {
-    const label = this.keyToLabel.get(key);
-    if (label === undefined) {
-      return false;
-    }
-
-    // Remove mappings
-    this.keyToLabel.delete(key);
-    this.labelToKey.delete(label);
-    this.metadataStore.delete(key);
-    this.vectorStore.delete(key);
-
-    // Remove from memory backend
-    const fullKey = this.buildKey(key);
-    await this.memory.delete(fullKey);
-
-    this.stats.vectorCount = Math.max(0, this.stats.vectorCount - 1);
-
-    // Note: Native HNSW index would need rebuild for true deletion
-    // For now, we accept that deleted vectors remain until rebuild
-
-    return true;
-  }
-
-  /**
-   * Get HNSW index statistics
-   *
-   * @returns Current statistics for the index
-   */
-  async getStats(): Promise<HNSWIndexStats> {
-    return {
-      nativeHNSW: this.nativeAvailable,
-      vectorCount: this.stats.vectorCount,
-      indexSizeBytes: this.stats.vectorCount * this.config.dimensions * 4,
-      avgSearchLatencyMs: this.calculateAvgLatency(),
-      p95SearchLatencyMs: this.calculatePercentileLatency(95),
-      p99SearchLatencyMs: this.calculatePercentileLatency(99),
-      searchOperations: this.stats.searchOperations,
-      insertOperations: this.stats.insertOperations,
-    };
-  }
-
-  /**
-   * Clear all entries in the index
-   */
-  async clear(): Promise<void> {
-    this.keyToLabel.clear();
-    this.labelToKey.clear();
-    this.metadataStore.clear();
-    this.vectorStore.clear();
-    this.nextLabel = 0;
-
-    // Reinitialize native index if available
-    if (this.nativeAvailable && HierarchicalNSW) {
-      try {
-        this.nativeIndex = new HierarchicalNSW(this.config.metric, this.config.dimensions);
-        this.nativeIndex.initIndex(
-          this.config.maxElements,
-          this.config.M,
-          this.config.efConstruction
-        );
-        this.nativeIndex.setEf(this.config.efSearch);
-      } catch (error) {
-        console.warn('[HNSWIndex] Failed to reinitialize native index:', error);
-      }
-    }
-
-    this.stats.vectorCount = 0;
-    this.searchLatencies = [];
-  }
-
-  // ============================================================================
-  // Private Helper Methods
-  // ============================================================================
-
-  private validateVector(vector: number[]): void {
-    if (vector.length !== this.config.dimensions) {
-      throw new Error(
-        `Vector dimension mismatch: expected ${this.config.dimensions}, got ${vector.length}`
-      );
-    }
-
-    for (let i = 0; i < vector.length; i++) {
-      if (!Number.isFinite(vector[i])) {
-        throw new Error(`Invalid vector value at index ${i}: ${vector[i]}`);
-      }
-    }
-  }
-
-  private buildKey(key: string): string {
-    return `${this.config.namespace}:${key}`;
-  }
-
-  private toHNSWResult(result: VectorSearchResult): HNSWSearchResult {
-    const key = this.extractKey(result.key);
-    return {
-      key,
-      score: result.score,
-      distance: 1 - result.score,
-      metadata: (result.metadata as CoverageVectorMetadata) || this.metadataStore.get(key),
-    };
-  }
-
-  private extractKey(fullKey: string): string {
-    const prefix = `${this.config.namespace}:`;
-    return fullKey.startsWith(prefix) ? fullKey.slice(prefix.length) : fullKey;
-  }
-
-  private distanceToSimilarity(distance: number): number {
-    switch (this.config.metric) {
-      case 'cosine':
-        return 1 - distance;
-      case 'l2':
-        return Math.exp(-distance);
-      case 'ip':
-        return -distance;
-      default:
-        return 1 - distance;
-    }
-  }
-
-  private recordSearchLatency(latencyMs: number): void {
-    this.searchLatencies.push(latencyMs);
-
-    const MAX_LATENCIES = 1000;
-    if (this.searchLatencies.length > MAX_LATENCIES) {
-      this.searchLatencies = this.searchLatencies.slice(-MAX_LATENCIES);
-    }
-  }
-
-  private calculateAvgLatency(): number {
-    if (this.searchLatencies.length === 0) return 0;
-    const sum = this.searchLatencies.reduce((a, b) => a + b, 0);
-    return sum / this.searchLatencies.length;
-  }
-
-  private calculatePercentileLatency(percentile: number): number {
-    if (this.searchLatencies.length === 0) return 0;
-
-    const sorted = [...this.searchLatencies].sort((a, b) => a - b);
-    const index = Math.ceil((percentile / 100) * sorted.length) - 1;
-    return sorted[Math.max(0, index)];
-  }
-
-  /**
-   * Update efSearch parameter for search quality/speed tradeoff
-   */
-  setEfSearch(ef: number): void {
-    if (this.nativeIndex) {
-      this.nativeIndex.setEf(ef);
-    }
-    this.config.efSearch = ef;
-  }
-}
-
-// ============================================================================
-// Internal Types
-// ============================================================================
-
-interface MutableStats {
-  vectorCount: number;
-  searchOperations: number;
-  insertOperations: number;
-}
-
-// ============================================================================
-// Factory Functions
-// ============================================================================
-
-/**
- * Create a new HNSW index instance
- *
- * @param memory - Memory backend for storage
- * @param config - Optional configuration overrides
- * @returns Configured HNSW index
- */
-export function createHNSWIndex(
-  memory: MemoryBackend,
-  config?: Partial<HNSWIndexConfig>
-): HNSWIndex {
-  return new HNSWIndex(memory, config);
-}
-
-/**
- * Run HNSW performance benchmark
- *
- * @param index - HNSW index to benchmark
- * @param vectorCount - Number of vectors to insert
- * @param searchCount - Number of searches to perform
- * @returns Benchmark results
- */
-export async function benchmarkHNSW(
-  index: HNSWIndex,
-  vectorCount: number = 10000,
-  searchCount: number = 1000
-): Promise<{
-  insertTimeMs: number;
-  searchTimeMs: number;
-  avgSearchLatencyMs: number;
-  isNative: boolean;
-}> {
-  const dimensions = 128;
-  const startInsert = performance.now();
-
-  // Insert vectors
-  for (let i = 0; i < vectorCount; i++) {
-    const vector = Array.from({ length: dimensions }, () => Math.random());
-    await index.insert(`bench-${i}`, vector);
-  }
-
-  const insertTimeMs = performance.now() - startInsert;
-
-  // Perform searches
-  const startSearch = performance.now();
-
-  for (let i = 0; i < searchCount; i++) {
-    const query = Array.from({ length: dimensions }, () => Math.random());
-    await index.search(query, 10);
-  }
-
-  const searchTimeMs = performance.now() - startSearch;
-
-  return {
-    insertTimeMs,
-    searchTimeMs,
-    avgSearchLatencyMs: searchTimeMs / searchCount,
-    isNative: index.isNativeAvailable(),
-  };
-}