agentdb 1.6.0 → 2.0.0-alpha.1

package/src/backends/hnswlib/HNSWLibBackend.ts

@@ -0,0 +1,413 @@
+/**
+ * HNSWLibBackend - Vector backend adapter for hnswlib-node
+ *
+ * Wraps existing HNSWIndex controller to implement VectorBackend interface.
+ * Handles string ID to numeric label mapping required by hnswlib.
+ *
+ * Features:
+ * - String ID support (maps to hnswlib numeric labels)
+ * - Metadata storage alongside vectors
+ * - Persistent save/load with mappings
+ * - Backward compatible with existing HNSWIndex usage
+ *
+ * Note: hnswlib-node doesn't support true deletion - removed IDs are
+ * tracked but vectors remain until rebuild.
+ */
+
+import type {
+  VectorBackend,
+  VectorConfig,
+  SearchResult,
+  SearchOptions,
+  VectorStats,
+} from '../VectorBackend.js';
+import hnswlibNode from 'hnswlib-node';
+import * as fs from 'fs/promises';
+import * as fsSync from 'fs';
+import * as path from 'path';
+
+const { HierarchicalNSW } = hnswlibNode as any;
+
+interface SavedMappings {
+  idToLabel: Record<string, number>;
+  labelToId: Record<string, string>;
+  metadata: Record<string, Record<string, any>>;
+  nextLabel: number;
+  config: VectorConfig;
+}
+
+export class HNSWLibBackend implements VectorBackend {
+  readonly name = 'hnswlib' as const;
+
+  private index: any | null = null;
+  private config: VectorConfig;
+
+  // String ID <-> Numeric Label mappings (hnswlib requires numeric labels)
+  private idToLabel: Map<string, number> = new Map();
+  private labelToId: Map<number, string> = new Map();
+  private metadata: Map<string, Record<string, any>> = new Map();
+  private nextLabel: number = 0;
+
+  // Tracking for deletions (hnswlib doesn't support true deletion)
+  private deletedIds: Set<string> = new Set();
+
+  constructor(config: VectorConfig) {
+    this.config = {
+      maxElements: 100000,
+      M: 16,
+      efConstruction: 200,
+      efSearch: 100,
+      ...config,
+    };
+  }
+
+  /**
+   * Initialize the HNSW index
+   * Must be called after construction
+   */
+  async initialize(): Promise<void> {
+    // Map metric names to hnswlib format
+    const metricMap: Record<string, string> = {
+      cosine: 'cosine',
+      l2: 'l2',
+      ip: 'ip',
+    };
+
+    const metric = metricMap[this.config.metric] || 'cosine';
+
+    // Create new HNSW index
+    this.index = new HierarchicalNSW(metric, this.config.dimension);
+    this.index.initIndex(
+      this.config.maxElements!,
+      this.config.M!,
+      this.config.efConstruction!
+    );
+    this.index.setEf(this.config.efSearch!);
+
+    console.log(
+      `[HNSWLibBackend] Initialized with dimension=${this.config.dimension}, ` +
+        `metric=${metric}, M=${this.config.M}, efConstruction=${this.config.efConstruction}`
+    );
+  }
+
+  /**
+   * Insert a single vector with optional metadata
+   */
+  insert(id: string, embedding: Float32Array, metadata?: Record<string, any>): void {
+    if (!this.index) {
+      throw new Error('Backend not initialized. Call initialize() first.');
+    }
+
+    // Check if ID already exists
+    if (this.idToLabel.has(id)) {
+      throw new Error(`Vector with ID '${id}' already exists`);
+    }
+
+    // Allocate numeric label
+    const label = this.nextLabel++;
+
+    // Add to index (hnswlib requires number[] not Float32Array)
+    this.index.addPoint(Array.from(embedding), label);
+
+    // Store mappings
+    this.idToLabel.set(id, label);
+    this.labelToId.set(label, id);
+
+    // Store metadata if provided
+    if (metadata) {
+      this.metadata.set(id, metadata);
+    }
+
+    // Remove from deleted set if re-inserting
+    this.deletedIds.delete(id);
+  }
+
+  /**
+   * Insert multiple vectors in batch
+   */
+  insertBatch(
+    items: Array<{
+      id: string;
+      embedding: Float32Array;
+      metadata?: Record<string, any>;
+    }>
+  ): void {
+    for (const item of items) {
+      this.insert(item.id, item.embedding, item.metadata);
+    }
+  }
+
+  /**
+   * Search for k-nearest neighbors
+   */
+  search(query: Float32Array, k: number, options?: SearchOptions): SearchResult[] {
+    if (!this.index) {
+      throw new Error('Backend not initialized. Call initialize() first.');
+    }
+
+    // Update efSearch if specified
+    if (options?.efSearch) {
+      this.index.setEf(options.efSearch);
+    }
+
+    // Perform HNSW search
+    const result = this.index.searchKnn(Array.from(query), k);
+
+    const results: SearchResult[] = [];
+
+    for (let i = 0; i < result.neighbors.length; i++) {
+      const label = result.neighbors[i];
+      const distance = result.distances[i];
+
+      // Map label back to ID
+      const id = this.labelToId.get(label);
+      if (!id) {
+        console.warn(`[HNSWLibBackend] Label ${label} not found in mapping`);
+        continue;
+      }
+
+      // Skip deleted IDs
+      if (this.deletedIds.has(id)) {
+        continue;
+      }
+
+      // Convert distance to similarity
+      const similarity = this.distanceToSimilarity(distance);
+
+      // Apply threshold if specified
+      if (options?.threshold !== undefined && similarity < options.threshold) {
+        continue;
+      }
+
+      results.push({
+        id,
+        distance,
+        similarity,
+        metadata: this.metadata.get(id),
+      });
+    }
+
+    // Apply metadata filters if specified (post-filtering)
+    if (options?.filter) {
+      return this.applyFilters(results, options.filter);
+    }
+
+    return results;
+  }
+
+  /**
+   * Remove a vector by ID
+   * Note: hnswlib doesn't support true deletion - we mark as deleted
+   */
+  remove(id: string): boolean {
+    const label = this.idToLabel.get(id);
+    if (label === undefined) {
+      return false; // Not found
+    }
+
+    // Mark as deleted (can't actually remove from hnswlib)
+    this.deletedIds.add(id);
+    this.metadata.delete(id);
+
+    // Note: We keep idToLabel/labelToId mappings for consistency
+    // A full rebuild would be needed to reclaim space
+
+    return true;
+  }
+
+  /**
+   * Get backend statistics
+   */
+  getStats(): VectorStats {
+    const activeCount = this.idToLabel.size - this.deletedIds.size;
+
+    return {
+      count: activeCount,
+      dimension: this.config.dimension,
+      metric: this.config.metric,
+      backend: 'hnswlib',
+      memoryUsage: 0, // hnswlib doesn't expose memory usage
+    };
+  }
+
+  /**
+   * Save index to disk with mappings
+   */
+  async save(savePath: string): Promise<void> {
+    if (!this.index) {
+      throw new Error('No index to save');
+    }
+
+    try {
+      // Create directory if needed
+      const indexDir = path.dirname(savePath);
+      if (!fsSync.existsSync(indexDir)) {
+        await fs.mkdir(indexDir, { recursive: true });
+      }
+
+      // Save HNSW index
+      this.index.writeIndex(savePath);
+
+      // Save mappings and metadata
+      const mappingsPath = savePath + '.mappings.json';
+      const mappings: SavedMappings = {
+        idToLabel: Object.fromEntries(this.idToLabel.entries()),
+        labelToId: Object.fromEntries(
+          Array.from(this.labelToId.entries()).map(([k, v]) => [k.toString(), v])
+        ),
+        metadata: Object.fromEntries(this.metadata.entries()),
+        nextLabel: this.nextLabel,
+        config: this.config,
+      };
+
+      await fs.writeFile(mappingsPath, JSON.stringify(mappings, null, 2));
+
+      console.log(`[HNSWLibBackend] Index saved to ${savePath}`);
+      console.log(`[HNSWLibBackend] Mappings saved to ${mappingsPath}`);
+    } catch (error) {
+      console.error('[HNSWLibBackend] Failed to save index:', error);
+      throw error;
+    }
+  }
+
+  /**
+   * Load index from disk with mappings
+   */
+  async load(loadPath: string): Promise<void> {
+    if (!fsSync.existsSync(loadPath)) {
+      throw new Error(`Index file not found: ${loadPath}`);
+    }
+
+    try {
+      console.log(`[HNSWLibBackend] Loading index from ${loadPath}...`);
+
+      // Initialize index first
+      const metricMap: Record<string, string> = {
+        cosine: 'cosine',
+        l2: 'l2',
+        ip: 'ip',
+      };
+      const metric = metricMap[this.config.metric] || 'cosine';
+
+      this.index = new HierarchicalNSW(metric, this.config.dimension);
+
+      // Load HNSW index
+      this.index.readIndex(loadPath);
+      this.index.setEf(this.config.efSearch!);
+
+      // Load mappings and metadata
+      const mappingsPath = loadPath + '.mappings.json';
+      if (fsSync.existsSync(mappingsPath)) {
+        const mappingsData: SavedMappings = JSON.parse(
+          await fs.readFile(mappingsPath, 'utf-8')
+        );
+
+        // Restore mappings
+        this.idToLabel = new Map(Object.entries(mappingsData.idToLabel));
+        this.labelToId = new Map(
+          Object.entries(mappingsData.labelToId).map(([k, v]) => [Number(k), v])
+        );
+        this.metadata = new Map(Object.entries(mappingsData.metadata || {}));
+        this.nextLabel = mappingsData.nextLabel;
+
+        // Update config if saved
+        if (mappingsData.config) {
+          this.config = { ...this.config, ...mappingsData.config };
+        }
+
+        console.log(
+          `[HNSWLibBackend] ✅ Index loaded successfully (${this.idToLabel.size} vectors)`
+        );
+      } else {
+        console.warn(
+          '[HNSWLibBackend] No mappings file found - index loaded without ID mappings'
+        );
+      }
+    } catch (error) {
+      console.error('[HNSWLibBackend] Failed to load index:', error);
+      this.index = null;
+      throw error;
+    }
+  }
+
+  /**
+   * Close and cleanup resources
+   */
+  close(): void {
+    this.index = null;
+    this.idToLabel.clear();
+    this.labelToId.clear();
+    this.metadata.clear();
+    this.deletedIds.clear();
+    this.nextLabel = 0;
+  }
+
+  /**
+   * Convert distance to similarity based on metric
+   * Maps to [0, 1] range where 1 = most similar
+   */
+  private distanceToSimilarity(distance: number): number {
+    switch (this.config.metric) {
+      case 'cosine':
+        // Cosine distance is 1 - similarity, so invert
+        return 1 - distance;
+
+      case 'l2':
+        // Euclidean distance: use exponential decay
+        return Math.exp(-distance);
+
+      case 'ip':
+        // Inner product: negate distance (higher IP = more similar)
+        return -distance;
+
+      default:
+        return 1 - distance;
+    }
+  }
+
+  /**
+   * Apply metadata filters (post-filtering)
+   */
+  private applyFilters(
+    results: SearchResult[],
+    filters: Record<string, any>
+  ): SearchResult[] {
+    return results.filter((result) => {
+      if (!result.metadata) return false;
+
+      // Check if all filter conditions match
+      return Object.entries(filters).every(([key, value]) => {
+        return result.metadata![key] === value;
+      });
+    });
+  }
+
+  /**
+   * Check if needs rebuilding (for backward compat with HNSWIndex)
+   * @param updateThreshold - Percentage of deletes to trigger rebuild (default: 0.1)
+   */
+  needsRebuild(updateThreshold: number = 0.1): boolean {
+    if (this.idToLabel.size === 0) return false;
+
+    const deletePercentage = this.deletedIds.size / this.idToLabel.size;
+    return deletePercentage > updateThreshold;
+  }
+
+  /**
+   * Update efSearch parameter
+   */
+  setEfSearch(ef: number): void {
+    if (this.index) {
+      this.index.setEf(ef);
+      this.config.efSearch = ef;
+      console.log(`[HNSWLibBackend] efSearch updated to ${ef}`);
+    }
+  }
+
+  /**
+   * Check if backend is ready
+   */
+  isReady(): boolean {
+    return this.index !== null;
+  }
+}

package/src/backends/hnswlib/index.ts

@@ -0,0 +1,7 @@
+/**
+ * HNSWLib Backend Exports
+ *
+ * Export HNSWLibBackend adapter for hnswlib-node integration
+ */
+
+export { HNSWLibBackend } from './HNSWLibBackend.js';

package/src/backends/index.ts

@@ -0,0 +1,32 @@
+/**
+ * AgentDB Backends - Unified Vector Storage Interface
+ *
+ * Provides automatic backend selection between RuVector and HNSWLib
+ * with graceful fallback and clear error messages.
+ */
+
+// Core interfaces
+export type {
+  VectorBackend,
+  VectorConfig,
+  SearchResult,
+  SearchOptions,
+  VectorStats
+} from './VectorBackend.js';
+
+// Backend implementations
+export { RuVectorBackend } from './ruvector/RuVectorBackend.js';
+export { RuVectorLearning } from './ruvector/RuVectorLearning.js';
+export { HNSWLibBackend } from './hnswlib/HNSWLibBackend.js';
+
+// Factory and detection
+export {
+  createBackend,
+  detectBackends,
+  getRecommendedBackend,
+  isBackendAvailable,
+  getInstallCommand
+} from './factory.js';
+
+export type { BackendType, BackendDetection } from './factory.js';
+export type { LearningConfig, EnhancementOptions } from './ruvector/RuVectorLearning.js';

package/src/backends/ruvector/RuVectorBackend.ts

@@ -0,0 +1,232 @@
+/**
+ * RuVectorBackend - High-Performance Vector Storage
+ *
+ * Implements VectorBackend using @ruvector/core with optional GNN support.
+ * Provides <100µs search latency with native SIMD optimizations.
+ *
+ * Features:
+ * - Automatic fallback when @ruvector packages not installed
+ * - Separate metadata storage for rich queries
+ * - Distance-to-similarity conversion for all metrics
+ * - Batch operations for optimal throughput
+ * - Persistent storage with separate metadata files
+ */
+
+import type { VectorBackend, VectorConfig, SearchResult, SearchOptions, VectorStats } from '../VectorBackend.js';
+
+export class RuVectorBackend implements VectorBackend {
+  readonly name = 'ruvector' as const;
+  private db: any; // VectorDB from @ruvector/core
+  private config: VectorConfig;
+  private metadata: Map<string, Record<string, any>> = new Map();
+  private initialized = false;
+
+  constructor(config: VectorConfig) {
+    this.config = config;
+  }
+
+  /**
+   * Initialize RuVector database with optional dependency handling
+   */
+  async initialize(): Promise<void> {
+    if (this.initialized) return;
+
+    try {
+      // Try main ruvector package first (includes core, gnn, graph)
+      let VectorDB;
+      try {
+        const ruvector = await import('ruvector');
+        VectorDB = ruvector.VectorDB || ruvector.default?.VectorDB;
+      } catch {
+        // Fallback to @ruvector/core for backward compatibility
+        const core = await import('@ruvector/core');
+        VectorDB = core.VectorDB || core.default;
+      }
+
+      this.db = new VectorDB(this.config.dimension, {
+        metric: this.config.metric,
+        maxElements: this.config.maxElements || 100000,
+        efConstruction: this.config.efConstruction || 200,
+        M: this.config.M || 16
+      });
+
+      this.initialized = true;
+    } catch (error) {
+      throw new Error(
+        `RuVector initialization failed. Please install: npm install ruvector\n` +
+        `Or legacy packages: npm install @ruvector/core\n` +
+        `Error: ${(error as Error).message}`
+      );
+    }
+  }
+
+  /**
+   * Insert single vector with optional metadata
+   */
+  insert(id: string, embedding: Float32Array, metadata?: Record<string, any>): void {
+    this.ensureInitialized();
+
+    // RuVector expects regular arrays
+    this.db.insert(id, Array.from(embedding));
+
+    if (metadata) {
+      this.metadata.set(id, metadata);
+    }
+  }
+
+  /**
+   * Batch insert for optimal performance
+   */
+  insertBatch(items: Array<{ id: string; embedding: Float32Array; metadata?: Record<string, any> }>): void {
+    this.ensureInitialized();
+
+    for (const item of items) {
+      this.insert(item.id, item.embedding, item.metadata);
+    }
+  }
+
+  /**
+   * Search for k-nearest neighbors with optional filtering
+   */
+  search(query: Float32Array, k: number, options?: SearchOptions): SearchResult[] {
+    this.ensureInitialized();
+
+    // Apply efSearch parameter if provided
+    if (options?.efSearch) {
+      this.db.setEfSearch(options.efSearch);
+    }
+
+    // Perform vector search
+    const results = this.db.search(Array.from(query), k);
+
+    // Convert results and apply filtering
+    return results
+      .map((r: { id: string; distance: number }) => ({
+        id: r.id,
+        distance: r.distance,
+        similarity: this.distanceToSimilarity(r.distance),
+        metadata: this.metadata.get(r.id)
+      }))
+      .filter((r: SearchResult) => {
+        // Apply similarity threshold
+        if (options?.threshold && r.similarity < options.threshold) {
+          return false;
+        }
+
+        // Apply metadata filters
+        if (options?.filter && r.metadata) {
+          return Object.entries(options.filter).every(
+            ([key, value]) => r.metadata![key] === value
+          );
+        }
+
+        return true;
+      });
+  }
+
+  /**
+   * Remove vector by ID
+   */
+  remove(id: string): boolean {
+    this.ensureInitialized();
+
+    this.metadata.delete(id);
+
+    try {
+      return this.db.remove(id);
+    } catch {
+      return false;
+    }
+  }
+
+  /**
+   * Get database statistics
+   */
+  getStats(): VectorStats {
+    this.ensureInitialized();
+
+    return {
+      count: this.db.count(),
+      dimension: this.config.dimension,
+      metric: this.config.metric,
+      backend: 'ruvector',
+      memoryUsage: this.db.memoryUsage?.() || 0
+    };
+  }
+
+  /**
+   * Save index and metadata to disk
+   */
+  async save(path: string): Promise<void> {
+    this.ensureInitialized();
+
+    // Save vector index
+    this.db.save(path);
+
+    // Save metadata separately as JSON
+    const metadataPath = path + '.meta.json';
+    const fs = await import('fs/promises');
+    await fs.writeFile(
+      metadataPath,
+      JSON.stringify(Object.fromEntries(this.metadata), null, 2)
+    );
+  }
+
+  /**
+   * Load index and metadata from disk
+   */
+  async load(path: string): Promise<void> {
+    this.ensureInitialized();
+
+    // Load vector index
+    this.db.load(path);
+
+    // Load metadata
+    const metadataPath = path + '.meta.json';
+    try {
+      const fs = await import('fs/promises');
+      const data = await fs.readFile(metadataPath, 'utf-8');
+      this.metadata = new Map(Object.entries(JSON.parse(data)));
+    } catch {
+      // No metadata file - this is okay for backward compatibility
+      console.debug(`[RuVectorBackend] No metadata file found at ${metadataPath}`);
+    }
+  }
+
+  /**
+   * Close and cleanup resources
+   */
+  close(): void {
+    // RuVector cleanup if needed
+    this.metadata.clear();
+  }
+
+  /**
+   * Convert distance to similarity score based on metric
+   *
+   * Cosine: distance is already in [0, 2], where 0 = identical
+   * L2: exponential decay for unbounded distances
+   * IP: negative inner product, so negate for similarity
+   */
+  private distanceToSimilarity(distance: number): number {
+    switch (this.config.metric) {
+      case 'cosine':
+        return 1 - distance; // cosine distance is 1 - similarity
+      case 'l2':
+        return Math.exp(-distance); // exponential decay
+      case 'ip':
+        return -distance; // inner product: higher is better
+      default:
+        return 1 - distance;
+    }
+  }
+
+  /**
+   * Ensure database is initialized before operations
+   */
+  private ensureInitialized(): void {
+    if (!this.initialized) {
+      throw new Error('RuVectorBackend not initialized. Call initialize() first.');
+    }
+  }
+}