agentdb 3.0.0-alpha.2 → 3.0.0-alpha.4

package/src/backends/hnswlib/HNSWLibBackend.ts

@@ -0,0 +1,419 @@
+/**
+ * HNSWLibBackend - Vector backend adapter for hnswlib-node
+ *
+ * Wraps existing HNSWIndex controller to implement VectorBackend interface.
+ * Handles string ID to numeric label mapping required by hnswlib.
+ *
+ * Features:
+ * - String ID support (maps to hnswlib numeric labels)
+ * - Metadata storage alongside vectors
+ * - Persistent save/load with mappings
+ * - Backward compatible with existing HNSWIndex usage
+ *
+ * Note: hnswlib-node doesn't support true deletion - removed IDs are
+ * tracked but vectors remain until rebuild.
+ */
+
+import type {
+  VectorBackend,
+  VectorConfig,
+  SearchResult,
+  SearchOptions,
+  VectorStats,
+} from '../VectorBackend.js';
+import hnswlibNode from 'hnswlib-node';
+import * as fs from 'fs/promises';
+import * as fsSync from 'fs';
+import * as path from 'path';
+
+const { HierarchicalNSW } = hnswlibNode as any;
+
+interface SavedMappings {
+  idToLabel: Record<string, number>;
+  labelToId: Record<string, string>;
+  metadata: Record<string, Record<string, any>>;
+  nextLabel: number;
+  config: VectorConfig;
+}
+
+export class HNSWLibBackend implements VectorBackend {
+  readonly name = 'hnswlib' as const;
+
+  private index: any | null = null;
+  private config: VectorConfig;
+
+  // String ID <-> Numeric Label mappings (hnswlib requires numeric labels)
+  private idToLabel: Map<string, number> = new Map();
+  private labelToId: Map<number, string> = new Map();
+  private metadata: Map<string, Record<string, any>> = new Map();
+  private nextLabel: number = 0;
+
+  // Tracking for deletions (hnswlib doesn't support true deletion)
+  private deletedIds: Set<string> = new Set();
+
+  constructor(config: VectorConfig) {
+    // Handle both dimension and dimensions for backward compatibility
+    const dimension = config.dimension ?? config.dimensions;
+    if (!dimension) {
+      throw new Error('Vector dimension is required (use dimension or dimensions)');
+    }
+    this.config = {
+      maxElements: 100000,
+      M: 16,
+      efConstruction: 200,
+      efSearch: 100,
+      ...config,
+      dimension,  // Ensure dimension (singular) is always set
+    };
+  }
+
+  /**
+   * Initialize the HNSW index
+   * Must be called after construction
+   */
+  async initialize(): Promise<void> {
+    // Map metric names to hnswlib format
+    const metricMap: Record<string, string> = {
+      cosine: 'cosine',
+      l2: 'l2',
+      ip: 'ip',
+    };
+
+    const metric = metricMap[this.config.metric] || 'cosine';
+
+    // Create new HNSW index
+    this.index = new HierarchicalNSW(metric, this.config.dimension);
+    this.index.initIndex(
+      this.config.maxElements!,
+      this.config.M!,
+      this.config.efConstruction!
+    );
+    this.index.setEf(this.config.efSearch!);
+
+    console.log(
+      `[HNSWLibBackend] Initialized with dimension=${this.config.dimension}, ` +
+        `metric=${metric}, M=${this.config.M}, efConstruction=${this.config.efConstruction}`
+    );
+  }
+
+  /**
+   * Insert a single vector with optional metadata
+   */
+  insert(id: string, embedding: Float32Array, metadata?: Record<string, any>): void {
+    if (!this.index) {
+      throw new Error('Backend not initialized. Call initialize() first.');
+    }
+
+    // Check if ID already exists
+    if (this.idToLabel.has(id)) {
+      throw new Error(`Vector with ID '${id}' already exists`);
+    }
+
+    // Allocate numeric label
+    const label = this.nextLabel++;
+
+    // Add to index (hnswlib requires number[] not Float32Array)
+    this.index.addPoint(Array.from(embedding), label);
+
+    // Store mappings
+    this.idToLabel.set(id, label);
+    this.labelToId.set(label, id);
+
+    // Store metadata if provided
+    if (metadata) {
+      this.metadata.set(id, metadata);
+    }
+
+    // Remove from deleted set if re-inserting
+    this.deletedIds.delete(id);
+  }
+
+  /**
+   * Insert multiple vectors in batch
+   */
+  insertBatch(
+    items: Array<{
+      id: string;
+      embedding: Float32Array;
+      metadata?: Record<string, any>;
+    }>
+  ): void {
+    for (const item of items) {
+      this.insert(item.id, item.embedding, item.metadata);
+    }
+  }
+
+  /**
+   * Search for k-nearest neighbors
+   */
+  search(query: Float32Array, k: number, options?: SearchOptions): SearchResult[] {
+    if (!this.index) {
+      throw new Error('Backend not initialized. Call initialize() first.');
+    }
+
+    // Update efSearch if specified
+    if (options?.efSearch) {
+      this.index.setEf(options.efSearch);
+    }
+
+    // Perform HNSW search
+    const result = this.index.searchKnn(Array.from(query), k);
+
+    const results: SearchResult[] = [];
+
+    for (let i = 0; i < result.neighbors.length; i++) {
+      const label = result.neighbors[i];
+      const distance = result.distances[i];
+
+      // Map label back to ID
+      const id = this.labelToId.get(label);
+      if (!id) {
+        console.warn(`[HNSWLibBackend] Label ${label} not found in mapping`);
+        continue;
+      }
+
+      // Skip deleted IDs
+      if (this.deletedIds.has(id)) {
+        continue;
+      }
+
+      // Convert distance to similarity
+      const similarity = this.distanceToSimilarity(distance);
+
+      // Apply threshold if specified
+      if (options?.threshold !== undefined && similarity < options.threshold) {
+        continue;
+      }
+
+      results.push({
+        id,
+        distance,
+        similarity,
+        metadata: this.metadata.get(id),
+      });
+    }
+
+    // Apply metadata filters if specified (post-filtering)
+    if (options?.filter) {
+      return this.applyFilters(results, options.filter);
+    }
+
+    return results;
+  }
+
+  /**
+   * Remove a vector by ID
+   * Note: hnswlib doesn't support true deletion - we mark as deleted
+   */
+  remove(id: string): boolean {
+    const label = this.idToLabel.get(id);
+    if (label === undefined) {
+      return false; // Not found
+    }
+
+    // Mark as deleted (can't actually remove from hnswlib)
+    this.deletedIds.add(id);
+    this.metadata.delete(id);
+
+    // Note: We keep idToLabel/labelToId mappings for consistency
+    // A full rebuild would be needed to reclaim space
+
+    return true;
+  }
+
+  /**
+   * Get backend statistics
+   */
+  getStats(): VectorStats {
+    const activeCount = this.idToLabel.size - this.deletedIds.size;
+
+    return {
+      count: activeCount,
+      dimension: this.config.dimension || 384,
+      metric: this.config.metric,
+      backend: 'hnswlib',
+      memoryUsage: 0, // hnswlib doesn't expose memory usage
+    };
+  }
+
+  /**
+   * Save index to disk with mappings
+   */
+  async save(savePath: string): Promise<void> {
+    if (!this.index) {
+      throw new Error('No index to save');
+    }
+
+    try {
+      // Create directory if needed
+      const indexDir = path.dirname(savePath);
+      if (!fsSync.existsSync(indexDir)) {
+        await fs.mkdir(indexDir, { recursive: true });
+      }
+
+      // Save HNSW index
+      this.index.writeIndex(savePath);
+
+      // Save mappings and metadata
+      const mappingsPath = savePath + '.mappings.json';
+      const mappings: SavedMappings = {
+        idToLabel: Object.fromEntries(this.idToLabel.entries()),
+        labelToId: Object.fromEntries(
+          Array.from(this.labelToId.entries()).map(([k, v]) => [k.toString(), v])
+        ),
+        metadata: Object.fromEntries(this.metadata.entries()),
+        nextLabel: this.nextLabel,
+        config: this.config,
+      };
+
+      await fs.writeFile(mappingsPath, JSON.stringify(mappings, null, 2));
+
+      console.log(`[HNSWLibBackend] Index saved to ${savePath}`);
+      console.log(`[HNSWLibBackend] Mappings saved to ${mappingsPath}`);
+    } catch (error) {
+      console.error('[HNSWLibBackend] Failed to save index:', error);
+      throw error;
+    }
+  }
+
+  /**
+   * Load index from disk with mappings
+   */
+  async load(loadPath: string): Promise<void> {
+    if (!fsSync.existsSync(loadPath)) {
+      throw new Error(`Index file not found: ${loadPath}`);
+    }
+
+    try {
+      console.log(`[HNSWLibBackend] Loading index from ${loadPath}...`);
+
+      // Initialize index first
+      const metricMap: Record<string, string> = {
+        cosine: 'cosine',
+        l2: 'l2',
+        ip: 'ip',
+      };
+      const metric = metricMap[this.config.metric] || 'cosine';
+
+      this.index = new HierarchicalNSW(metric, this.config.dimension);
+
+      // Load HNSW index
+      this.index.readIndex(loadPath);
+      this.index.setEf(this.config.efSearch!);
+
+      // Load mappings and metadata
+      const mappingsPath = loadPath + '.mappings.json';
+      if (fsSync.existsSync(mappingsPath)) {
+        const mappingsData: SavedMappings = JSON.parse(
+          await fs.readFile(mappingsPath, 'utf-8')
+        );
+
+        // Restore mappings
+        this.idToLabel = new Map(Object.entries(mappingsData.idToLabel));
+        this.labelToId = new Map(
+          Object.entries(mappingsData.labelToId).map(([k, v]) => [Number(k), v])
+        );
+        this.metadata = new Map(Object.entries(mappingsData.metadata || {}));
+        this.nextLabel = mappingsData.nextLabel;
+
+        // Update config if saved
+        if (mappingsData.config) {
+          this.config = { ...this.config, ...mappingsData.config };
+        }
+
+        console.log(
+          `[HNSWLibBackend] ✅ Index loaded successfully (${this.idToLabel.size} vectors)`
+        );
+      } else {
+        console.warn(
+          '[HNSWLibBackend] No mappings file found - index loaded without ID mappings'
+        );
+      }
+    } catch (error) {
+      console.error('[HNSWLibBackend] Failed to load index:', error);
+      this.index = null;
+      throw error;
+    }
+  }
+
+  /**
+   * Close and cleanup resources
+   */
+  close(): void {
+    this.index = null;
+    this.idToLabel.clear();
+    this.labelToId.clear();
+    this.metadata.clear();
+    this.deletedIds.clear();
+    this.nextLabel = 0;
+  }
+
+  /**
+   * Convert distance to similarity based on metric
+   * Maps to [0, 1] range where 1 = most similar
+   */
+  private distanceToSimilarity(distance: number): number {
+    switch (this.config.metric) {
+      case 'cosine':
+        // Cosine distance is 1 - similarity, so invert
+        return 1 - distance;
+
+      case 'l2':
+        // Euclidean distance: use exponential decay
+        return Math.exp(-distance);
+
+      case 'ip':
+        // Inner product: negate distance (higher IP = more similar)
+        return -distance;
+
+      default:
+        return 1 - distance;
+    }
+  }
+
+  /**
+   * Apply metadata filters (post-filtering)
+   */
+  private applyFilters(
+    results: SearchResult[],
+    filters: Record<string, any>
+  ): SearchResult[] {
+    return results.filter((result) => {
+      if (!result.metadata) return false;
+
+      // Check if all filter conditions match
+      return Object.entries(filters).every(([key, value]) => {
+        return result.metadata![key] === value;
+      });
+    });
+  }
+
+  /**
+   * Check if needs rebuilding (for backward compat with HNSWIndex)
+   * @param updateThreshold - Percentage of deletes to trigger rebuild (default: 0.1)
+   */
+  needsRebuild(updateThreshold: number = 0.1): boolean {
+    if (this.idToLabel.size === 0) return false;
+
+    const deletePercentage = this.deletedIds.size / this.idToLabel.size;
+    return deletePercentage > updateThreshold;
+  }
+
+  /**
+   * Update efSearch parameter
+   */
+  setEfSearch(ef: number): void {
+    if (this.index) {
+      this.index.setEf(ef);
+      this.config.efSearch = ef;
+      console.log(`[HNSWLibBackend] efSearch updated to ${ef}`);
+    }
+  }
+
+  /**
+   * Check if backend is ready
+   */
+  isReady(): boolean {
+    return this.index !== null;
+  }
+}

package/src/backends/hnswlib/index.ts

@@ -0,0 +1,7 @@
+/**
+ * HNSWLib Backend Exports
+ *
+ * Export HNSWLibBackend adapter for hnswlib-node integration
+ */
+
+export { HNSWLibBackend } from './HNSWLibBackend.js';

package/src/backends/index.ts

@@ -0,0 +1,32 @@
+/**
+ * AgentDB Backends - Unified Vector Storage Interface
+ *
+ * Provides automatic backend selection between RuVector and HNSWLib
+ * with graceful fallback and clear error messages.
+ */
+
+// Core interfaces
+export type {
+  VectorBackend,
+  VectorConfig,
+  SearchResult,
+  SearchOptions,
+  VectorStats
+} from './VectorBackend.js';
+
+// Backend implementations
+export { RuVectorBackend } from './ruvector/RuVectorBackend.js';
+export { RuVectorLearning } from './ruvector/RuVectorLearning.js';
+export { HNSWLibBackend } from './hnswlib/HNSWLibBackend.js';
+
+// Factory and detection
+export {
+  createBackend,
+  detectBackends,
+  getRecommendedBackend,
+  isBackendAvailable,
+  getInstallCommand
+} from './factory.js';
+
+export type { BackendType, BackendDetection } from './factory.js';
+export type { LearningConfig, EnhancementOptions } from './ruvector/RuVectorLearning.js';

package/src/backends/ruvector/GuardedVectorBackend.ts

@@ -0,0 +1,219 @@
+/**
+ * GuardedVectorBackend - Proof-Gated State Mutation (ADR-060)
+ *
+ * A drop-in replacement for RuVectorBackend that requires cryptographic
+ * mutation proofs before allowing any state-changing operation. Read-only
+ * operations pass through without proof.
+ *
+ * Architecture:
+ * - Wraps any VectorBackend implementation
+ * - MutationGuard validates and attests each mutation
+ * - AttestationLog records all proof attempts for audit
+ * - ProofDeniedError thrown when mutation is rejected
+ *
+ * Mutating operations (require proof):
+ *   insert, insertBatch, search, remove, save, load
+ *
+ * Read-only operations (pass through):
+ *   getStats, getLearning, setLearning, close
+ */
+
+import type { VectorBackend, VectorConfig, SearchResult, SearchOptions, VectorStats } from '../VectorBackend.js';
+import type { MutationProof, MutationDenial, AttestationToken } from '../../security/MutationGuard.js';
+import { MutationGuard } from '../../security/MutationGuard.js';
+import type { AttestationLog } from '../../security/AttestationLog.js';
+
+/**
+ * Error thrown when a mutation proof is denied by the guard.
+ *
+ * Contains the denial reason, error code, operation name,
+ * and the full MutationDenial object for programmatic handling.
+ */
+export class ProofDeniedError extends Error {
+  public readonly code: string;
+  public readonly operation: string;
+  public readonly denial: MutationDenial;
+
+  constructor(reason: string, code: string, operation: string, denial: MutationDenial) {
+    super(`Proof denied for ${operation}: ${reason}`);
+    this.name = 'ProofDeniedError';
+    this.code = code;
+    this.operation = operation;
+    this.denial = denial;
+  }
+}
+
+/**
+ * GuardedVectorBackend - VectorBackend wrapper with proof-gated mutations
+ *
+ * Every mutating method requires an optional AttestationToken. When provided,
+ * the token is validated by MutationGuard before the operation proceeds.
+ * If the proof is denied, a ProofDeniedError is thrown and the operation
+ * is not forwarded to the inner backend.
+ */
+export class GuardedVectorBackend implements VectorBackend {
+  readonly name = 'ruvector' as const;
+
+  private readonly inner: VectorBackend;
+  private readonly guard: MutationGuard;
+  private readonly log?: AttestationLog;
+
+  constructor(inner: VectorBackend, guard: MutationGuard, log?: AttestationLog) {
+    this.inner = inner;
+    this.guard = guard;
+    this.log = log;
+  }
+
+  private requireProof(
+    result: MutationProof | MutationDenial,
+    operation: string,
+    token?: AttestationToken,
+  ): asserts result is MutationProof {
+    if (MutationGuard.isDenial(result)) {
+      if (this.log) {
+        this.log.recordDenial(result, token?.agentId ?? 'unknown', token?.namespace ?? 'default');
+      }
+      throw new ProofDeniedError(result.reason, result.code, operation, result);
+    }
+    if (this.log) {
+      this.log.record(result);
+    }
+  }
+
+  private wrapBackendError(err: unknown, operation: string, token?: AttestationToken): never {
+    const msg = (err as Error).message ?? String(err);
+    const denial: MutationDenial = { operation, reason: `Backend error: ${msg}`, code: 'BACKEND_ERROR', timestamp: Date.now() };
+    if (this.log) this.log.recordDenial(denial, token?.agentId ?? 'unknown', token?.namespace ?? 'default');
+    throw new ProofDeniedError(denial.reason, denial.code, operation, denial);
+  }
+
+  insert(
+    id: string,
+    embedding: Float32Array,
+    metadata?: Record<string, any>,
+    token?: AttestationToken
+  ): void {
+    this.requireProof(this.guard.proveInsert(id, embedding, metadata, token), 'insert', token);
+    try {
+      this.inner.insert(id, embedding, metadata);
+    } catch (err) {
+      this.wrapBackendError(err, 'insert', token);
+    }
+  }
+
+  insertBatch(
+    items: Array<{ id: string; embedding: Float32Array; metadata?: Record<string, any> }>,
+    token?: AttestationToken
+  ): void {
+    this.requireProof(this.guard.proveBatchInsert(items, token), 'insertBatch', token);
+    try {
+      this.inner.insertBatch(items);
+    } catch (err) {
+      this.wrapBackendError(err, 'insertBatch', token);
+    }
+  }
+
+  search(
+    query: Float32Array,
+    k: number,
+    options?: SearchOptions,
+    token?: AttestationToken
+  ): SearchResult[] {
+    this.requireProof(this.guard.proveSearch(query, k, options, token), 'search', token);
+    try {
+      return this.inner.search(query, k, options);
+    } catch (err) {
+      this.wrapBackendError(err, 'search', token);
+    }
+  }
+
+  remove(id: string, token?: AttestationToken): boolean {
+    this.requireProof(this.guard.proveRemove(id, token), 'remove', token);
+    try {
+      return this.inner.remove(id);
+    } catch (err) {
+      this.wrapBackendError(err, 'remove', token);
+    }
+  }
+
+  async save(path: string, token?: AttestationToken): Promise<void> {
+    this.requireProof(this.guard.proveSave(path, token), 'save', token);
+    try {
+      return await this.inner.save(path);
+    } catch (err) {
+      this.wrapBackendError(err, 'save', token);
+    }
+  }
+
+  async load(path: string, token?: AttestationToken): Promise<void> {
+    this.requireProof(this.guard.proveLoad(path, token), 'load', token);
+    try {
+      return await this.inner.load(path);
+    } catch (err) {
+      this.wrapBackendError(err, 'load', token);
+    }
+  }
+
+  /**
+   * Get backend statistics (read-only, no proof required).
+   *
+   * Merges inner backend stats with guard stats and optional log stats.
+   */
+  getStats(): VectorStats {
+    const innerStats = this.inner.getStats();
+    const guardStats = this.guard.getStats();
+
+    return {
+      ...innerStats,
+      ...(guardStats.proofsIssued !== undefined && {
+        totalProofs: guardStats.proofsIssued,
+        totalDenials: guardStats.denials,
+      }),
+    } as VectorStats & {
+      totalProofs?: number;
+      totalDenials?: number;
+    };
+  }
+
+  /**
+   * Get the learning instance from the inner backend (read-only, no proof required).
+   */
+  getLearning(): any {
+    return (this.inner as any).getLearning?.() ?? null;
+  }
+
+  /**
+   * Set the learning instance on the inner backend (pass-through, no proof required).
+   */
+  setLearning(learning: any): void {
+    (this.inner as any).setLearning?.(learning);
+  }
+
+  /**
+   * Close and cleanup resources (pass-through, no proof required).
+   */
+  close(): void {
+    this.inner.close();
+  }
+
+  /**
+   * Get the underlying inner backend for direct access when needed.
+   */
+  getInner(): VectorBackend {
+    return this.inner;
+  }
+
+  /**
+   * Get the mutation guard instance.
+   */
+  getGuard(): MutationGuard {
+    return this.guard;
+  }
+
+  /**
+   * Get the attestation log instance, if configured.
+   */
+  getLog(): AttestationLog | undefined {
+    return this.log;
+  }
+}