@claude-flow/memory 3.0.0-alpha.2 → 3.0.0-alpha.21

package/src/agentdb-adapter.ts

@@ -1,884 +0,0 @@
-/**
- * V3 AgentDB Adapter
- *
- * Unified memory backend implementation using AgentDB with HNSW indexing
- * for 150x-12,500x faster vector search. Implements IMemoryBackend interface.
- *
- * @module v3/memory/agentdb-adapter
- */
-
-import { EventEmitter } from 'node:events';
-import {
-  IMemoryBackend,
-  MemoryEntry,
-  MemoryEntryInput,
-  MemoryEntryUpdate,
-  MemoryQuery,
-  SearchOptions,
-  SearchResult,
-  BackendStats,
-  HealthCheckResult,
-  ComponentHealth,
-  MemoryType,
-  EmbeddingGenerator,
-  generateMemoryId,
-  createDefaultEntry,
-  CacheStats,
-  HNSWStats,
-} from './types.js';
-import { HNSWIndex } from './hnsw-index.js';
-import { CacheManager } from './cache-manager.js';
-
-/**
- * Configuration for AgentDB Adapter
- */
-export interface AgentDBAdapterConfig {
-  /** Vector dimensions for embeddings (default: 1536 for OpenAI) */
-  dimensions: number;
-
-  /** Maximum number of entries */
-  maxEntries: number;
-
-  /** Enable caching */
-  cacheEnabled: boolean;
-
-  /** Maximum cache size */
-  cacheSize: number;
-
-  /** Cache TTL in milliseconds */
-  cacheTtl: number;
-
-  /** HNSW M parameter (max connections per layer) */
-  hnswM: number;
-
-  /** HNSW efConstruction parameter */
-  hnswEfConstruction: number;
-
-  /** Default namespace */
-  defaultNamespace: string;
-
-  /** Embedding generator function */
-  embeddingGenerator?: EmbeddingGenerator;
-
-  /** Enable persistence to disk */
-  persistenceEnabled: boolean;
-
-  /** Persistence path */
-  persistencePath?: string;
-}
-
-/**
- * Default configuration values
- */
-const DEFAULT_CONFIG: AgentDBAdapterConfig = {
-  dimensions: 1536,
-  maxEntries: 1000000,
-  cacheEnabled: true,
-  cacheSize: 10000,
-  cacheTtl: 300000, // 5 minutes
-  hnswM: 16,
-  hnswEfConstruction: 200,
-  defaultNamespace: 'default',
-  persistenceEnabled: false,
-};
-
-/**
- * AgentDB Memory Backend Adapter
- *
- * Provides unified memory storage with:
- * - HNSW-based vector search (150x-12,500x faster than brute force)
- * - LRU caching with TTL support
- * - Namespace-based organization
- * - Full-text and metadata filtering
- * - Event-driven architecture
- */
-export class AgentDBAdapter extends EventEmitter implements IMemoryBackend {
-  private config: AgentDBAdapterConfig;
-  private entries: Map<string, MemoryEntry> = new Map();
-  private index: HNSWIndex;
-  private cache: CacheManager<MemoryEntry>;
-  private namespaceIndex: Map<string, Set<string>> = new Map();
-  private keyIndex: Map<string, string> = new Map(); // namespace:key -> id
-  private tagIndex: Map<string, Set<string>> = new Map();
-  private initialized: boolean = false;
-
-  // Performance tracking
-  private stats = {
-    queryCount: 0,
-    totalQueryTime: 0,
-    searchCount: 0,
-    totalSearchTime: 0,
-    writeCount: 0,
-    totalWriteTime: 0,
-  };
-
-  constructor(config: Partial<AgentDBAdapterConfig> = {}) {
-    super();
-    this.config = { ...DEFAULT_CONFIG, ...config };
-
-    // Initialize HNSW index
-    this.index = new HNSWIndex({
-      dimensions: this.config.dimensions,
-      M: this.config.hnswM,
-      efConstruction: this.config.hnswEfConstruction,
-      maxElements: this.config.maxEntries,
-      metric: 'cosine',
-    });
-
-    // Initialize cache
-    this.cache = new CacheManager<MemoryEntry>({
-      maxSize: this.config.cacheSize,
-      ttl: this.config.cacheTtl,
-      lruEnabled: true,
-    });
-
-    // Forward events
-    this.index.on('point:added', (data) => this.emit('index:added', data));
-    this.cache.on('cache:hit', (data) => this.emit('cache:hit', data));
-    this.cache.on('cache:miss', (data) => this.emit('cache:miss', data));
-  }
-
-  /**
-   * Initialize the adapter
-   */
-  async initialize(): Promise<void> {
-    if (this.initialized) return;
-
-    // Load persisted data if enabled
-    if (this.config.persistenceEnabled && this.config.persistencePath) {
-      await this.loadFromDisk();
-    }
-
-    this.initialized = true;
-    this.emit('initialized');
-  }
-
-  /**
-   * Shutdown the adapter
-   */
-  async shutdown(): Promise<void> {
-    if (!this.initialized) return;
-
-    // Persist data if enabled
-    if (this.config.persistenceEnabled && this.config.persistencePath) {
-      await this.saveToDisk();
-    }
-
-    this.cache.shutdown();
-    this.initialized = false;
-    this.emit('shutdown');
-  }
-
-  /**
-   * Store a memory entry
-   */
-  async store(entry: MemoryEntry): Promise<void> {
-    const startTime = performance.now();
-
-    // Generate embedding if content provided but no embedding
-    if (entry.content && !entry.embedding && this.config.embeddingGenerator) {
-      entry.embedding = await this.config.embeddingGenerator(entry.content);
-    }
-
-    // Store in main storage
-    this.entries.set(entry.id, entry);
-
-    // Update namespace index
-    const namespace = entry.namespace || this.config.defaultNamespace;
-    if (!this.namespaceIndex.has(namespace)) {
-      this.namespaceIndex.set(namespace, new Set());
-    }
-    this.namespaceIndex.get(namespace)!.add(entry.id);
-
-    // Update key index
-    const keyIndexKey = `${namespace}:${entry.key}`;
-    this.keyIndex.set(keyIndexKey, entry.id);
-
-    // Update tag index
-    for (const tag of entry.tags) {
-      if (!this.tagIndex.has(tag)) {
-        this.tagIndex.set(tag, new Set());
-      }
-      this.tagIndex.get(tag)!.add(entry.id);
-    }
-
-    // Index embedding if available
-    if (entry.embedding) {
-      await this.index.addPoint(entry.id, entry.embedding);
-    }
-
-    // Update cache
-    if (this.config.cacheEnabled) {
-      this.cache.set(entry.id, entry);
-    }
-
-    const duration = performance.now() - startTime;
-    this.stats.writeCount++;
-    this.stats.totalWriteTime += duration;
-
-    this.emit('entry:stored', { id: entry.id, duration });
-  }
-
-  /**
-   * Get a memory entry by ID
-   */
-  async get(id: string): Promise<MemoryEntry | null> {
-    // Check cache first
-    if (this.config.cacheEnabled) {
-      const cached = this.cache.get(id);
-      if (cached) {
-        this.updateAccessStats(cached);
-        return cached;
-      }
-    }
-
-    const entry = this.entries.get(id);
-    if (entry) {
-      this.updateAccessStats(entry);
-      if (this.config.cacheEnabled) {
-        this.cache.set(id, entry);
-      }
-    }
-
-    return entry || null;
-  }
-
-  /**
-   * Get a memory entry by key within a namespace
-   */
-  async getByKey(namespace: string, key: string): Promise<MemoryEntry | null> {
-    const keyIndexKey = `${namespace}:${key}`;
-    const id = this.keyIndex.get(keyIndexKey);
-    if (!id) return null;
-    return this.get(id);
-  }
-
-  /**
-   * Update a memory entry
-   */
-  async update(id: string, update: MemoryEntryUpdate): Promise<MemoryEntry | null> {
-    const entry = this.entries.get(id);
-    if (!entry) return null;
-
-    // Apply updates
-    if (update.content !== undefined) {
-      entry.content = update.content;
-      // Regenerate embedding if content changed
-      if (this.config.embeddingGenerator) {
-        entry.embedding = await this.config.embeddingGenerator(entry.content);
-        // Re-index
-        await this.index.removePoint(id);
-        await this.index.addPoint(id, entry.embedding);
-      }
-    }
-
-    if (update.tags !== undefined) {
-      // Update tag index
-      for (const oldTag of entry.tags) {
-        this.tagIndex.get(oldTag)?.delete(id);
-      }
-      entry.tags = update.tags;
-      for (const newTag of update.tags) {
-        if (!this.tagIndex.has(newTag)) {
-          this.tagIndex.set(newTag, new Set());
-        }
-        this.tagIndex.get(newTag)!.add(id);
-      }
-    }
-
-    if (update.metadata !== undefined) {
-      entry.metadata = { ...entry.metadata, ...update.metadata };
-    }
-
-    if (update.accessLevel !== undefined) {
-      entry.accessLevel = update.accessLevel;
-    }
-
-    if (update.expiresAt !== undefined) {
-      entry.expiresAt = update.expiresAt;
-    }
-
-    if (update.references !== undefined) {
-      entry.references = update.references;
-    }
-
-    entry.updatedAt = Date.now();
-    entry.version++;
-
-    // Update cache
-    if (this.config.cacheEnabled) {
-      this.cache.set(id, entry);
-    }
-
-    this.emit('entry:updated', { id });
-    return entry;
-  }
-
-  /**
-   * Delete a memory entry
-   */
-  async delete(id: string): Promise<boolean> {
-    const entry = this.entries.get(id);
-    if (!entry) return false;
-
-    // Remove from main storage
-    this.entries.delete(id);
-
-    // Remove from namespace index
-    this.namespaceIndex.get(entry.namespace)?.delete(id);
-
-    // Remove from key index
-    const keyIndexKey = `${entry.namespace}:${entry.key}`;
-    this.keyIndex.delete(keyIndexKey);
-
-    // Remove from tag index
-    for (const tag of entry.tags) {
-      this.tagIndex.get(tag)?.delete(id);
-    }
-
-    // Remove from vector index
-    if (entry.embedding) {
-      await this.index.removePoint(id);
-    }
-
-    // Remove from cache
-    if (this.config.cacheEnabled) {
-      this.cache.delete(id);
-    }
-
-    this.emit('entry:deleted', { id });
-    return true;
-  }
-
-  /**
-   * Query memory entries with filters
-   */
-  async query(query: MemoryQuery): Promise<MemoryEntry[]> {
-    const startTime = performance.now();
-    let results: MemoryEntry[] = [];
-
-    switch (query.type) {
-      case 'exact':
-        if (query.key && query.namespace) {
-          const entry = await this.getByKey(query.namespace, query.key);
-          if (entry) results = [entry];
-        }
-        break;
-
-      case 'prefix':
-        results = this.queryByPrefix(query);
-        break;
-
-      case 'tag':
-        results = this.queryByTags(query);
-        break;
-
-      case 'semantic':
-      case 'hybrid':
-        results = await this.querySemanticWithFilters(query);
-        break;
-
-      default:
-        results = this.queryWithFilters(query);
-    }
-
-    // Apply common filters
-    results = this.applyFilters(results, query);
-
-    // Apply pagination
-    const offset = query.offset || 0;
-    results = results.slice(offset, offset + query.limit);
-
-    const duration = performance.now() - startTime;
-    this.stats.queryCount++;
-    this.stats.totalQueryTime += duration;
-
-    return results;
-  }
-
-  /**
-   * Semantic vector search
-   */
-  async search(
-    embedding: Float32Array,
-    options: SearchOptions
-  ): Promise<SearchResult[]> {
-    const startTime = performance.now();
-
-    const indexResults = await this.index.search(embedding, options.k, options.ef);
-
-    const results: SearchResult[] = [];
-    for (const { id, distance } of indexResults) {
-      const entry = this.entries.get(id);
-      if (!entry) continue;
-
-      // Apply threshold filter
-      const score = 1 - distance; // Convert distance to similarity
-      if (options.threshold && score < options.threshold) continue;
-
-      // Apply additional filters if provided
-      if (options.filters) {
-        const filtered = this.applyFilters([entry], options.filters);
-        if (filtered.length === 0) continue;
-      }
-
-      results.push({ entry, score, distance });
-    }
-
-    const duration = performance.now() - startTime;
-    this.stats.searchCount++;
-    this.stats.totalSearchTime += duration;
-
-    return results;
-  }
-
-  /**
-   * Bulk insert entries
-   */
-  async bulkInsert(entries: MemoryEntry[]): Promise<void> {
-    const startTime = performance.now();
-
-    for (const entry of entries) {
-      await this.store(entry);
-    }
-
-    const duration = performance.now() - startTime;
-    this.emit('bulk:inserted', { count: entries.length, duration });
-  }
-
-  /**
-   * Bulk delete entries
-   */
-  async bulkDelete(ids: string[]): Promise<number> {
-    let deleted = 0;
-    for (const id of ids) {
-      if (await this.delete(id)) {
-        deleted++;
-      }
-    }
-    return deleted;
-  }
-
-  /**
-   * Get entry count
-   */
-  async count(namespace?: string): Promise<number> {
-    if (namespace) {
-      return this.namespaceIndex.get(namespace)?.size || 0;
-    }
-    return this.entries.size;
-  }
-
-  /**
-   * List all namespaces
-   */
-  async listNamespaces(): Promise<string[]> {
-    return Array.from(this.namespaceIndex.keys());
-  }
-
-  /**
-   * Clear all entries in a namespace
-   */
-  async clearNamespace(namespace: string): Promise<number> {
-    const ids = this.namespaceIndex.get(namespace);
-    if (!ids) return 0;
-
-    let deleted = 0;
-    for (const id of ids) {
-      if (await this.delete(id)) {
-        deleted++;
-      }
-    }
-
-    return deleted;
-  }
-
-  /**
-   * Get backend statistics
-   */
-  async getStats(): Promise<BackendStats> {
-    const entriesByNamespace: Record<string, number> = {};
-    for (const [namespace, ids] of this.namespaceIndex) {
-      entriesByNamespace[namespace] = ids.size;
-    }
-
-    const entriesByType: Record<MemoryType, number> = {
-      episodic: 0,
-      semantic: 0,
-      procedural: 0,
-      working: 0,
-      cache: 0,
-    };
-
-    for (const entry of this.entries.values()) {
-      entriesByType[entry.type]++;
-    }
-
-    return {
-      totalEntries: this.entries.size,
-      entriesByNamespace,
-      entriesByType,
-      memoryUsage: this.estimateMemoryUsage(),
-      hnswStats: this.index.getStats(),
-      cacheStats: this.cache.getStats(),
-      avgQueryTime:
-        this.stats.queryCount > 0
-          ? this.stats.totalQueryTime / this.stats.queryCount
-          : 0,
-      avgSearchTime:
-        this.stats.searchCount > 0
-          ? this.stats.totalSearchTime / this.stats.searchCount
-          : 0,
-    };
-  }
-
-  /**
-   * Perform health check
-   */
-  async healthCheck(): Promise<HealthCheckResult> {
-    const issues: string[] = [];
-    const recommendations: string[] = [];
-
-    // Check storage health
-    const storageHealth = this.checkStorageHealth(issues, recommendations);
-
-    // Check index health
-    const indexHealth = this.checkIndexHealth(issues, recommendations);
-
-    // Check cache health
-    const cacheHealth = this.checkCacheHealth(issues, recommendations);
-
-    // Determine overall status
-    let status: 'healthy' | 'degraded' | 'unhealthy' = 'healthy';
-    if (
-      storageHealth.status === 'unhealthy' ||
-      indexHealth.status === 'unhealthy' ||
-      cacheHealth.status === 'unhealthy'
-    ) {
-      status = 'unhealthy';
-    } else if (
-      storageHealth.status === 'degraded' ||
-      indexHealth.status === 'degraded' ||
-      cacheHealth.status === 'degraded'
-    ) {
-      status = 'degraded';
-    }
-
-    return {
-      status,
-      components: {
-        storage: storageHealth,
-        index: indexHealth,
-        cache: cacheHealth,
-      },
-      timestamp: Date.now(),
-      issues,
-      recommendations,
-    };
-  }
-
-  // ===== Convenience Methods =====
-
-  /**
-   * Store a new entry from input
-   */
-  async storeEntry(input: MemoryEntryInput): Promise<MemoryEntry> {
-    const entry = createDefaultEntry(input);
-    await this.store(entry);
-    return entry;
-  }
-
-  /**
-   * Semantic search by content string
-   */
-  async semanticSearch(
-    content: string,
-    k: number = 10,
-    threshold?: number
-  ): Promise<SearchResult[]> {
-    if (!this.config.embeddingGenerator) {
-      throw new Error('Embedding generator not configured');
-    }
-
-    const embedding = await this.config.embeddingGenerator(content);
-    return this.search(embedding, { k, threshold });
-  }
-
-  // ===== Private Methods =====
-
-  private queryByPrefix(query: MemoryQuery): MemoryEntry[] {
-    const results: MemoryEntry[] = [];
-    const prefix = query.keyPrefix || '';
-    const namespace = query.namespace || this.config.defaultNamespace;
-
-    for (const [key, id] of this.keyIndex) {
-      if (key.startsWith(`${namespace}:${prefix}`)) {
-        const entry = this.entries.get(id);
-        if (entry) results.push(entry);
-      }
-    }
-
-    return results;
-  }
-
-  private queryByTags(query: MemoryQuery): MemoryEntry[] {
-    if (!query.tags || query.tags.length === 0) {
-      return Array.from(this.entries.values());
-    }
-
-    // Get intersection of entries for all tags
-    let matchingIds: Set<string> | null = null;
-
-    for (const tag of query.tags) {
-      const tagIds = this.tagIndex.get(tag);
-      if (!tagIds) {
-        return []; // Tag doesn't exist
-      }
-
-      if (matchingIds === null) {
-        matchingIds = new Set(tagIds);
-      } else {
-        // Intersect with previous results
-        for (const id of matchingIds) {
-          if (!tagIds.has(id)) {
-            matchingIds.delete(id);
-          }
-        }
-      }
-    }
-
-    if (!matchingIds) return [];
-
-    const results: MemoryEntry[] = [];
-    for (const id of matchingIds) {
-      const entry = this.entries.get(id);
-      if (entry) results.push(entry);
-    }
-
-    return results;
-  }
-
-  private async querySemanticWithFilters(
-    query: MemoryQuery
-  ): Promise<MemoryEntry[]> {
-    if (!query.content && !query.embedding) {
-      return this.queryWithFilters(query);
-    }
-
-    let embedding = query.embedding;
-    if (!embedding && query.content && this.config.embeddingGenerator) {
-      embedding = await this.config.embeddingGenerator(query.content);
-    }
-
-    if (!embedding) {
-      return this.queryWithFilters(query);
-    }
-
-    const searchResults = await this.search(embedding, {
-      k: query.limit * 2, // Over-fetch for filtering
-      threshold: query.threshold,
-      filters: query,
-    });
-
-    return searchResults.map((r) => r.entry);
-  }
-
-  private queryWithFilters(query: MemoryQuery): MemoryEntry[] {
-    let entries: MemoryEntry[] = [];
-
-    // Start with namespace filter if provided
-    if (query.namespace) {
-      const namespaceIds = this.namespaceIndex.get(query.namespace);
-      if (!namespaceIds) return [];
-      for (const id of namespaceIds) {
-        const entry = this.entries.get(id);
-        if (entry) entries.push(entry);
-      }
-    } else {
-      entries = Array.from(this.entries.values());
-    }
-
-    return entries;
-  }
-
-  private applyFilters(
-    entries: MemoryEntry[],
-    query: MemoryQuery
-  ): MemoryEntry[] {
-    return entries.filter((entry) => {
-      // Namespace filter
-      if (query.namespace && entry.namespace !== query.namespace) {
-        return false;
-      }
-
-      // Memory type filter
-      if (query.memoryType && entry.type !== query.memoryType) {
-        return false;
-      }
-
-      // Access level filter
-      if (query.accessLevel && entry.accessLevel !== query.accessLevel) {
-        return false;
-      }
-
-      // Owner filter
-      if (query.ownerId && entry.ownerId !== query.ownerId) {
-        return false;
-      }
-
-      // Tags filter
-      if (query.tags && query.tags.length > 0) {
-        if (!query.tags.every((tag) => entry.tags.includes(tag))) {
-          return false;
-        }
-      }
-
-      // Time range filters
-      if (query.createdAfter && entry.createdAt < query.createdAfter) {
-        return false;
-      }
-      if (query.createdBefore && entry.createdAt > query.createdBefore) {
-        return false;
-      }
-      if (query.updatedAfter && entry.updatedAt < query.updatedAfter) {
-        return false;
-      }
-      if (query.updatedBefore && entry.updatedAt > query.updatedBefore) {
-        return false;
-      }
-
-      // Expiration filter
-      if (!query.includeExpired && entry.expiresAt) {
-        if (entry.expiresAt < Date.now()) {
-          return false;
-        }
-      }
-
-      // Metadata filters
-      if (query.metadata) {
-        for (const [key, value] of Object.entries(query.metadata)) {
-          if (entry.metadata[key] !== value) {
-            return false;
-          }
-        }
-      }
-
-      return true;
-    });
-  }
-
-  private updateAccessStats(entry: MemoryEntry): void {
-    entry.accessCount++;
-    entry.lastAccessedAt = Date.now();
-  }
-
-  private estimateMemoryUsage(): number {
-    let total = 0;
-
-    // Estimate entry storage
-    for (const entry of this.entries.values()) {
-      total += this.estimateEntrySize(entry);
-    }
-
-    // Add index memory
-    total += this.index.getStats().memoryUsage;
-
-    // Add cache memory
-    total += this.cache.getStats().memoryUsage;
-
-    return total;
-  }
-
-  private estimateEntrySize(entry: MemoryEntry): number {
-    let size = 0;
-
-    // Base object overhead
-    size += 100;
-
-    // String fields
-    size += (entry.id.length + entry.key.length + entry.content.length) * 2;
-
-    // Embedding (Float32Array)
-    if (entry.embedding) {
-      size += entry.embedding.length * 4;
-    }
-
-    // Tags and references
-    size += entry.tags.join('').length * 2;
-    size += entry.references.join('').length * 2;
-
-    // Metadata (rough estimate)
-    size += JSON.stringify(entry.metadata).length * 2;
-
-    return size;
-  }
-
-  private checkStorageHealth(
-    issues: string[],
-    recommendations: string[]
-  ): ComponentHealth {
-    const utilizationPercent =
-      (this.entries.size / this.config.maxEntries) * 100;
-
-    if (utilizationPercent > 95) {
-      issues.push('Storage utilization critical (>95%)');
-      recommendations.push('Increase maxEntries or cleanup old data');
-      return { status: 'unhealthy', latency: 0, message: 'Storage near capacity' };
-    }
-
-    if (utilizationPercent > 80) {
-      issues.push('Storage utilization high (>80%)');
-      recommendations.push('Consider cleanup or capacity increase');
-      return { status: 'degraded', latency: 0, message: 'Storage utilization high' };
-    }
-
-    return { status: 'healthy', latency: 0 };
-  }
-
-  private checkIndexHealth(
-    issues: string[],
-    recommendations: string[]
-  ): ComponentHealth {
-    const stats = this.index.getStats();
-
-    if (stats.avgSearchTime > 10) {
-      issues.push('Index search time degraded (>10ms)');
-      recommendations.push('Consider rebuilding index or increasing ef');
-      return { status: 'degraded', latency: stats.avgSearchTime };
-    }
-
-    return { status: 'healthy', latency: stats.avgSearchTime };
-  }
-
-  private checkCacheHealth(
-    issues: string[],
-    recommendations: string[]
-  ): ComponentHealth {
-    const stats = this.cache.getStats();
-
-    if (stats.hitRate < 0.5) {
-      issues.push('Cache hit rate low (<50%)');
-      recommendations.push('Consider increasing cache size');
-      return {
-        status: 'degraded',
-        latency: 0,
-        message: `Hit rate: ${(stats.hitRate * 100).toFixed(1)}%`,
-      };
-    }
-
-    return { status: 'healthy', latency: 0 };
-  }
-
-  private async loadFromDisk(): Promise<void> {
-    // Placeholder for persistence implementation
-    // Would use SQLite or file-based storage
-    this.emit('persistence:loaded');
-  }
-
-  private async saveToDisk(): Promise<void> {
-    // Placeholder for persistence implementation
-    this.emit('persistence:saved');
-  }
-}
-
-export default AgentDBAdapter;