@claude-flow/memory 3.0.0-alpha.2 → 3.0.0-alpha.20

package/src/hybrid-backend.ts

@@ -1,694 +0,0 @@
-/**
- * HybridBackend - Combines SQLite (structured queries) + AgentDB (vector search)
- *
- * Per ADR-009: "HybridBackend (SQLite + AgentDB) as default"
- * - SQLite for: Structured queries, ACID transactions, exact matches
- * - AgentDB for: Semantic search, vector similarity, RAG
- *
- * @module v3/memory/hybrid-backend
- */
-
-import { EventEmitter } from 'node:events';
-import {
-  IMemoryBackend,
-  MemoryEntry,
-  MemoryEntryInput,
-  MemoryEntryUpdate,
-  MemoryQuery,
-  SearchOptions,
-  SearchResult,
-  BackendStats,
-  HealthCheckResult,
-  ComponentHealth,
-  EmbeddingGenerator,
-  createDefaultEntry,
-  QueryType,
-} from './types.js';
-import { SQLiteBackend, SQLiteBackendConfig } from './sqlite-backend.js';
-import { AgentDBBackend, AgentDBBackendConfig } from './agentdb-backend.js';
-
-/**
- * Configuration for HybridBackend
- */
-export interface HybridBackendConfig {
-  /** SQLite configuration */
-  sqlite?: Partial<SQLiteBackendConfig>;
-
-  /** AgentDB configuration */
-  agentdb?: Partial<AgentDBBackendConfig>;
-
-  /** Default namespace */
-  defaultNamespace?: string;
-
-  /** Embedding generator function */
-  embeddingGenerator?: EmbeddingGenerator;
-
-  /** Query routing strategy */
-  routingStrategy?: 'auto' | 'sqlite-first' | 'agentdb-first';
-
-  /** Enable dual-write (write to both backends) */
-  dualWrite?: boolean;
-
-  /** Semantic search threshold for hybrid queries */
-  semanticThreshold?: number;
-
-  /** Maximum results to fetch from each backend in hybrid queries */
-  hybridMaxResults?: number;
-}
-
-/**
- * Default configuration
- */
-const DEFAULT_CONFIG: Required<HybridBackendConfig> = {
-  sqlite: {},
-  agentdb: {},
-  defaultNamespace: 'default',
-  embeddingGenerator: undefined as any,
-  routingStrategy: 'auto',
-  dualWrite: true,
-  semanticThreshold: 0.7,
-  hybridMaxResults: 100,
-};
-
-/**
- * Structured Query Interface
- * Optimized for SQLite's strengths
- */
-export interface StructuredQuery {
-  /** Exact key match */
-  key?: string;
-
-  /** Key prefix match */
-  keyPrefix?: string;
-
-  /** Namespace filter */
-  namespace?: string;
-
-  /** Owner filter */
-  ownerId?: string;
-
-  /** Type filter */
-  type?: string;
-
-  /** Time range filters */
-  createdAfter?: number;
-  createdBefore?: number;
-  updatedAfter?: number;
-  updatedBefore?: number;
-
-  /** Pagination */
-  limit?: number;
-  offset?: number;
-}
-
-/**
- * Semantic Query Interface
- * Optimized for AgentDB's vector search
- */
-export interface SemanticQuery {
-  /** Content to search for (will be embedded) */
-  content?: string;
-
-  /** Pre-computed embedding */
-  embedding?: Float32Array;
-
-  /** Number of results */
-  k?: number;
-
-  /** Similarity threshold (0-1) */
-  threshold?: number;
-
-  /** Additional filters */
-  filters?: Partial<MemoryQuery>;
-}
-
-/**
- * Hybrid Query Interface
- * Combines structured + semantic search
- */
-export interface HybridQuery {
-  /** Semantic component */
-  semantic: SemanticQuery;
-
-  /** Structured component */
-  structured?: StructuredQuery;
-
-  /** How to combine results */
-  combineStrategy?: 'union' | 'intersection' | 'semantic-first' | 'structured-first';
-
-  /** Weights for score combination */
-  weights?: {
-    semantic: number;
-    structured: number;
-  };
-}
-
-/**
- * HybridBackend Implementation
- *
- * Intelligently routes queries between SQLite and AgentDB:
- * - Exact matches, prefix queries → SQLite
- * - Semantic search, similarity → AgentDB
- * - Complex hybrid queries → Both backends with intelligent merging
- */
-export class HybridBackend extends EventEmitter implements IMemoryBackend {
-  private sqlite: SQLiteBackend;
-  private agentdb: AgentDBBackend;
-  private config: Required<HybridBackendConfig>;
-  private initialized: boolean = false;
-
-  // Performance tracking
-  private stats = {
-    sqliteQueries: 0,
-    agentdbQueries: 0,
-    hybridQueries: 0,
-    totalQueryTime: 0,
-  };
-
-  constructor(config: HybridBackendConfig = {}) {
-    super();
-    this.config = { ...DEFAULT_CONFIG, ...config };
-
-    // Initialize SQLite backend
-    this.sqlite = new SQLiteBackend({
-      ...this.config.sqlite,
-      defaultNamespace: this.config.defaultNamespace,
-      embeddingGenerator: this.config.embeddingGenerator,
-    });
-
-    // Initialize AgentDB backend
-    this.agentdb = new AgentDBBackend({
-      ...this.config.agentdb,
-      namespace: this.config.defaultNamespace,
-      embeddingGenerator: this.config.embeddingGenerator,
-    });
-
-    // Forward events from both backends
-    this.sqlite.on('entry:stored', (data) => this.emit('sqlite:stored', data));
-    this.sqlite.on('entry:updated', (data) => this.emit('sqlite:updated', data));
-    this.sqlite.on('entry:deleted', (data) => this.emit('sqlite:deleted', data));
-
-    this.agentdb.on('entry:stored', (data) => this.emit('agentdb:stored', data));
-    this.agentdb.on('entry:updated', (data) => this.emit('agentdb:updated', data));
-    this.agentdb.on('entry:deleted', (data) => this.emit('agentdb:deleted', data));
-    this.agentdb.on('cache:hit', (data) => this.emit('cache:hit', data));
-    this.agentdb.on('cache:miss', (data) => this.emit('cache:miss', data));
-  }
-
-  /**
-   * Initialize both backends
-   */
-  async initialize(): Promise<void> {
-    if (this.initialized) return;
-
-    await Promise.all([this.sqlite.initialize(), this.agentdb.initialize()]);
-
-    this.initialized = true;
-    this.emit('initialized');
-  }
-
-  /**
-   * Shutdown both backends
-   */
-  async shutdown(): Promise<void> {
-    if (!this.initialized) return;
-
-    await Promise.all([this.sqlite.shutdown(), this.agentdb.shutdown()]);
-
-    this.initialized = false;
-    this.emit('shutdown');
-  }
-
-  /**
-   * Store in both backends (dual-write for consistency)
-   */
-  async store(entry: MemoryEntry): Promise<void> {
-    if (this.config.dualWrite) {
-      // Write to both backends in parallel
-      await Promise.all([this.sqlite.store(entry), this.agentdb.store(entry)]);
-    } else {
-      // Write to primary backend only (AgentDB has vector search)
-      await this.agentdb.store(entry);
-    }
-
-    this.emit('entry:stored', { id: entry.id });
-  }
-
-  /**
-   * Get from AgentDB (has caching enabled)
-   */
-  async get(id: string): Promise<MemoryEntry | null> {
-    return this.agentdb.get(id);
-  }
-
-  /**
-   * Get by key (SQLite optimized for exact matches)
-   */
-  async getByKey(namespace: string, key: string): Promise<MemoryEntry | null> {
-    return this.sqlite.getByKey(namespace, key);
-  }
-
-  /**
-   * Update in both backends
-   */
-  async update(id: string, update: MemoryEntryUpdate): Promise<MemoryEntry | null> {
-    if (this.config.dualWrite) {
-      // Update both backends
-      const [sqliteResult, agentdbResult] = await Promise.all([
-        this.sqlite.update(id, update),
-        this.agentdb.update(id, update),
-      ]);
-      return agentdbResult || sqliteResult;
-    } else {
-      return this.agentdb.update(id, update);
-    }
-  }
-
-  /**
-   * Delete from both backends
-   */
-  async delete(id: string): Promise<boolean> {
-    if (this.config.dualWrite) {
-      const [sqliteResult, agentdbResult] = await Promise.all([
-        this.sqlite.delete(id),
-        this.agentdb.delete(id),
-      ]);
-      return sqliteResult || agentdbResult;
-    } else {
-      return this.agentdb.delete(id);
-    }
-  }
-
-  /**
-   * Query routing - semantic goes to AgentDB, structured to SQLite
-   */
-  async query(query: MemoryQuery): Promise<MemoryEntry[]> {
-    const startTime = performance.now();
-
-    let results: MemoryEntry[];
-
-    // Route based on query type
-    switch (query.type) {
-      case 'exact':
-        // SQLite optimized for exact matches
-        this.stats.sqliteQueries++;
-        results = await this.sqlite.query(query);
-        break;
-
-      case 'prefix':
-        // SQLite optimized for prefix queries
-        this.stats.sqliteQueries++;
-        results = await this.sqlite.query(query);
-        break;
-
-      case 'tag':
-        // Both can handle tags, use SQLite for structured filtering
-        this.stats.sqliteQueries++;
-        results = await this.sqlite.query(query);
-        break;
-
-      case 'semantic':
-        // AgentDB optimized for semantic search
-        this.stats.agentdbQueries++;
-        results = await this.agentdb.query(query);
-        break;
-
-      case 'hybrid':
-        // Use hybrid query combining both backends
-        this.stats.hybridQueries++;
-        results = await this.queryHybridInternal(query);
-        break;
-
-      default:
-        // Auto-routing based on query properties
-        results = await this.autoRoute(query);
-    }
-
-    const duration = performance.now() - startTime;
-    this.stats.totalQueryTime += duration;
-
-    this.emit('query:completed', { type: query.type, duration, count: results.length });
-    return results;
-  }
-
-  /**
-   * Structured queries (SQL)
-   * Routes to SQLite for optimal performance
-   */
-  async queryStructured(query: StructuredQuery): Promise<MemoryEntry[]> {
-    this.stats.sqliteQueries++;
-
-    const memoryQuery: MemoryQuery = {
-      type: query.key ? 'exact' : query.keyPrefix ? 'prefix' : 'hybrid',
-      key: query.key,
-      keyPrefix: query.keyPrefix,
-      namespace: query.namespace,
-      ownerId: query.ownerId,
-      memoryType: query.type as any,
-      createdAfter: query.createdAfter,
-      createdBefore: query.createdBefore,
-      updatedAfter: query.updatedAfter,
-      updatedBefore: query.updatedBefore,
-      limit: query.limit || 100,
-      offset: query.offset || 0,
-    };
-
-    return this.sqlite.query(memoryQuery);
-  }
-
-  /**
-   * Semantic queries (vector)
-   * Routes to AgentDB for HNSW-based vector search
-   */
-  async querySemantic(query: SemanticQuery): Promise<MemoryEntry[]> {
-    this.stats.agentdbQueries++;
-
-    let embedding = query.embedding;
-
-    // Generate embedding if content provided
-    if (!embedding && query.content && this.config.embeddingGenerator) {
-      embedding = await this.config.embeddingGenerator(query.content);
-    }
-
-    if (!embedding) {
-      throw new Error('SemanticQuery requires either content or embedding');
-    }
-
-    const searchResults = await this.agentdb.search(embedding, {
-      k: query.k || 10,
-      threshold: query.threshold || this.config.semanticThreshold,
-      filters: query.filters as MemoryQuery | undefined,
-    });
-
-    return searchResults.map((r) => r.entry);
-  }
-
-  /**
-   * Hybrid queries (combine both)
-   * Intelligently merges results from both backends
-   */
-  async queryHybrid(query: HybridQuery): Promise<MemoryEntry[]> {
-    this.stats.hybridQueries++;
-
-    const strategy = query.combineStrategy || 'semantic-first';
-    const weights = query.weights || { semantic: 0.7, structured: 0.3 };
-
-    // Execute both queries in parallel
-    const [semanticResults, structuredResults] = await Promise.all([
-      this.querySemantic(query.semantic),
-      query.structured ? this.queryStructured(query.structured) : Promise.resolve([]),
-    ]);
-
-    // Combine results based on strategy
-    switch (strategy) {
-      case 'union':
-        return this.combineUnion(semanticResults, structuredResults);
-
-      case 'intersection':
-        return this.combineIntersection(semanticResults, structuredResults);
-
-      case 'semantic-first':
-        return this.combineSemanticFirst(semanticResults, structuredResults);
-
-      case 'structured-first':
-        return this.combineStructuredFirst(semanticResults, structuredResults);
-
-      default:
-        return this.combineUnion(semanticResults, structuredResults);
-    }
-  }
-
-  /**
-   * Semantic vector search (routes to AgentDB)
-   */
-  async search(embedding: Float32Array, options: SearchOptions): Promise<SearchResult[]> {
-    this.stats.agentdbQueries++;
-    return this.agentdb.search(embedding, options);
-  }
-
-  /**
-   * Bulk insert to both backends
-   */
-  async bulkInsert(entries: MemoryEntry[]): Promise<void> {
-    if (this.config.dualWrite) {
-      await Promise.all([this.sqlite.bulkInsert(entries), this.agentdb.bulkInsert(entries)]);
-    } else {
-      await this.agentdb.bulkInsert(entries);
-    }
-  }
-
-  /**
-   * Bulk delete from both backends
-   */
-  async bulkDelete(ids: string[]): Promise<number> {
-    if (this.config.dualWrite) {
-      const [sqliteCount, agentdbCount] = await Promise.all([
-        this.sqlite.bulkDelete(ids),
-        this.agentdb.bulkDelete(ids),
-      ]);
-      return Math.max(sqliteCount, agentdbCount);
-    } else {
-      return this.agentdb.bulkDelete(ids);
-    }
-  }
-
-  /**
-   * Count entries (use SQLite for efficiency)
-   */
-  async count(namespace?: string): Promise<number> {
-    return this.sqlite.count(namespace);
-  }
-
-  /**
-   * List namespaces (use SQLite)
-   */
-  async listNamespaces(): Promise<string[]> {
-    return this.sqlite.listNamespaces();
-  }
-
-  /**
-   * Clear namespace in both backends
-   */
-  async clearNamespace(namespace: string): Promise<number> {
-    if (this.config.dualWrite) {
-      const [sqliteCount, agentdbCount] = await Promise.all([
-        this.sqlite.clearNamespace(namespace),
-        this.agentdb.clearNamespace(namespace),
-      ]);
-      return Math.max(sqliteCount, agentdbCount);
-    } else {
-      return this.agentdb.clearNamespace(namespace);
-    }
-  }
-
-  /**
-   * Get combined statistics from both backends
-   */
-  async getStats(): Promise<BackendStats> {
-    const [sqliteStats, agentdbStats] = await Promise.all([
-      this.sqlite.getStats(),
-      this.agentdb.getStats(),
-    ]);
-
-    return {
-      totalEntries: Math.max(sqliteStats.totalEntries, agentdbStats.totalEntries),
-      entriesByNamespace: agentdbStats.entriesByNamespace,
-      entriesByType: agentdbStats.entriesByType,
-      memoryUsage: sqliteStats.memoryUsage + agentdbStats.memoryUsage,
-      hnswStats: agentdbStats.hnswStats,
-      cacheStats: agentdbStats.cacheStats,
-      avgQueryTime:
-        this.stats.hybridQueries + this.stats.sqliteQueries + this.stats.agentdbQueries > 0
-          ? this.stats.totalQueryTime /
-            (this.stats.hybridQueries + this.stats.sqliteQueries + this.stats.agentdbQueries)
-          : 0,
-      avgSearchTime: agentdbStats.avgSearchTime,
-    };
-  }
-
-  /**
-   * Health check for both backends
-   */
-  async healthCheck(): Promise<HealthCheckResult> {
-    const [sqliteHealth, agentdbHealth] = await Promise.all([
-      this.sqlite.healthCheck(),
-      this.agentdb.healthCheck(),
-    ]);
-
-    const allIssues = [...sqliteHealth.issues, ...agentdbHealth.issues];
-    const allRecommendations = [
-      ...sqliteHealth.recommendations,
-      ...agentdbHealth.recommendations,
-    ];
-
-    // Determine overall status
-    let status: 'healthy' | 'degraded' | 'unhealthy' = 'healthy';
-    if (
-      sqliteHealth.status === 'unhealthy' ||
-      agentdbHealth.status === 'unhealthy'
-    ) {
-      status = 'unhealthy';
-    } else if (
-      sqliteHealth.status === 'degraded' ||
-      agentdbHealth.status === 'degraded'
-    ) {
-      status = 'degraded';
-    }
-
-    return {
-      status,
-      components: {
-        storage: sqliteHealth.components.storage,
-        index: agentdbHealth.components.index,
-        cache: agentdbHealth.components.cache,
-      },
-      timestamp: Date.now(),
-      issues: allIssues,
-      recommendations: allRecommendations,
-    };
-  }
-
-  // ===== Private Methods =====
-
-  /**
-   * Auto-route queries based on properties
-   */
-  private async autoRoute(query: MemoryQuery): Promise<MemoryEntry[]> {
-    // If has embedding or content, use semantic search (AgentDB)
-    const hasEmbeddingGenerator = typeof this.config.embeddingGenerator === 'function';
-    if (query.embedding || (query.content && hasEmbeddingGenerator)) {
-      this.stats.agentdbQueries++;
-      return this.agentdb.query(query);
-    }
-
-    // If has exact key or prefix, use structured search (SQLite)
-    if (query.key || query.keyPrefix) {
-      this.stats.sqliteQueries++;
-      return this.sqlite.query(query);
-    }
-
-    // For other filters, use routing strategy
-    switch (this.config.routingStrategy) {
-      case 'sqlite-first':
-        this.stats.sqliteQueries++;
-        return this.sqlite.query(query);
-
-      case 'agentdb-first':
-        this.stats.agentdbQueries++;
-        return this.agentdb.query(query);
-
-      case 'auto':
-      default:
-        // Default to AgentDB (has caching)
-        this.stats.agentdbQueries++;
-        return this.agentdb.query(query);
-    }
-  }
-
-  /**
-   * Internal hybrid query implementation
-   */
-  private async queryHybridInternal(query: MemoryQuery): Promise<MemoryEntry[]> {
-    // If semantic component exists, use hybrid
-    if (query.embedding || query.content) {
-      const semanticQuery: SemanticQuery = {
-        content: query.content,
-        embedding: query.embedding,
-        k: query.limit || 10,
-        threshold: query.threshold,
-        filters: query,
-      };
-
-      const structuredQuery: StructuredQuery = {
-        namespace: query.namespace,
-        key: query.key,
-        keyPrefix: query.keyPrefix,
-        ownerId: query.ownerId,
-        type: query.memoryType,
-        createdAfter: query.createdAfter,
-        createdBefore: query.createdBefore,
-        updatedAfter: query.updatedAfter,
-        updatedBefore: query.updatedBefore,
-        limit: query.limit,
-        offset: query.offset,
-      };
-
-      return this.queryHybrid({
-        semantic: semanticQuery,
-        structured: structuredQuery,
-        combineStrategy: 'semantic-first',
-      });
-    }
-
-    // Otherwise, route to structured
-    return this.autoRoute(query);
-  }
-
-  /**
-   * Combine results using union (all unique results)
-   */
-  private combineUnion(
-    semanticResults: MemoryEntry[],
-    structuredResults: MemoryEntry[]
-  ): MemoryEntry[] {
-    const seen = new Set<string>();
-    const combined: MemoryEntry[] = [];
-
-    for (const entry of [...semanticResults, ...structuredResults]) {
-      if (!seen.has(entry.id)) {
-        seen.add(entry.id);
-        combined.push(entry);
-      }
-    }
-
-    return combined;
-  }
-
-  /**
-   * Combine results using intersection (only common results)
-   */
-  private combineIntersection(
-    semanticResults: MemoryEntry[],
-    structuredResults: MemoryEntry[]
-  ): MemoryEntry[] {
-    const semanticIds = new Set(semanticResults.map((e) => e.id));
-    return structuredResults.filter((e) => semanticIds.has(e.id));
-  }
-
-  /**
-   * Semantic-first: Prefer semantic results, add structured if not present
-   */
-  private combineSemanticFirst(
-    semanticResults: MemoryEntry[],
-    structuredResults: MemoryEntry[]
-  ): MemoryEntry[] {
-    const semanticIds = new Set(semanticResults.map((e) => e.id));
-    const additional = structuredResults.filter((e) => !semanticIds.has(e.id));
-    return [...semanticResults, ...additional];
-  }
-
-  /**
-   * Structured-first: Prefer structured results, add semantic if not present
-   */
-  private combineStructuredFirst(
-    semanticResults: MemoryEntry[],
-    structuredResults: MemoryEntry[]
-  ): MemoryEntry[] {
-    const structuredIds = new Set(structuredResults.map((e) => e.id));
-    const additional = semanticResults.filter((e) => !structuredIds.has(e.id));
-    return [...structuredResults, ...additional];
-  }
-
-  /**
-   * Get underlying backends for advanced operations
-   */
-  getSQLiteBackend(): SQLiteBackend {
-    return this.sqlite;
-  }
-
-  getAgentDBBackend(): AgentDBBackend {
-    return this.agentdb;
-  }
-}
-
-export default HybridBackend;