@sparkleideas/embeddings 3.0.0-alpha.17 → 3.0.0-alpha.27

package/src/embedding-service.ts

@@ -19,6 +19,7 @@ import type {
   OpenAIEmbeddingConfig,
   TransformersEmbeddingConfig,
   MockEmbeddingConfig,
+  AgenticFlowEmbeddingConfig,
   EmbeddingResult,
   BatchEmbeddingResult,
   IEmbeddingService,
@@ -26,7 +27,11 @@ import type {
   EmbeddingEventListener,
   SimilarityMetric,
   SimilarityResult,
+  NormalizationType,
+  PersistentCacheConfig,
 } from './types.js';
+import { normalize } from './normalization.js';
+import { PersistentEmbeddingCache } from './persistent-cache.js';
 
 // ============================================================================
 // LRU Cache Implementation
@@ -98,16 +103,55 @@ class LRUCache<K, V> {
 abstract class BaseEmbeddingService extends EventEmitter implements IEmbeddingService {
   abstract readonly provider: EmbeddingProvider;
   protected cache: LRUCache<string, Float32Array>;
+  protected persistentCache: PersistentEmbeddingCache | null = null;
   protected embeddingListeners: Set<EmbeddingEventListener> = new Set();
+  protected normalizationType: NormalizationType;
 
   constructor(protected readonly config: EmbeddingConfig) {
     super();
     this.cache = new LRUCache(config.cacheSize ?? 1000);
+    this.normalizationType = config.normalization ?? 'none';
+
+    // Initialize persistent cache if configured
+    if (config.persistentCache?.enabled) {
+      const pcConfig: PersistentCacheConfig = config.persistentCache;
+      this.persistentCache = new PersistentEmbeddingCache({
+        dbPath: pcConfig.dbPath ?? '.cache/embeddings.db',
+        maxSize: pcConfig.maxSize ?? 10000,
+        ttlMs: pcConfig.ttlMs,
+      });
+    }
   }
 
   abstract embed(text: string): Promise<EmbeddingResult>;
   abstract embedBatch(texts: string[]): Promise<BatchEmbeddingResult>;
 
+  /**
+   * Apply normalization to embedding if configured
+   */
+  protected applyNormalization(embedding: Float32Array): Float32Array {
+    if (this.normalizationType === 'none') {
+      return embedding;
+    }
+    return normalize(embedding, { type: this.normalizationType });
+  }
+
+  /**
+   * Check persistent cache for embedding
+   */
+  protected async checkPersistentCache(text: string): Promise<Float32Array | null> {
+    if (!this.persistentCache) return null;
+    return this.persistentCache.get(text);
+  }
+
+  /**
+   * Store embedding in persistent cache
+   */
+  protected async storePersistentCache(text: string, embedding: Float32Array): Promise<void> {
+    if (!this.persistentCache) return;
+    await this.persistentCache.set(text, embedding);
+  }
+
   protected emitEvent(event: EmbeddingEvent): void {
     for (const listener of this.embeddingListeners) {
       try {
@@ -431,10 +475,17 @@ export class MockEmbeddingService extends BaseEmbeddingService {
   private readonly dimensions: number;
   private readonly simulatedLatency: number;
 
-  constructor(config: MockEmbeddingConfig) {
-    super(config);
-    this.dimensions = config.dimensions ?? 384;
-    this.simulatedLatency = config.simulatedLatency ?? 0;
+  constructor(config: Partial<MockEmbeddingConfig> = {}) {
+    const fullConfig: MockEmbeddingConfig = {
+      provider: 'mock',
+      dimensions: config.dimensions ?? 384,
+      cacheSize: config.cacheSize ?? 1000,
+      simulatedLatency: config.simulatedLatency ?? 0,
+      enableCache: config.enableCache ?? true,
+    };
+    super(fullConfig);
+    this.dimensions = fullConfig.dimensions!;
+    this.simulatedLatency = fullConfig.simulatedLatency!;
   }
 
   async embed(text: string): Promise<EmbeddingResult> {
@@ -532,12 +583,286 @@ export class MockEmbeddingService extends BaseEmbeddingService {
   }
 }
 
+// ============================================================================
+// Agentic-Flow Embedding Service
+// ============================================================================
+
+/**
+ * Agentic-Flow embedding service using OptimizedEmbedder
+ *
+ * Features:
+ * - ONNX-based embeddings with SIMD acceleration
+ * - 256-entry LRU cache with FNV-1a hash
+ * - 8x loop unrolling for cosine similarity
+ * - Pre-allocated buffers (no GC pressure)
+ * - 3-4x faster batch processing
+ */
+export class AgenticFlowEmbeddingService extends BaseEmbeddingService {
+  readonly provider: EmbeddingProvider = 'agentic-flow';
+  private embedder: any = null;
+  private initialized = false;
+  private readonly modelId: string;
+  private readonly dimensions: number;
+  private readonly embedderCacheSize: number;
+  private readonly modelDir: string | undefined;
+  private readonly autoDownload: boolean;
+
+  constructor(config: AgenticFlowEmbeddingConfig) {
+    super(config);
+    this.modelId = config.modelId ?? 'all-MiniLM-L6-v2';
+    this.dimensions = config.dimensions ?? 384;
+    this.embedderCacheSize = config.embedderCacheSize ?? 256;
+    this.modelDir = config.modelDir;
+    this.autoDownload = config.autoDownload ?? false;
+  }
+
+  private async initialize(): Promise<void> {
+    if (this.initialized) return;
+
+    let lastError: Error | undefined;
+
+    const createEmbedder = async (modulePath: string): Promise<boolean> => {
+      try {
+        // Use file:// protocol for absolute paths
+        const importPath = modulePath.startsWith('/') ? `file://${modulePath}` : modulePath;
+        const module = await import(/* webpackIgnore: true */ importPath);
+        const getOptimizedEmbedder = module.getOptimizedEmbedder || module.default?.getOptimizedEmbedder;
+        if (!getOptimizedEmbedder) {
+          lastError = new Error(`Module loaded but getOptimizedEmbedder not found`);
+          return false;
+        }
+
+        // Only include defined values to not override defaults
+        const embedderConfig: Record<string, unknown> = {
+          modelId: this.modelId,
+          dimension: this.dimensions,
+          cacheSize: this.embedderCacheSize,
+          autoDownload: this.autoDownload,
+        };
+        if (this.modelDir !== undefined) {
+          embedderConfig.modelDir = this.modelDir;
+        }
+        this.embedder = getOptimizedEmbedder(embedderConfig);
+        await this.embedder.init();
+        this.initialized = true;
+        return true;
+      } catch (error) {
+        lastError = error instanceof Error ? error : new Error(String(error));
+        return false;
+      }
+    };
+
+    // Build list of possible module paths to try
+    const possiblePaths: string[] = [];
+
+    // Try proper package exports first (preferred)
+    possiblePaths.push('agentic-flow/embeddings');
+
+    // Try node_modules resolution from different locations (for file:// imports)
+    try {
+      const path = await import('path');
+      const { existsSync } = await import('fs');
+      const cwd = process.cwd();
+
+      // Prioritize absolute paths that exist (for file:// import fallback)
+      const absolutePaths = [
+        path.join(cwd, 'node_modules/agentic-flow/dist/embeddings/optimized-embedder.js'),
+        path.join(cwd, '../node_modules/agentic-flow/dist/embeddings/optimized-embedder.js'),
+        '/workspaces/claude-flow/node_modules/agentic-flow/dist/embeddings/optimized-embedder.js',
+      ];
+
+      for (const p of absolutePaths) {
+        if (existsSync(p)) {
+          possiblePaths.push(p);
+        }
+      }
+    } catch {
+      // fs/path module not available
+    }
+
+    // Try each path
+    for (const modulePath of possiblePaths) {
+      if (await createEmbedder(modulePath)) {
+        return;
+      }
+    }
+
+    const errorDetail = lastError?.message ? ` Last error: ${lastError.message}` : '';
+    throw new Error(
+      `Failed to initialize agentic-flow embeddings.${errorDetail} ` +
+      `Ensure agentic-flow is installed and ONNX model is downloaded: ` +
+      `npx agentic-flow@alpha embeddings init`
+    );
+  }
+
+  async embed(text: string): Promise<EmbeddingResult> {
+    await this.initialize();
+
+    // Check our LRU cache first
+    const cached = this.cache.get(text);
+    if (cached) {
+      this.emitEvent({ type: 'cache_hit', text });
+      return {
+        embedding: cached,
+        latencyMs: 0,
+        cached: true,
+      };
+    }
+
+    this.emitEvent({ type: 'embed_start', text });
+    const startTime = performance.now();
+
+    try {
+      // Use agentic-flow's optimized embedder (has its own internal cache)
+      const embedding = await this.embedder.embed(text);
+
+      // Store in our cache as well
+      this.cache.set(text, embedding);
+
+      const latencyMs = performance.now() - startTime;
+      this.emitEvent({ type: 'embed_complete', text, latencyMs });
+
+      return {
+        embedding,
+        latencyMs,
+      };
+    } catch (error) {
+      const message = error instanceof Error ? error.message : 'Unknown error';
+      this.emitEvent({ type: 'embed_error', text, error: message });
+      throw new Error(`Agentic-flow embedding failed: ${message}`);
+    }
+  }
+
+  async embedBatch(texts: string[]): Promise<BatchEmbeddingResult> {
+    await this.initialize();
+
+    this.emitEvent({ type: 'batch_start', count: texts.length });
+    const startTime = performance.now();
+
+    // Check cache for each text
+    const cached: Array<{ index: number; embedding: Float32Array }> = [];
+    const uncached: Array<{ index: number; text: string }> = [];
+
+    texts.forEach((text, index) => {
+      const cachedEmbedding = this.cache.get(text);
+      if (cachedEmbedding) {
+        cached.push({ index, embedding: cachedEmbedding });
+        this.emitEvent({ type: 'cache_hit', text });
+      } else {
+        uncached.push({ index, text });
+      }
+    });
+
+    // Use optimized batch embedding for uncached texts
+    let batchEmbeddings: Float32Array[] = [];
+    if (uncached.length > 0) {
+      const uncachedTexts = uncached.map(u => u.text);
+      batchEmbeddings = await this.embedder.embedBatch(uncachedTexts);
+
+      // Cache results
+      uncached.forEach((item, i) => {
+        this.cache.set(item.text, batchEmbeddings[i]);
+      });
+    }
+
+    // Reconstruct result array in original order
+    const embeddings: Float32Array[] = new Array(texts.length);
+    cached.forEach(c => {
+      embeddings[c.index] = c.embedding;
+    });
+    uncached.forEach((u, i) => {
+      embeddings[u.index] = batchEmbeddings[i];
+    });
+
+    const totalLatencyMs = performance.now() - startTime;
+    this.emitEvent({ type: 'batch_complete', count: texts.length, latencyMs: totalLatencyMs });
+
+    return {
+      embeddings,
+      totalLatencyMs,
+      avgLatencyMs: totalLatencyMs / texts.length,
+      cacheStats: {
+        hits: cached.length,
+        misses: uncached.length,
+      },
+    };
+  }
+
+  /**
+   * Get combined cache statistics from both our LRU cache and embedder's internal cache
+   */
+  override getCacheStats() {
+    const baseStats = super.getCacheStats();
+
+    if (this.embedder && this.embedder.getCacheStats) {
+      const embedderStats = this.embedder.getCacheStats();
+      return {
+        size: baseStats.size + embedderStats.size,
+        maxSize: baseStats.maxSize + embedderStats.maxSize,
+        hitRate: baseStats.hitRate,
+        embedderCache: embedderStats,
+      };
+    }
+
+    return baseStats;
+  }
+
+  override async shutdown(): Promise<void> {
+    if (this.embedder && this.embedder.clearCache) {
+      this.embedder.clearCache();
+    }
+    await super.shutdown();
+  }
+}
+
 // ============================================================================
 // Factory Functions
 // ============================================================================
 
 /**
- * Create embedding service based on configuration
+ * Check if agentic-flow is available
+ */
+async function isAgenticFlowAvailable(): Promise<boolean> {
+  try {
+    await import('agentic-flow/embeddings');
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Auto-install agentic-flow and initialize model
+ */
+async function autoInstallAgenticFlow(): Promise<boolean> {
+  const { exec } = await import('child_process');
+  const { promisify } = await import('util');
+  const execAsync = promisify(exec);
+
+  try {
+    // Check if already available
+    if (await isAgenticFlowAvailable()) {
+      return true;
+    }
+
+    console.log('[embeddings] Installing agentic-flow@alpha...');
+    await execAsync('npm install agentic-flow@alpha --save', { timeout: 120000 });
+
+    // Initialize the model
+    console.log('[embeddings] Downloading embedding model...');
+    await execAsync('npx agentic-flow@alpha embeddings init', { timeout: 300000 });
+
+    // Verify installation
+    return await isAgenticFlowAvailable();
+  } catch (error) {
+    console.warn('[embeddings] Auto-install failed:', error instanceof Error ? error.message : error);
+    return false;
+  }
+}
+
+/**
+ * Create embedding service based on configuration (sync version)
+ * Note: For 'auto' provider or smart fallback, use createEmbeddingServiceAsync
  */
 export function createEmbeddingService(config: EmbeddingConfig): IEmbeddingService {
   switch (config.provider) {
@@ -547,12 +872,159 @@ export function createEmbeddingService(config: EmbeddingConfig): IEmbeddingServi
       return new TransformersEmbeddingService(config as TransformersEmbeddingConfig);
     case 'mock':
       return new MockEmbeddingService(config as MockEmbeddingConfig);
+    case 'agentic-flow':
+      return new AgenticFlowEmbeddingService(config as AgenticFlowEmbeddingConfig);
     default:
       console.warn(`Unknown provider, using mock`);
       return new MockEmbeddingService({ provider: 'mock', dimensions: 384 });
   }
 }
 
+/**
+ * Extended config with auto provider option
+ */
+export interface AutoEmbeddingConfig {
+  /** Provider: 'auto' will pick best available (agentic-flow > transformers > mock) */
+  provider: EmbeddingProvider | 'auto';
+  /** Fallback provider if primary fails */
+  fallback?: EmbeddingProvider;
+  /** Auto-install agentic-flow if not available (default: true for 'auto' provider) */
+  autoInstall?: boolean;
+  /** Model ID for agentic-flow */
+  modelId?: string;
+  /** Model name for transformers */
+  model?: string;
+  /** Dimensions */
+  dimensions?: number;
+  /** Cache size */
+  cacheSize?: number;
+  /** OpenAI API key (required for openai provider) */
+  apiKey?: string;
+}
+
+/**
+ * Create embedding service with automatic provider detection and fallback
+ *
+ * Features:
+ * - 'auto' provider picks best available: agentic-flow > transformers > mock
+ * - Automatic fallback if primary provider fails to initialize
+ * - Pre-validates provider availability before returning
+ *
+ * @example
+ * // Auto-select best provider
+ * const service = await createEmbeddingServiceAsync({ provider: 'auto' });
+ *
+ * // Try agentic-flow, fallback to transformers
+ * const service = await createEmbeddingServiceAsync({
+ *   provider: 'agentic-flow',
+ *   fallback: 'transformers'
+ * });
+ */
+export async function createEmbeddingServiceAsync(
+  config: AutoEmbeddingConfig
+): Promise<IEmbeddingService> {
+  const { provider, fallback, autoInstall = true, ...rest } = config;
+
+  // Auto provider selection
+  if (provider === 'auto') {
+    // Try agentic-flow first (fastest, ONNX-based)
+    let agenticFlowAvailable = await isAgenticFlowAvailable();
+
+    // Auto-install if not available and autoInstall is enabled
+    if (!agenticFlowAvailable && autoInstall) {
+      agenticFlowAvailable = await autoInstallAgenticFlow();
+    }
+
+    if (agenticFlowAvailable) {
+      try {
+        const service = new AgenticFlowEmbeddingService({
+          provider: 'agentic-flow',
+          modelId: rest.modelId ?? 'all-MiniLM-L6-v2',
+          dimensions: rest.dimensions ?? 384,
+          cacheSize: rest.cacheSize,
+        });
+        // Validate it can initialize
+        await service.embed('test');
+        return service;
+      } catch {
+        // Fall through to next option
+      }
+    }
+
+    // Try transformers (good quality, built-in)
+    try {
+      const service = new TransformersEmbeddingService({
+        provider: 'transformers',
+        model: rest.model ?? 'Xenova/all-MiniLM-L6-v2',
+        cacheSize: rest.cacheSize,
+      });
+      // Validate it can initialize
+      await service.embed('test');
+      return service;
+    } catch {
+      // Fall through to mock
+    }
+
+    // Fallback to mock (always works)
+    console.warn('[embeddings] Using mock provider - install agentic-flow or @xenova/transformers for real embeddings');
+    return new MockEmbeddingService({
+      dimensions: rest.dimensions ?? 384,
+      cacheSize: rest.cacheSize,
+    });
+  }
+
+  // Specific provider with optional fallback
+  const createPrimary = (): IEmbeddingService => {
+    switch (provider) {
+      case 'agentic-flow':
+        return new AgenticFlowEmbeddingService({
+          provider: 'agentic-flow',
+          modelId: rest.modelId ?? 'all-MiniLM-L6-v2',
+          dimensions: rest.dimensions ?? 384,
+          cacheSize: rest.cacheSize,
+        });
+      case 'transformers':
+        return new TransformersEmbeddingService({
+          provider: 'transformers',
+          model: rest.model ?? 'Xenova/all-MiniLM-L6-v2',
+          cacheSize: rest.cacheSize,
+        });
+      case 'openai':
+        if (!rest.apiKey) throw new Error('OpenAI provider requires apiKey');
+        return new OpenAIEmbeddingService({
+          provider: 'openai',
+          apiKey: rest.apiKey,
+          dimensions: rest.dimensions,
+          cacheSize: rest.cacheSize,
+        });
+      case 'mock':
+        return new MockEmbeddingService({
+          dimensions: rest.dimensions ?? 384,
+          cacheSize: rest.cacheSize,
+        });
+      default:
+        throw new Error(`Unknown provider: ${provider}`);
+    }
+  };
+
+  const primary = createPrimary();
+
+  // Try to validate primary provider
+  try {
+    await primary.embed('test');
+    return primary;
+  } catch (error) {
+    if (!fallback) {
+      throw error;
+    }
+
+    // Try fallback
+    console.warn(`[embeddings] Primary provider '${provider}' failed, using fallback '${fallback}'`);
+    const fallbackConfig: AutoEmbeddingConfig = { ...rest, provider: fallback };
+    return createEmbeddingServiceAsync(fallbackConfig);
+  }
+}
+
 /**
  * Convenience function for quick embeddings
  */