@retrivora-ai/rag-engine 0.1.6 → 0.1.8

package/src/core/ConfigValidator.ts

@@ -3,6 +3,10 @@
  *
  * Validates RagConfig against provider requirements and best practices.
  * Returns detailed errors for misconfiguration.
+ *
+ * Key invariant: option keys validated here MUST match the keys each
+ * Provider constructor reads. See each provider's constructor for the
+ * canonical option names.
  */
 
 import { RagConfig, VectorDBConfig, LLMConfig, EmbeddingConfig } from '../config/RagConfig';
@@ -22,7 +26,6 @@ export class ConfigValidator {
   static validate(config: RagConfig): ValidationError[] {
     const errors: ValidationError[] = [];
 
-    // Validate root level
     if (!config.projectId) {
       errors.push({
         field: 'projectId',
@@ -31,31 +34,24 @@ export class ConfigValidator {
       });
     }
 
-    // Validate Vector DB configuration
     errors.push(...this.validateVectorDbConfig(config.vectorDb));
-
-    // Validate LLM configuration
     errors.push(...this.validateLLMConfig(config.llm));
 
-    // Validate Embedding configuration (if provided)
     if (config.embedding) {
       errors.push(...this.validateEmbeddingConfig(config.embedding));
     } else if (config.llm.provider === 'anthropic') {
-      // Anthropic should have a separate embedding provider
       errors.push({
         field: 'embedding',
         message: 'Embedding config is required when using Anthropic LLM',
-        suggestion: 'Provide embedding config with provider (e.g., "openai")',
+        suggestion: 'Provide embedding config with provider (e.g., "openai" or "ollama")',
         severity: 'error',
       });
     }
 
-    // Validate UI config if provided
     if (config.ui) {
       errors.push(...this.validateUIConfig(config.ui as Record<string, unknown>));
     }
 
-    // Validate RAG tuning if provided
     if (config.rag) {
       errors.push(...this.validateRAGConfig(config.rag as Record<string, unknown>));
     }
@@ -83,7 +79,6 @@ export class ConfigValidator {
       });
     }
 
-    // Provider-specific validation
     switch (config.provider) {
       case 'pinecone':
         errors.push(...this.validatePineconeConfig(config));
@@ -119,6 +114,10 @@ export class ConfigValidator {
     return errors;
   }
 
+  /**
+   * Pinecone — only needs apiKey (environment was removed in SDK v7+).
+   * Options key: { apiKey: string }
+   */
   private static validatePineconeConfig(config: VectorDBConfig): ValidationError[] {
     const errors: ValidationError[] = [];
     const opts = config.options as Record<string, unknown>;
@@ -132,18 +131,13 @@ export class ConfigValidator {
       });
     }
 
-    if (!opts.environment) {
-      errors.push({
-        field: 'vectorDb.options.environment',
-        message: 'Pinecone environment is required',
-        suggestion: 'Set PINECONE_ENVIRONMENT environment variable (e.g., "gcp-starter")',
-        severity: 'error',
-      });
-    }
-
     return errors;
   }
 
+  /**
+   * PostgreSQL / pgvector — needs a connection string.
+   * Options key: { connectionString: string }
+   */
   private static validatePostgresConfig(config: VectorDBConfig): ValidationError[] {
     const errors: ValidationError[] = [];
     const opts = config.options as Record<string, unknown>;
@@ -160,6 +154,10 @@ export class ConfigValidator {
     return errors;
   }
 
+  /**
+   * MongoDB — needs uri, database, and collection.
+   * Options keys: { uri: string, database: string, collection: string }
+   */
   private static validateMongoDBConfig(config: VectorDBConfig): ValidationError[] {
     const errors: ValidationError[] = [];
     const opts = config.options as Record<string, unknown>;
@@ -194,24 +192,23 @@ export class ConfigValidator {
     return errors;
   }
 
+  /**
+   * Milvus — accepts baseUrl OR uri (preferred) OR host+port.
+   * MilvusProvider reads opts.baseUrl ?? opts.uri.
+   * Options keys: { baseUrl?: string, uri?: string, host?: string, port?: number }
+   */
   private static validateMilvusConfig(config: VectorDBConfig): ValidationError[] {
     const errors: ValidationError[] = [];
     const opts = config.options as Record<string, unknown>;
 
-    if (!opts.host) {
-      errors.push({
-        field: 'vectorDb.options.host',
-        message: 'Milvus host is required',
-        suggestion: 'Set MILVUS_HOST environment variable',
-        severity: 'error',
-      });
-    }
+    const hasBaseUrl = Boolean(opts.baseUrl || opts.uri);
+    const hasHostPort = Boolean(opts.host && opts.port);
 
-    if (!opts.port) {
+    if (!hasBaseUrl && !hasHostPort) {
       errors.push({
-        field: 'vectorDb.options.port',
-        message: 'Milvus port is required',
-        suggestion: 'Set MILVUS_PORT environment variable (default: 19530)',
+        field: 'vectorDb.options.baseUrl',
+        message: 'Milvus connection is required: provide baseUrl (e.g., "http://localhost:19530") or host + port',
+        suggestion: 'Set MILVUS_URL environment variable',
         severity: 'error',
       });
     }
@@ -219,6 +216,10 @@ export class ConfigValidator {
     return errors;
   }
 
+  /**
+   * Qdrant — needs baseUrl (or url alias).
+   * Options keys: { baseUrl: string, apiKey?: string }
+   */
   private static validateQdrantConfig(config: VectorDBConfig): ValidationError[] {
     const errors: ValidationError[] = [];
     const opts = config.options as Record<string, unknown>;
@@ -235,15 +236,20 @@ export class ConfigValidator {
     return errors;
   }
 
+  /**
+   * ChromaDB — accepts baseUrl OR host.
+   * ChromaDBProvider reads opts.baseUrl.
+   * Options keys: { baseUrl?: string, host?: string, port?: number }
+   */
   private static validateChromaDBConfig(config: VectorDBConfig): ValidationError[] {
     const errors: ValidationError[] = [];
     const opts = config.options as Record<string, unknown>;
 
-    if (!opts.host && !opts.path) {
+    if (!opts.baseUrl && !opts.host) {
       errors.push({
-        field: 'vectorDb.options.host',
-        message: 'ChromaDB host or path is required',
-        suggestion: 'Set CHROMADB_HOST or CHROMADB_PATH environment variable',
+        field: 'vectorDb.options.baseUrl',
+        message: 'ChromaDB connection is required: provide baseUrl (e.g., "http://localhost:8000") or host',
+        suggestion: 'Set CHROMADB_URL environment variable',
         severity: 'error',
       });
     }
@@ -251,15 +257,23 @@ export class ConfigValidator {
     return errors;
   }
 
+  /**
+   * Redis — accepts baseUrl OR url OR host+port.
+   * RedisProvider reads opts.baseUrl.
+   * Options keys: { baseUrl?: string, url?: string, host?: string, port?: number, apiKey?: string }
+   */
   private static validateRedisConfig(config: VectorDBConfig): ValidationError[] {
     const errors: ValidationError[] = [];
     const opts = config.options as Record<string, unknown>;
 
-    if (!opts.url && (!opts.host || !opts.port)) {
+    const hasUrl = Boolean(opts.baseUrl || opts.url);
+    const hasHostPort = Boolean(opts.host && opts.port);
+
+    if (!hasUrl && !hasHostPort) {
       errors.push({
-        field: 'vectorDb.options.url',
-        message: 'Redis connection URL or host:port is required',
-        suggestion: 'Set REDIS_URL or REDIS_HOST/REDIS_PORT environment variables',
+        field: 'vectorDb.options.baseUrl',
+        message: 'Redis connection is required: provide baseUrl/url or host + port',
+        suggestion: 'Set REDIS_URL environment variable',
         severity: 'error',
       });
     }
@@ -267,13 +281,18 @@ export class ConfigValidator {
     return errors;
   }
 
+  /**
+   * Weaviate — accepts baseUrl OR url.
+   * WeaviateProvider reads opts.baseUrl.
+   * Options keys: { baseUrl?: string, url?: string, apiKey?: string }
+   */
   private static validateWeaviateConfig(config: VectorDBConfig): ValidationError[] {
     const errors: ValidationError[] = [];
     const opts = config.options as Record<string, unknown>;
 
-    if (!opts.url) {
+    if (!opts.baseUrl && !opts.url) {
       errors.push({
-        field: 'vectorDb.options.url',
+        field: 'vectorDb.options.baseUrl',
         message: 'Weaviate instance URL is required',
         suggestion: 'Set WEAVIATE_URL environment variable',
         severity: 'error',
@@ -283,6 +302,10 @@ export class ConfigValidator {
     return errors;
   }
 
+  /**
+   * Universal REST / custom REST adapter.
+   * Options key: { baseUrl: string }
+   */
   private static validateRestConfig(config: VectorDBConfig): ValidationError[] {
     const errors: ValidationError[] = [];
     const opts = config.options as Record<string, unknown>;
@@ -320,7 +343,6 @@ export class ConfigValidator {
       });
     }
 
-    // Provider-specific validation
     switch (config.provider) {
       case 'openai':
         if (!config.apiKey) {
@@ -365,7 +387,6 @@ export class ConfigValidator {
         break;
     }
 
-    // Validate temperature
     if (config.temperature !== undefined) {
       if (config.temperature < 0 || config.temperature > 2) {
         errors.push({
@@ -376,7 +397,6 @@ export class ConfigValidator {
       }
     }
 
-    // Validate maxTokens
     if (config.maxTokens !== undefined && config.maxTokens <= 0) {
       errors.push({
         field: 'llm.maxTokens',
@@ -404,12 +424,11 @@ export class ConfigValidator {
       errors.push({
         field: 'embedding.model',
         message: 'Embedding model name is required',
-        suggestion: 'e.g., "text-embedding-3-small" for OpenAI',
+        suggestion: 'e.g., "text-embedding-3-small" for OpenAI, "nomic-embed-text" for Ollama',
         severity: 'error',
       });
     }
 
-    // Provider-specific validation
     if (config.provider === 'openai' && !config.apiKey) {
       errors.push({
         field: 'embedding.apiKey',
@@ -422,13 +441,12 @@ export class ConfigValidator {
     if (config.provider === 'ollama' && !config.baseUrl) {
       errors.push({
         field: 'embedding.baseUrl',
-        message: 'Ollama base URL is required',
+        message: 'Ollama base URL is required for embedding',
         suggestion: 'Set EMBEDDING_BASE_URL environment variable',
         severity: 'error',
       });
     }
 
-    // Validate dimensions
     if (config.dimensions !== undefined && config.dimensions <= 0) {
       errors.push({
         field: 'embedding.dimensions',
@@ -526,10 +544,17 @@ export class ConfigValidator {
   }
 
   private static isValidCSSColor(color: string): boolean {
-    // Check if valid CSS color
-    const s = new Option().style;
-    s.color = color;
-    return s.color !== '';
+    const hexRegex = /^#([0-9a-f]{3}){1,2}$/i;
+    const rgbRegex = /^rgba?\((\d+),\s*(\d+),\s*(\d+)(?:,\s*(\d+(?:\.\d+)?))?\)$/i;
+    const hslRegex = /^hsla?\((\d+),\s*(\d+)%,\s*(\d+)%(?:,\s*(\d+(?:\.\d+)?))?\)$/i;
+    const namedColors = ['red', 'blue', 'green', 'black', 'white', 'transparent'];
+
+    return (
+      hexRegex.test(color) ||
+      rgbRegex.test(color) ||
+      hslRegex.test(color) ||
+      namedColors.includes(color.toLowerCase())
+    );
   }
 
   /**

package/src/core/Pipeline.ts

@@ -4,8 +4,7 @@ import { DocumentChunker } from '../rag/DocumentChunker';
 import { RagConfig } from '../config/RagConfig';
 import { ProviderRegistry } from './ProviderRegistry';
 import { BatchProcessor, BatchOptions } from './BatchProcessor';
-
-// Moved to src/types/index.ts
+import { EmbeddingStrategyResolver } from '../config/EmbeddingStrategy';
 import { IngestDocument, ChatResponse } from '../types';
 
 /**
@@ -14,9 +13,9 @@ import { IngestDocument, ChatResponse } from '../types';
  * Features:
  * - Lazy initialization of providers
  * - Batch processing with retry logic
- * - Configurable embedding strategies
+ * - Smart embedding strategy (integrated / separate / external)
  * - Error recovery for transient failures
- * - Support for multi-tenancy via namespacing
+ * - Multi-tenancy support via namespacing
  */
 export class Pipeline {
   private vectorDB!: BaseVectorProvider;
@@ -37,26 +36,32 @@ export class Pipeline {
   async initialize(): Promise<void> {
     if (this.initialised) return;
 
+    // Resolve vector DB provider
     this.vectorDB = await ProviderRegistry.createVectorProvider(this.config.vectorDb);
-    this.llmProvider = ProviderRegistry.createLLMProvider(this.config.llm, this.config.embedding);
-
-    if (this.config.llm.provider === 'anthropic') {
-      // Anthropic doesn't support embeddings, so use separate provider
-      const { LLMFactory } = await import('../llm/LLMFactory');
-      this.embeddingProvider = LLMFactory.createEmbeddingProvider(this.config.embedding);
-    } else {
-      this.embeddingProvider = this.llmProvider;
-    }
+
+    // Resolve LLM + embedding providers using EmbeddingStrategyResolver.
+    // This handles all cases: integrated (same provider for chat+embed),
+    // separate (e.g. Anthropic + OpenAI embedding), and external.
+    const { llmProvider, embeddingProvider } = await EmbeddingStrategyResolver.resolve(
+      this.config.llm,
+      this.config.embedding
+    );
+
+    this.llmProvider = llmProvider;
+    this.embeddingProvider = embeddingProvider;
 
     await this.vectorDB.initialize();
     this.initialised = true;
   }
 
   /**
-   * Ingest documents with automatic chunking, embedding, and batch processing.
+   * Ingest documents with automatic chunking, embedding, and batch upsert.
    * Handles retries for transient failures.
    */
-  async ingest(documents: IngestDocument[], namespace?: string): Promise<Array<{ docId: string | number; chunksIngested: number }>> {
+  async ingest(
+    documents: IngestDocument[],
+    namespace?: string
+  ): Promise<Array<{ docId: string | number; chunksIngested: number }>> {
     await this.initialize();
     const ns = namespace ?? this.config.projectId;
     const results = [];
@@ -68,9 +73,9 @@ export class Pipeline {
           metadata: doc.metadata,
         });
 
-        // Batch embed chunks with retry logic
-        const batchOptions: BatchOptions = {
-          batchSize: 50, // Embedding batch size
+        // Batch embed with retry
+        const embedBatchOptions: BatchOptions = {
+          batchSize: 50,
           maxRetries: 3,
           initialDelayMs: 100,
         };
@@ -78,11 +83,11 @@ export class Pipeline {
         const vectors = await BatchProcessor.mapWithRetry(
           chunks.map(c => c.content),
           (text) => this.embeddingProvider.embed(text),
-          batchOptions
+          embedBatchOptions
         );
 
         if (vectors.length !== chunks.length) {
-          throw new Error(`Embedding failed: got ${vectors.length} vectors for ${chunks.length} chunks`);
+          throw new Error(`Embedding mismatch: got ${vectors.length} vectors for ${chunks.length} chunks`);
         }
 
         const upsertDocs = chunks.map((chunk, i) => ({
@@ -92,7 +97,7 @@ export class Pipeline {
           metadata: chunk.metadata,
         }));
 
-        // Batch upsert with retry logic
+        // Batch upsert with retry
         const upsertBatchOptions: BatchOptions = {
           batchSize: 100,
           maxRetries: 3,
@@ -115,10 +120,7 @@ export class Pipeline {
         });
       } catch (error) {
         console.error(`[Pipeline] Failed to ingest document ${doc.docId}:`, error);
-        results.push({
-          docId: doc.docId,
-          chunksIngested: 0,
-        });
+        results.push({ docId: doc.docId, chunksIngested: 0 });
       }
     }
 
@@ -134,7 +136,7 @@ export class Pipeline {
     try {
       const queryVector = await this.embeddingProvider.embed(question);
       const rawMatches = await this.vectorDB.query(queryVector, topK, ns);
-      
+
       const sources = rawMatches.filter((m) => m.score >= scoreThreshold);
       const context = sources.length
         ? sources.map((m, i) => `[Source ${i + 1}]\n${m.content}`).join('\n\n---\n\n')

package/src/core/PluginManager.ts

@@ -0,0 +1,277 @@
+/**
+ * PluginManager — Universal plugin registration and lifecycle management
+ *
+ * Provides a flexible architecture for registering custom vector DBs, LLMs,
+ * and other providers without modifying core code.
+ *
+ * Features:
+ * - Dynamic plugin registration
+ * - Dependency resolution
+ * - Plugin validation
+ * - Lifecycle management
+ */
+
+export interface PluginMetadata {
+  /** Unique identifier (e.g., 'myprovider.vectordb.custom') */
+  id: string;
+  /** Human-readable name */
+  name: string;
+  /** Plugin version */
+  version: string;
+  /** Detailed description */
+  description?: string;
+  /** Author information */
+  author?: string;
+  /** Supported capabilities */
+  capabilities: PluginCapability[];
+  /** Required dependencies (other plugin IDs) */
+  dependencies?: string[];
+}
+
+export type PluginCapability = 'chat' | 'embed' | 'vector-db' | 'reranker' | 'parser';
+
+export interface IPluginProvider {
+  /**
+   * Get plugin metadata
+   */
+  getMetadata(): PluginMetadata;
+
+  /**
+   * Validate configuration for this plugin
+   */
+  validateConfig?(config: Record<string, unknown>): {
+    valid: boolean;
+    errors?: string[];
+  };
+
+  /**
+   * Initialize plugin with configuration
+   */
+  initialize(config: Record<string, unknown>): Promise<void>;
+
+  /**
+   * Check if plugin is ready
+   */
+  isReady(): Promise<boolean>;
+
+  /**
+   * Gracefully shutdown plugin
+   */
+  shutdown(): Promise<void>;
+
+  /**
+   * Get plugin-specific health status
+   */
+  getHealthStatus?(): Promise<{
+    healthy: boolean;
+    details?: Record<string, unknown>;
+  }>;
+}
+
+export interface PluginDescriptor {
+  metadata: PluginMetadata;
+  factory: (config: Record<string, unknown>) => Promise<IPluginProvider>;
+  validateConfig?: (config: Record<string, unknown>) => {
+    valid: boolean;
+    errors?: string[];
+  };
+}
+
+export class PluginManager {
+  private static readonly registeredPlugins: Map<string, PluginDescriptor> = new Map();
+  private static readonly instances: Map<string, IPluginProvider> = new Map();
+
+  /**
+   * Register a new plugin provider
+   */
+  static registerPlugin(descriptor: PluginDescriptor): void {
+    const { metadata } = descriptor;
+
+    if (this.registeredPlugins.has(metadata.id)) {
+      throw new Error(
+        `[PluginManager] Plugin with ID "${metadata.id}" is already registered. ` +
+        `Use unregisterPlugin() first if you need to replace it.`
+      );
+    }
+
+    // Validate metadata
+    if (!metadata.id || !metadata.name || !metadata.version) {
+      throw new Error(
+        '[PluginManager] Plugin metadata must include id, name, and version'
+      );
+    }
+
+    if (metadata.capabilities.length === 0) {
+      throw new Error('[PluginManager] Plugin must declare at least one capability');
+    }
+
+    this.registeredPlugins.set(metadata.id, descriptor);
+    console.log(
+      `[PluginManager] Registered plugin: ${metadata.name} v${metadata.version} (${metadata.id})`
+    );
+  }
+
+  /**
+   * Unregister a plugin and cleanup its instance
+   */
+  static async unregisterPlugin(pluginId: string): Promise<void> {
+    if (this.instances.has(pluginId)) {
+      const instance = this.instances.get(pluginId)!;
+      await instance.shutdown();
+      this.instances.delete(pluginId);
+    }
+
+    this.registeredPlugins.delete(pluginId);
+  }
+
+  /**
+   * Get plugin metadata
+   */
+  static getPluginMetadata(pluginId: string): PluginMetadata | null {
+    return this.registeredPlugins.get(pluginId)?.metadata ?? null;
+  }
+
+  /**
+   * List all registered plugins with optional filtering by capability
+   */
+  static listPlugins(capability?: PluginCapability): PluginMetadata[] {
+    const plugins = Array.from(this.registeredPlugins.values()).map(d => d.metadata);
+
+    if (capability) {
+      return plugins.filter(p => p.capabilities.includes(capability));
+    }
+
+    return plugins;
+  }
+
+  /**
+   * Create and cache a plugin instance with dependency resolution
+   */
+  static async createInstance(
+    pluginId: string,
+    config: Record<string, unknown>
+  ): Promise<IPluginProvider> {
+    // Return cached instance if already created
+    if (this.instances.has(pluginId)) {
+      return this.instances.get(pluginId)!;
+    }
+
+    const descriptor = this.registeredPlugins.get(pluginId);
+    if (!descriptor) {
+      throw new Error(
+        `[PluginManager] Plugin "${pluginId}" not registered. ` +
+        `Registered plugins: ${Array.from(this.registeredPlugins.keys()).join(', ')}`
+      );
+    }
+
+    const { metadata, factory } = descriptor;
+
+    // Validate configuration
+    if (descriptor.validateConfig) {
+      const validation = descriptor.validateConfig(config);
+      if (!validation.valid) {
+        throw new Error(
+          `[PluginManager] Invalid configuration for plugin "${pluginId}": ` +
+          `${validation.errors?.join('; ') || 'Unknown error'}`
+        );
+      }
+    }
+
+    // Resolve dependencies
+    if (metadata.dependencies && metadata.dependencies.length > 0) {
+      for (const depId of metadata.dependencies) {
+        if (!this.instances.has(depId)) {
+          throw new Error(
+            `[PluginManager] Plugin "${pluginId}" requires "${depId}" to be initialized first`
+          );
+        }
+      }
+    }
+
+    // Initialize plugin
+    console.log(`[PluginManager] Initializing plugin: ${pluginId}`);
+    const instance = await factory(config);
+
+    // Verify plugin is ready
+    const isReady = await instance.isReady();
+    if (!isReady) {
+      throw new Error(
+        `[PluginManager] Plugin "${pluginId}" failed to initialize properly`
+      );
+    }
+
+    // Cache instance
+    this.instances.set(pluginId, instance);
+
+    return instance;
+  }
+
+  /**
+   * Get cached instance of a plugin
+   */
+  static getInstance(pluginId: string): IPluginProvider | null {
+    return this.instances.get(pluginId) ?? null;
+  }
+
+  /**
+   * Check if a plugin is ready (either registered or instantiated)
+   */
+  static isPluginAvailable(pluginId: string): boolean {
+    return this.registeredPlugins.has(pluginId) || this.instances.has(pluginId);
+  }
+
+  /**
+   * Shutdown all plugin instances
+   */
+  static async shutdownAll(): Promise<void> {
+    const errors: Error[] = [];
+
+    for (const [pluginId, instance] of this.instances.entries()) {
+      try {
+        await instance.shutdown();
+        console.log(`[PluginManager] Shutdown plugin: ${pluginId}`);
+      } catch (error) {
+        errors.push(
+          new Error(`Failed to shutdown plugin ${pluginId}: ${error instanceof Error ? error.message : String(error)}`)
+        );
+      }
+    }
+
+    this.instances.clear();
+
+    if (errors.length > 0) {
+      throw new AggregateError(errors, '[PluginManager] Errors during shutdown');
+    }
+  }
+
+  /**
+   * Get health status of all instantiated plugins
+   */
+  static async getHealthStatus(): Promise<
+    Map<string, { healthy: boolean; details?: Record<string, unknown> }>
+  > {
+    const status = new Map();
+
+    for (const [pluginId, instance] of this.instances.entries()) {
+      try {
+        const health = await instance.getHealthStatus?.();
+        status.set(pluginId, health ?? { healthy: true });
+      } catch (error) {
+        status.set(pluginId, {
+          healthy: false,
+          details: { error: error instanceof Error ? error.message : String(error) },
+        });
+      }
+    }
+
+    return status;
+  }
+
+  /**
+   * Clear all registered and instantiated plugins (for testing)
+   */
+  static reset(): void {
+    this.registeredPlugins.clear();
+    this.instances.clear();
+  }
+}