@retrivora-ai/rag-engine 0.4.5 → 1.0.0

package/src/llm/providers/OllamaProvider.ts

@@ -1,18 +1,12 @@
 /**
  * Ollama LLM Provider (local / self-hosted)
- *
- * Communicates with an Ollama server via its REST API.
- * Supports both chat completion and embedding (requires a model with embed support,
- * e.g. "nomic-embed-text", "mxbai-embed-large").
- *
- * Required LLMConfig fields:
- *   - model:    string  – e.g. "llama3", "mistral", "gemma2"
- *   - baseUrl?: string  – Ollama server URL (default "http://localhost:11434")
  */
 
 import axios, { AxiosInstance } from 'axios';
 import { ILLMProvider, ChatMessage, ChatOptions, EmbedOptions } from '../ILLMProvider';
 import { LLMConfig, EmbeddingConfig } from '../../config/RagConfig';
+import { IProviderValidator, IProviderHealthChecker, HealthCheckResult } from '../../core/ProviderInterfaces';
+import { ValidationError } from '../../core/ConfigValidator';
 
 interface OllamaChatResponse {
   message: { content: string };
@@ -34,14 +28,63 @@ export class OllamaProvider implements ILLMProvider {
     this.embeddingConfig = embeddingConfig;
   }
 
-  async chat(messages: ChatMessage[], context: string, options?: ChatOptions): Promise<string> {
-    const systemPrompt =
-      this.llmConfig.systemPrompt ??
-      `You are a helpful assistant. Use the provided context to answer the user's question.\n\nContext:\n${context}`;
+  static getValidator(): IProviderValidator {
+    return {
+      validate(config: Record<string, unknown>): ValidationError[] {
+        const errors: ValidationError[] = [];
+        if (!config.model) {
+          const isEmbedding = config.provider === 'ollama' && 'dimensions' in config;
+          const prefix = isEmbedding ? 'embedding' : 'llm';
+          errors.push({
+            field: `${prefix}.model`,
+            message: 'Ollama model name is required',
+            suggestion: 'e.g., "llama3" or "mistral"',
+            severity: 'error',
+          });
+        }
+        return errors;
+      }
+    };
+  }
 
-    const system = systemPrompt.includes('{{context}}')
-      ? systemPrompt.replace('{{context}}', context)
-      : `${systemPrompt}\n\nContext:\n${context}`;
+  static getHealthChecker(): IProviderHealthChecker {
+    return {
+      async check(config: Record<string, unknown>): Promise<HealthCheckResult> {
+        const timestamp = Date.now();
+        const baseUrl = (config.baseUrl as string) || 'http://localhost:11434';
+        const modelName = config.model as string;
+        
+        try {
+          const axios = (await import('axios')).default;
+          const { data } = await axios.get(`${baseUrl}/api/tags`);
+          const models = (data.models as { name: string }[]) || [];
+          const hasModel = models.some((m) => m.name === modelName || m.name.startsWith(`${modelName}:`));
+          
+          return {
+            healthy: true,
+            provider: 'ollama',
+            capabilities: { 
+              baseUrl, 
+              model: modelName, 
+              available: hasModel,
+              totalModels: models.length 
+            },
+            timestamp,
+          };
+        } catch {
+          return {
+            healthy: false,
+            provider: 'ollama',
+            error: `Ollama server not reachable at ${baseUrl}. Is it running?`,
+            timestamp,
+          };
+        }
+      }
+    };
+  }
+
+  async chat(messages: ChatMessage[], context: string, options?: ChatOptions): Promise<string> {
+    const system = this.buildSystemPrompt(context);
 
     const { data } = await this.http.post<OllamaChatResponse>('/api/chat', {
       model: this.llmConfig.model,
@@ -59,6 +102,48 @@ export class OllamaProvider implements ILLMProvider {
     return data.message.content;
   }
 
+  async *chatStream(messages: ChatMessage[], context: string, options?: ChatOptions): AsyncIterable<string> {
+    const system = this.buildSystemPrompt(context);
+
+    const response = await this.http.post('/api/chat', {
+      model: this.llmConfig.model,
+      stream: true,
+      options: {
+        temperature: options?.temperature ?? this.llmConfig.temperature ?? 0.7,
+        num_predict: options?.maxTokens ?? this.llmConfig.maxTokens ?? 1024,
+      },
+      messages: [
+        { role: 'system', content: system },
+        ...messages.map((m) => ({ role: m.role, content: m.content })),
+      ],
+    }, { responseType: 'stream' });
+
+    for await (const chunk of response.data) {
+      const lines = chunk.toString().split('\n').filter(Boolean);
+      for (const line of lines) {
+        try {
+          const json = JSON.parse(line);
+          if (json.message?.content) {
+            yield json.message.content;
+          }
+          if (json.done) return;
+        } catch {
+          // Ignore parsing errors for partial lines
+        }
+      }
+    }
+  }
+
+  private buildSystemPrompt(context: string): string {
+    const systemPrompt =
+      this.llmConfig.systemPrompt ??
+      `You are a helpful assistant. Use the provided context to answer the user's question.\n\nContext:\n${context}`;
+
+    return systemPrompt.includes('{{context}}')
+      ? systemPrompt.replace('{{context}}', context)
+      : `${systemPrompt}\n\nContext:\n${context}`;
+  }
+
   async embed(text: string, options?: EmbedOptions): Promise<number[]> {
     const model =
       options?.model ??

package/src/llm/providers/OpenAIProvider.ts

@@ -1,21 +1,12 @@
 /**
  * OpenAI LLM Provider
- *
- * Handles both chat completion (GPT-4o, GPT-4-turbo, etc.) and
- * embedding (text-embedding-ada-002, text-embedding-3-small/large).
- *
- * Required LLMConfig fields:
- *   - apiKey: string  – OpenAI API key
- *   - model:  string  – e.g. "gpt-4o"
- *
- * Required EmbeddingConfig fields (when used for embedding):
- *   - apiKey: string  – same or separate key
- *   - model:  string  – e.g. "text-embedding-3-small"
  */
 
 import OpenAI from 'openai';
 import { ILLMProvider, ChatMessage, ChatOptions, EmbedOptions } from '../ILLMProvider';
 import { LLMConfig, EmbeddingConfig } from '../../config/RagConfig';
+import { IProviderValidator, IProviderHealthChecker, HealthCheckResult } from '../../core/ProviderInterfaces';
+import { ValidationError } from '../../core/ConfigValidator';
 
 export class OpenAIProvider implements ILLMProvider {
   private readonly client: OpenAI;
@@ -29,6 +20,64 @@ export class OpenAIProvider implements ILLMProvider {
     this.embeddingConfig = embeddingConfig;
   }
 
+  static getValidator(): IProviderValidator {
+    return {
+      validate(config: Record<string, unknown>): ValidationError[] {
+        const errors: ValidationError[] = [];
+        const isEmbedding = config.provider === 'openai' && 'dimensions' in config;
+        const prefix = isEmbedding ? 'embedding' : 'llm';
+
+        if (!config.apiKey) {
+          errors.push({
+            field: `${prefix}.apiKey`,
+            message: 'OpenAI API key is required',
+            suggestion: 'Set OPENAI_API_KEY environment variable',
+            severity: 'error',
+          });
+        }
+        if (!config.model) {
+          errors.push({
+            field: `${prefix}.model`,
+            message: 'OpenAI model name is required',
+            suggestion: isEmbedding ? 'e.g., "text-embedding-3-small"' : 'e.g., "gpt-4o"',
+            severity: 'error',
+          });
+        }
+        return errors;
+      }
+    };
+  }
+
+  static getHealthChecker(): IProviderHealthChecker {
+    return {
+      async check(config: Record<string, unknown>): Promise<HealthCheckResult> {
+        const timestamp = Date.now();
+        const apiKey = config.apiKey as string;
+        const modelName = config.model as string;
+        
+        try {
+          const OpenAI = await import('openai');
+          const client = new OpenAI.default({ apiKey });
+          const models = await client.models.list();
+          const hasModel = models.data.some((m) => m.id === modelName);
+          return {
+            healthy: true,
+            provider: 'openai',
+            capabilities: { model: modelName, available: hasModel, totalModels: models.data.length },
+            timestamp,
+          };
+        } catch (error) {
+          return {
+            healthy: false,
+            provider: 'openai',
+            error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
+            timestamp,
+          };
+        }
+      }
+    };
+  }
+
   async chat(messages: ChatMessage[], context: string, options?: ChatOptions): Promise<string> {
     const systemContent =
       this.llmConfig.systemPrompt ??

package/src/providers/vectordb/BaseVectorProvider.ts

@@ -1,5 +1,6 @@
 import { VectorDBConfig } from '../../config/RagConfig';
 import { VectorMatch, UpsertDocument } from '../../types';
+import { IProviderValidator, IProviderHealthChecker } from '../../core/ProviderInterfaces';
 
 /**
  * BaseVectorProvider — Abstract base class for all vector database providers.
@@ -59,6 +60,16 @@ export abstract class BaseVectorProvider {
    */
   abstract disconnect(): Promise<void>;
 
+  /**
+   * Returns a validator for this provider's configuration.
+   */
+  static getValidator?(): IProviderValidator;
+
+  /**
+   * Returns a health checker for this provider.
+   */
+  static getHealthChecker?(): IProviderHealthChecker;
+
   /**
    * Remove internal keys (starting with __) and queryText from a filter object
    * before passing it to an external provider that might not support them.

package/src/providers/vectordb/MilvusProvider.ts

@@ -76,6 +76,10 @@ export class MilvusProvider extends BaseVectorProvider {
       limit: topK,
       filter: namespace ? `namespace == "${namespace}"` : undefined,
       outputFields: ['content', 'metadata'],
+      searchParams: {
+        nprobe: (this.config.options.nprobe as number) || 16,
+        ef: (this.config.options.efSearch as number) || Math.max(topK * 10, 64)
+      }
     };
     const { data } = await this.http.post('/v1/vector/search', payload);
     return (data.data || []).map((res: Record<string, unknown>) => ({

package/src/providers/vectordb/MongoDBProvider.ts

@@ -2,6 +2,8 @@ import { MongoClient, Collection, Db } from 'mongodb';
 import { VectorDBConfig } from '../../config/RagConfig';
 import { BaseVectorProvider } from './BaseVectorProvider';
 import { VectorMatch, UpsertDocument } from '../../types';
+import { IProviderValidator, IProviderHealthChecker, HealthCheckResult } from '../../core/ProviderInterfaces';
+import { ValidationError } from '../../core/ConfigValidator';
 
 /**
  * MongoDBProvider — MongoDB Atlas Vector Search implementation.
@@ -31,6 +33,75 @@ export class MongoDBProvider extends BaseVectorProvider {
     this.metadataKey = (opts.metadataKey as string) || 'metadata';
   }
 
+  static getValidator(): IProviderValidator {
+    return {
+      validate(config: Record<string, unknown>): ValidationError[] {
+        const errors: ValidationError[] = [];
+        const opts = (config.options || {}) as Record<string, unknown>;
+
+        if (!opts.uri) {
+          errors.push({
+            field: 'vectorDb.options.uri',
+            message: 'MongoDB connection URI is required',
+            suggestion: 'Set MONGODB_URI environment variable',
+            severity: 'error',
+          });
+        }
+        if (!opts.database) {
+          errors.push({
+            field: 'vectorDb.options.database',
+            message: 'MongoDB database name is required',
+            severity: 'error',
+          });
+        }
+        if (!opts.collection) {
+          errors.push({
+            field: 'vectorDb.options.collection',
+            message: 'MongoDB collection name is required',
+            severity: 'error',
+          });
+        }
+        return errors;
+      }
+    };
+  }
+
+  static getHealthChecker(): IProviderHealthChecker {
+    return {
+      async check(config: Record<string, unknown>): Promise<HealthCheckResult> {
+        const opts = (config.options || {}) as Record<string, string>;
+        const timestamp = Date.now();
+        try {
+          const { MongoClient } = await import('mongodb');
+          const client = new MongoClient(opts.uri);
+          await client.connect();
+          const db = client.db(opts.database);
+          await db.command({ ping: 1 });
+          const collections = await db.listCollections({ name: opts.collection }).toArray();
+          await client.close();
+
+          return {
+            healthy: true,
+            provider: 'mongodb',
+            capabilities: { 
+              database: opts.database, 
+              collection: opts.collection,
+              exists: collections.length > 0 
+            },
+            timestamp,
+          };
+        } catch (error) {
+          return {
+            healthy: false,
+            provider: 'mongodb',
+            error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
+            timestamp,
+          };
+        }
+      }
+    };
+  }
+
   async initialize(): Promise<void> {
     await this.client.connect();
     this.db = this.client.db(this.dbName);
@@ -74,7 +145,7 @@ export class MongoDBProvider extends BaseVectorProvider {
           index: this.config.indexName || 'vector_index',
           path: this.embeddingKey,
           queryVector: vector,
-          numCandidates: Math.max(topK * 10, 100),
+          numCandidates: (this.config.options.numCandidates as number) || Math.max(topK * 20, 200),
           limit: topK,
           ...(Object.keys(publicFilter).length > 0 || namespace
             ? { filter: { ...publicFilter, ...(namespace ? { namespace } : {}) } }
@@ -110,23 +181,16 @@ export class MongoDBProvider extends BaseVectorProvider {
     await this.collection!.deleteMany({ namespace });
   }
 
-  /**
-   * Sanitise and flatten filter for MongoDB.
-   * Strips internal engine fields and keywords.
-   */
   protected sanitizeFilter(filter?: Record<string, unknown>): Record<string, unknown> {
     const sanitized = super.sanitizeFilter(filter);
     const mongoFilter: Record<string, unknown> = {};
 
     for (const [key, value] of Object.entries(sanitized)) {
-      // Skip internal engine fields and generic keywords that don't exist in metadata
       if (key.startsWith('__') || key === 'keywords' || key === 'queryText') continue;
 
-      // If the key is already a special key, keep it (e.g. namespace)
       if (key === 'namespace') {
         mongoFilter[key] = value;
       } else {
-        // Assume all other non-prefixed fields are metadata fields
         mongoFilter[`${this.metadataKey}.${key}`] = value;
       }
     }

package/src/providers/vectordb/PineconeProvider.ts

@@ -2,14 +2,11 @@ import { Pinecone, RecordMetadata } from '@pinecone-database/pinecone';
 import { VectorDBConfig } from '../../config/RagConfig';
 import { BaseVectorProvider } from './BaseVectorProvider';
 import { VectorMatch, UpsertDocument } from '../../types';
+import { IProviderValidator, IProviderHealthChecker, HealthCheckResult } from '../../core/ProviderInterfaces';
+import { ValidationError } from '../../core/ConfigValidator';
 
 /**
  * PineconeProvider — Pinecone vector database implementation.
- *
- * Compatible with @pinecone-database/pinecone v7.x SDK.
- *
- * Required options: { apiKey: string }
- * The `environment` option is no longer needed (removed in SDK v5+).
  */
 export class PineconeProvider extends BaseVectorProvider {
   private client!: Pinecone;
@@ -22,6 +19,64 @@ export class PineconeProvider extends BaseVectorProvider {
     this.apiKey = opts.apiKey;
   }
 
+  static getValidator(): IProviderValidator {
+    return {
+      validate(config: Record<string, unknown>): ValidationError[] {
+        const errors: ValidationError[] = [];
+        const opts = (config.options || {}) as Record<string, unknown>;
+
+        if (!opts.apiKey) {
+          errors.push({
+            field: 'vectorDb.options.apiKey',
+            message: 'Pinecone API key is required',
+            suggestion: 'Set PINECONE_API_KEY environment variable',
+            severity: 'error',
+          });
+        }
+        return errors;
+      }
+    };
+  }
+
+  static getHealthChecker(): IProviderHealthChecker {
+    return {
+      async check(config: Record<string, unknown>): Promise<HealthCheckResult> {
+        const opts = (config.options || {}) as Record<string, string>;
+        const indexName = config.indexName as string;
+        const timestamp = Date.now();
+        
+        try {
+          const { Pinecone } = await import('@pinecone-database/pinecone');
+          const client = new Pinecone({ apiKey: opts.apiKey });
+          const indexes = await client.listIndexes();
+          const indexNames = indexes.indexes?.map((i) => i.name) ?? [];
+
+          if (!indexNames.includes(indexName)) {
+            return {
+              healthy: false,
+              provider: 'pinecone',
+              error: `Index "${indexName}" not found. Available: ${indexNames.join(', ')}`,
+              timestamp,
+            };
+          }
+          return {
+            healthy: true,
+            provider: 'pinecone',
+            capabilities: { indexes: indexNames.length, targetIndex: indexName },
+            timestamp,
+          };
+        } catch (error) {
+          return {
+            healthy: false,
+            provider: 'pinecone',
+            error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
+            timestamp,
+          };
+        }
+      }
+    };
+  }
+
   async initialize(): Promise<void> {
     this.client = new Pinecone({ apiKey: this.apiKey });
     const indexes = await this.client.listIndexes();

package/src/providers/vectordb/PostgreSQLProvider.ts

@@ -2,6 +2,8 @@ import { Pool, PoolClient } from 'pg';
 import { VectorDBConfig } from '../../config/RagConfig';
 import { BaseVectorProvider } from './BaseVectorProvider';
 import { VectorMatch, UpsertDocument } from '../../types';
+import { IProviderValidator, IProviderHealthChecker, HealthCheckResult } from '../../core/ProviderInterfaces';
+import { ValidationError } from '../../core/ConfigValidator';
 
 /**
  * PostgreSQLProvider — PostgreSQL implementation using the pgvector extension.
@@ -21,6 +23,66 @@ export class PostgreSQLProvider extends BaseVectorProvider {
     this.dimensions = (opts.dimensions as number) ?? 1536;
   }
 
+  static getValidator(): IProviderValidator {
+    return {
+      validate(config: Record<string, unknown>): ValidationError[] {
+        const errors: ValidationError[] = [];
+        const opts = (config.options || {}) as Record<string, unknown>;
+
+        if (!opts.connectionString) {
+          errors.push({
+            field: 'vectorDb.options.connectionString',
+            message: 'PostgreSQL connection string is required',
+            suggestion: 'Set PGVECTOR_CONNECTION_STRING environment variable',
+            severity: 'error',
+          });
+        }
+
+        if (opts.tables && typeof opts.tables !== 'string' && !Array.isArray(opts.tables)) {
+          errors.push({
+            field: 'vectorDb.options.tables',
+            message: 'PostgreSQL tables must be a string or a string array',
+            severity: 'error',
+          });
+        }
+
+        return errors;
+      }
+    };
+  }
+
+  static getHealthChecker(): IProviderHealthChecker {
+    return {
+      async check(config: Record<string, unknown>): Promise<HealthCheckResult> {
+        const opts = (config.options || {}) as Record<string, string>;
+        const timestamp = Date.now();
+        try {
+          const { Client } = await import('pg');
+          const client = new Client({ connectionString: opts.connectionString });
+          await client.connect();
+          const result = await client.query(`
+            SELECT EXISTS(SELECT 1 FROM pg_extension WHERE extname = 'vector');
+          `);
+          const hasVector = result.rows[0].exists;
+          await client.end();
+          return {
+            healthy: true,
+            provider: 'postgresql',
+            capabilities: { pgvectorInstalled: hasVector },
+            timestamp,
+          };
+        } catch (error) {
+          return {
+            healthy: false,
+            provider: 'postgresql',
+            error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
+            timestamp,
+          };
+        }
+      }
+    };
+  }
+
   async initialize(): Promise<void> {
     this.pool = new Pool({ connectionString: this.connectionString });
     const client: PoolClient = await this.pool.connect();
@@ -115,21 +177,29 @@ export class PostgreSQLProvider extends BaseVectorProvider {
       whereClause = whereClause ? `${whereClause} AND ${filterConditions}` : `WHERE ${filterConditions}`;
     }
 
-    const result = await this.pool.query(
-      `SELECT id, content, metadata, 1 - (embedding <=> $1::vector) AS score
-       FROM ${this.tableName}
-       ${whereClause}
-       ORDER BY embedding <=> $1::vector
-       LIMIT $2`,
-      params
-    );
+    const client = await this.pool.connect();
+    try {
+      const efSearch = (this.config.options.efSearch as number) || Math.max(topK * 10, 40);
+      await client.query(`SET LOCAL hnsw.ef_search = ${efSearch}`);
 
-    return result.rows.map((row: Record<string, unknown>) => ({
-      id: String(row['id']),
-      score: parseFloat(String(row['score'])),
-      content: String(row['content']),
-      metadata: row['metadata'] as Record<string, unknown>,
-    }));
+      const result = await client.query(
+        `SELECT id, content, metadata, 1 - (embedding <=> $1::vector) AS score
+         FROM ${this.tableName}
+         ${whereClause}
+         ORDER BY embedding <=> $1::vector
+         LIMIT $2`,
+        params
+      );
+
+      return result.rows.map((row: Record<string, unknown>) => ({
+        id: String(row['id']),
+        score: parseFloat(String(row['score'])),
+        content: String(row['content']),
+        metadata: row['metadata'] as Record<string, unknown>,
+      }));
+    } finally {
+      client.release();
+    }
   }
 
   async delete(id: string | number, namespace?: string): Promise<void> {

package/src/providers/vectordb/QdrantProvider.ts

@@ -112,6 +112,10 @@ export class QdrantProvider extends BaseVectorProvider {
       vector: vector,
       limit: topK,
       with_payload: true,
+      params: {
+        hnsw_ef: (this.config.options.efSearch as number) || Math.max(topK * 20, 128),
+        exact: false
+      },
       filter: namespace ? {
         must: [{ key: 'namespace', match: { value: namespace } }]
       } : undefined,

package/src/providers/vectordb/WeaviateProvider.ts

@@ -69,16 +69,20 @@ export class WeaviateProvider extends BaseVectorProvider {
     await this.http.post('/v1/batch/objects', payload);
   }
 
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
   async query(vector: number[], topK: number, namespace?: string, _filter?: Record<string, unknown>): Promise<VectorMatch[]> {
+    const sanitizedFilter = this.sanitizeFilter(_filter);
+    const queryText = sanitizedFilter.queryText as string;
+
+    const searchParams = queryText 
+      ? `hybrid: { query: ${JSON.stringify(queryText)}, alpha: 0.5 }`
+      : `nearVector: { vector: ${JSON.stringify(vector)} }`;
+
     const graphqlQuery = {
       query: `
         {
           Get {
             ${this.indexName}(
-              nearVector: {
-                vector: ${JSON.stringify(vector)}
-              }
+              ${searchParams}
               limit: ${topK}
               ${namespace ? `where: { path: ["namespace"], operator: Equal, valueString: "${namespace}" }` : ''}
             ) {