@retrivora-ai/rag-engine 0.1.3 → 0.1.5

package/src/rag/RAGPipeline.ts

@@ -1,200 +0,0 @@
-/**
- * RAGPipeline — the central orchestrator for the RAG workflow.
- *
- * Responsibilities:
- *  1. INGEST:  chunk → embed → upsert into vector DB
- *  2. ASK:     embed query → retrieve top-K chunks → call LLM with context
- *
- * It is LLM-agnostic and vector-DB-agnostic; concrete implementations are
- * injected via ILLMProvider and IVectorDB.
- */
-
-import { IVectorDB, VectorMatch } from '../vectordb/IVectorDB';
-import { ILLMProvider, ChatMessage } from '../llm/ILLMProvider';
-import { VectorDBFactory } from '../vectordb/VectorDBFactory';
-import { LLMFactory } from '../llm/LLMFactory';
-import { DocumentChunker } from './DocumentChunker';
-import { RagConfig } from '../config/RagConfig';
-
-export interface IngestDocument {
-  /** Unique document ID (used as chunk ID prefix) */
-  docId: string;
-  /** Full text content of the document */
-  content: string;
-  /** Optional metadata stored alongside each chunk */
-  metadata?: Record<string, unknown>;
-}
-
-export interface ChatResponse {
-  /** The LLM's answer */
-  reply: string;
-  /** Retrieved source chunks used to generate the answer */
-  sources: VectorMatch[];
-}
-
-export class RAGPipeline {
-  private readonly vectorDB: IVectorDB;
-  private readonly llmProvider: ILLMProvider;
-  private readonly embeddingProvider: ILLMProvider;
-  private readonly chunker: DocumentChunker;
-  private readonly config: RagConfig;
-  private initialised = false;
-
-  constructor(
-    config: RagConfig,
-    adapters?: {
-      vectorDB?: IVectorDB;
-      llmProvider?: ILLMProvider;
-      embeddingProvider?: ILLMProvider;
-    }
-  ) {
-    this.config = config;
-    this.vectorDB = adapters?.vectorDB ?? VectorDBFactory.create(config.vectorDb);
-    this.llmProvider = adapters?.llmProvider ?? LLMFactory.create(config.llm, config.embedding);
-
-    // Use custom embedding provider if provided
-    if (adapters?.embeddingProvider) {
-      this.embeddingProvider = adapters.embeddingProvider;
-    } else if (config.llm.provider === 'anthropic') {
-      // If the LLM provider is Anthropic and no custom embedding is provided, spin up a dedicated embedding provider
-      this.embeddingProvider = LLMFactory.createEmbeddingProvider(config.embedding);
-    } else {
-      this.embeddingProvider = this.llmProvider;
-    }
-
-    this.chunker = new DocumentChunker(
-      config.rag?.chunkSize ?? 1000,
-      config.rag?.chunkOverlap ?? 200
-    );
-  }
-
-  /** Lazily initialise the vector DB connection */
-  async init(): Promise<void> {
-    if (!this.initialised) {
-      await this.vectorDB.initialize();
-      this.initialised = true;
-    }
-  }
-
-  // ---------------------------------------------------------------------------
-  // INGEST
-  // ---------------------------------------------------------------------------
-
-  /**
-   * Ingest one or more documents into the vector DB.
-   * Automatically chunks, embeds, and upserts each chunk.
-   */
-  async ingest(
-    documents: IngestDocument[],
-    namespace?: string
-  ): Promise<{ docId: string; chunksIngested: number }[]> {
-    await this.init();
-
-    const ns = namespace ?? this.config.projectId;
-    const results: { docId: string; chunksIngested: number }[] = [];
-
-    for (const doc of documents) {
-      const chunks = this.chunker.chunk(doc.content, {
-        docId: doc.docId,
-        metadata: doc.metadata,
-      });
-
-      const chunkTexts = chunks.map((c) => c.content);
-      const vectors = await this.embeddingProvider.batchEmbed(chunkTexts);
-
-      const upsertDocs = chunks.map((chunk, i) => ({
-        id: chunk.id,
-        vector: vectors[i],
-        content: chunk.content,
-        metadata: chunk.metadata,
-      }));
-
-      await this.vectorDB.batchUpsert(upsertDocs, ns);
-      results.push({ docId: doc.docId, chunksIngested: chunks.length });
-      console.log(`[RAGPipeline] Ingested ${chunks.length} chunks for doc: ${doc.docId}`);
-    }
-
-    return results;
-  }
-
-  // ---------------------------------------------------------------------------
-  // ASK
-  // ---------------------------------------------------------------------------
-
-  /**
-   * Run a full RAG query:
-   *  1. Embed the user question
-   *  2. Retrieve top-K relevant chunks from the vector DB
-   *  3. Filter by score threshold
-   *  4. Build context string
-   *  5. Call the LLM with chat history + context
-   */
-  async ask(
-    question: string,
-    history: ChatMessage[] = [],
-    namespace?: string
-  ): Promise<ChatResponse> {
-    await this.init();
-
-    const ns = namespace ?? this.config.projectId;
-    const topK = this.config.rag?.topK ?? 5;
-    const scoreThreshold = this.config.rag?.scoreThreshold ?? 0.0;
-
-    console.log(`[RAGPipeline] Searching Pinecone (Namespace: "${ns}", TopK: ${topK})...`);
-
-    // 1. Embed the question
-    const queryVector = await this.embeddingProvider.embed(question);
-
-    // 2. Retrieve relevant chunks
-    const rawMatches = await this.vectorDB.query(queryVector, topK, ns);
-    console.log(`[RAGPipeline] Found ${rawMatches.length} raw matches in Pinecone.`);
-
-    // 3. Apply score threshold filter
-    const sources = rawMatches.filter((m) => m.score >= scoreThreshold);
-    console.log(`[RAGPipeline] ${sources.length} sources remaining after threshold filter (min: ${scoreThreshold}).`);
-
-    // 4. Build context
-    const context = sources.length
-      ? sources.map((m, i) => `[Source ${i + 1}]\n${m.content}`).join('\n\n---\n\n')
-      : 'No relevant context found.';
-
-    // 5. Build message list (append current question)
-    const messages: ChatMessage[] = [
-      ...history,
-      { role: 'user', content: question },
-    ];
-
-    // 6. Call the LLM
-    const reply = await this.llmProvider.chat(messages, context);
-
-    return { reply, sources };
-  }
-
-  // ---------------------------------------------------------------------------
-  // HEALTH
-  // ---------------------------------------------------------------------------
-
-  async health(): Promise<{ vectorDB: boolean; llm: boolean }> {
-    const [vectorDB, llm] = await Promise.all([
-      this.vectorDB.ping().catch(() => false),
-      this.llmProvider.ping().catch(() => false),
-    ]);
-    return { vectorDB, llm };
-  }
-
-  // ---------------------------------------------------------------------------
-  // DATA MANAGEMENT
-  // ---------------------------------------------------------------------------
-
-  async deleteDocument(docId: string, namespace?: string): Promise<void> {
-    await this.init();
-    const ns = namespace ?? this.config.projectId;
-    await this.vectorDB.delete(docId, ns);
-  }
-
-  async deleteProject(namespace?: string): Promise<void> {
-    await this.init();
-    const ns = namespace ?? this.config.projectId;
-    await this.vectorDB.deleteNamespace(ns);
-  }
-}

package/src/vectordb/IVectorDB.ts

@@ -1,75 +0,0 @@
-/**
- * Generic Vector Database interface.
- * Any adapter (Pinecone, pgVector, REST, Chroma, Qdrant, …) must implement this.
- */
-
-export interface VectorMatch {
-  /** Unique identifier of the stored chunk */
-  id: string;
-  /** Cosine or dot-product similarity score (0–1) */
-  score: number;
-  /** The original text content of the chunk */
-  content: string;
-  /** Arbitrary metadata attached at upsert time */
-  metadata?: Record<string, unknown>;
-}
-
-export interface UpsertDocument {
-  id: string;
-  vector: number[];
-  content: string;
-  metadata?: Record<string, unknown>;
-}
-
-export interface IVectorDB {
-  /**
-   * Initialise the connection (create tables, verify index, etc.)
-   * Called once during RAGPipeline construction.
-   */
-  initialize(): Promise<void>;
-
-  /**
-   * Insert or update a single vector + content pair.
-   * @param namespace – optional project/tenant namespace
-   */
-  upsert(doc: UpsertDocument, namespace?: string): Promise<void>;
-
-  /**
-   * Batch upsert for efficient ingestion of many documents.
-   */
-  batchUpsert(docs: UpsertDocument[], namespace?: string): Promise<void>;
-
-  /**
-   * Find the top-K most similar vectors to the query vector.
-   * @param vector     – embedding of the user query
-   * @param topK       – number of results to return
-   * @param namespace  – optional project/tenant namespace
-   * @param filter     – optional metadata filter (provider-specific shape)
-   */
-  query(
-    vector: number[],
-    topK: number,
-    namespace?: string,
-    filter?: Record<string, unknown>
-  ): Promise<VectorMatch[]>;
-
-  /**
-   * Delete a stored vector by ID.
-   */
-  delete(id: string, namespace?: string): Promise<void>;
-
-  /**
-   * Delete all vectors in a namespace (full data reset for a project).
-   */
-  deleteNamespace(namespace: string): Promise<void>;
-
-  /**
-   * Check if the underlying DB is reachable.
-   */
-  ping(): Promise<boolean>;
-
-  /**
-   * Gracefully close the connection.
-   */
-  disconnect(): Promise<void>;
-}

package/src/vectordb/VectorDBFactory.ts

@@ -1,41 +0,0 @@
-/**
- * VectorDBFactory — instantiates the correct IVectorDB adapter
- * based on the VectorDBConfig.provider value.
- *
- * Usage:
- *   const db = VectorDBFactory.create(config.vectorDb);
- *   await db.initialize();
- */
-
-import { IVectorDB } from './IVectorDB';
-import { VectorDBConfig } from '../config/RagConfig';
-import { PineconeAdapter } from './adapters/PineconeAdapter';
-import { PgVectorAdapter } from './adapters/PgVectorAdapter';
-import { UniversalVectorDBAdapter } from './adapters/UniversalVectorDBAdapter';
-import { MongoDbAdapter } from './adapters/MongoDbAdapter';
-
-export class VectorDBFactory {
-  static create(config: VectorDBConfig): IVectorDB {
-    switch (config.provider) {
-      case 'pinecone':
-        return new PineconeAdapter(config);
-      case 'pgvector':
-        return new PgVectorAdapter(config);
-      case 'mongodb':
-        return new MongoDbAdapter(config.options);
-      case 'rest':
-      case 'universal_rest':
-      case 'custom':
-        return new UniversalVectorDBAdapter(config);
-      default:
-        // Fallback to Universal Adapter if baseUrl is present in options
-        if ((config.options as Record<string, unknown>)?.baseUrl) {
-          return new UniversalVectorDBAdapter(config);
-        }
-        throw new Error(
-          `[VectorDBFactory] Unknown provider "${(config as VectorDBConfig).provider}". ` +
-            `Supported: pinecone | pgvector | mongodb | rest`
-        );
-    }
-  }
-}

package/src/vectordb/adapters/MongoDbAdapter.ts

@@ -1,175 +0,0 @@
-import { MongoClient, Collection, Db } from 'mongodb';
-import { IVectorDB, UpsertDocument, VectorMatch } from '../IVectorDB';
-
-interface MongoDoc {
-  _id: string;
-  [key: string]: unknown;
-}
-
-/**
- * MongoDB Adapter for Atlas Vector Search.
- *
- * Requirements:
- * 1. A MongoDB Atlas cluster (v6.0.11+, v7.0.2+).
- * 2. A Vector Search Index created on the collection.
- *
- * Config Options:
- * - uri: string (Required)
- * - database: string (Required)
- * - collection: string (Required)
- * - indexName: string (Default: 'vector_index')
- * - embeddingKey: string (Default: 'embedding')
- * - contentKey: string (Default: 'content')
- * - metadataKey: string (Default: 'metadata')
- */
-export class MongoDbAdapter implements IVectorDB {
-  private client: MongoClient;
-  private db?: Db;
-  private collection?: Collection<MongoDoc>;
-  private uri: string;
-  private dbName: string;
-  private collectionName: string;
-  private indexName: string;
-  private embeddingKey: string;
-  private contentKey: string;
-  private metadataKey: string;
-
-  constructor(options: Record<string, unknown>) {
-    this.uri = options.uri as string;
-    this.dbName = options.database as string;
-    this.collectionName = options.collection as string;
-    this.indexName = (options.indexName as string) || 'vector_index';
-    this.embeddingKey = (options.embeddingKey as string) || 'embedding';
-    this.contentKey = (options.contentKey as string) || 'content';
-    this.metadataKey = (options.metadataKey as string) || 'metadata';
-
-    if (!this.uri || !this.dbName || !this.collectionName) {
-      throw new Error('MongoDbAdapter requires uri, database, and collection options.');
-    }
-
-    this.client = new MongoClient(this.uri);
-  }
-
-  async initialize(): Promise<void> {
-    await this.client.connect();
-    this.db = this.client.db(this.dbName);
-    this.collection = this.db.collection(this.collectionName);
-  }
-
-  async upsert(doc: UpsertDocument, namespace?: string): Promise<void> {
-    if (!this.collection) await this.initialize();
-
-    const document = {
-      _id: doc.id as string,
-      [this.embeddingKey]: doc.vector,
-      [this.contentKey]: doc.content,
-      [this.metadataKey]: doc.metadata || {},
-      ...(namespace ? { namespace } : {}),
-    };
-
-    await this.collection!.updateOne(
-      { _id: doc.id as string },
-      { $set: document },
-      { upsert: true }
-    );
-  }
-
-  async batchUpsert(docs: UpsertDocument[], namespace?: string): Promise<void> {
-    if (!this.collection) await this.initialize();
-
-    const operations = docs.map((doc) => ({
-      updateOne: {
-        filter: { _id: doc.id as string },
-        update: {
-          $set: {
-            _id: doc.id as string,
-            [this.embeddingKey]: doc.vector,
-            [this.contentKey]: doc.content,
-            [this.metadataKey]: doc.metadata || {},
-            ...(namespace ? { namespace } : {}),
-          },
-        },
-        upsert: true,
-      },
-    }));
-
-    await this.collection!.bulkWrite(operations);
-  }
-
-  async query(
-    vector: number[],
-    topK: number,
-    namespace?: string,
-    filter?: Record<string, unknown>
-  ): Promise<VectorMatch[]> {
-    if (!this.collection) await this.initialize();
-
-    const pipeline: Record<string, unknown>[] = [
-      {
-        $vectorSearch: {
-          index: this.indexName,
-          path: this.embeddingKey,
-          queryVector: vector,
-          numCandidates: Math.max(topK * 10, 100),
-          limit: topK,
-          ...(filter || namespace
-            ? {
-              filter: {
-                ...(filter || {}),
-                ...(namespace ? { namespace } : {}),
-              },
-            }
-            : {}),
-        },
-      },
-      {
-        $project: {
-          score: { $meta: 'vectorSearchScore' },
-          content: `$${this.contentKey}`,
-          metadata: `$${this.metadataKey}`,
-        },
-      },
-    ];
-
-    const results = await this.collection!.aggregate(pipeline).toArray();
-
-    return (results as unknown as MongoDoc[]).map((res) => ({
-      id: res._id.toString(),
-      score: res.score as number,
-      content: res.content as string,
-      metadata: res.metadata as Record<string, unknown>,
-    }));
-  }
-
-  async delete(id: string, namespace?: string): Promise<void> {
-    if (!this.collection) await this.initialize();
-    const query = { _id: id, ...(namespace ? { namespace } : {}) };
-    await this.collection!.deleteOne(query as unknown as MongoDoc);
-  }
-
-  async deleteNamespace(namespace: string): Promise<void> {
-    if (!this.collection) await this.initialize();
-    await this.collection!.deleteMany({ namespace });
-  }
-
-  async ping(): Promise<boolean> {
-    try {
-      if (!this.collection) await this.initialize();
-      await this.db!.command({ ping: 1 });
-      return true;
-    } catch (err) {
-      console.error('[MongoDbAdapter] Ping failed:', err);
-      return false;
-    }
-  }
-
-  async disconnect(): Promise<void> {
-    try {
-      await this.client.close();
-      this.db = undefined;
-      this.collection = undefined;
-    } catch (err) {
-      console.error('[MongoDbAdapter] Failed to disconnect:', err);
-    }
-  }
-}

package/src/vectordb/adapters/UniversalVectorDBAdapter.ts

@@ -1,177 +0,0 @@
-import axios, { AxiosInstance } from 'axios';
-import { IVectorDB, VectorMatch, UpsertDocument } from '../IVectorDB';
-import { VectorDBConfig } from '../../config/RagConfig';
-import { resolvePath, buildPayload } from '../../utils/templateUtils';
-import { VECTOR_PROFILES } from '../../config/UniversalProfiles';
-
-interface UniversalVectorDBOptions {
-  baseUrl?: string;
-  headers?: Record<string, string>;
-  queryPath?: string;
-  upsertPath?: string;
-  deletePath?: string;
-  deleteNamespacePath?: string;
-  pingPath?: string;
-  queryPayloadTemplate?: string;
-  upsertPayloadTemplate?: string;
-  batchUpsertPayloadTemplate?: string;
-  responseExtractPath?: string;
-  idPath?: string;
-  scorePath?: string;
-  contentPath?: string;
-  metadataPath?: string;
-  timeout?: number;
-}
-
-export class UniversalVectorDBAdapter implements IVectorDB {
-  private readonly http: AxiosInstance;
-  private readonly indexName: string;
-  private readonly opts: UniversalVectorDBOptions;
-
-  constructor(config: VectorDBConfig) {
-    this.indexName = config.indexName;
-    
-    // Merge profile defaults if specified
-    const options = (config.options as Record<string, unknown>) ?? {};
-    let profile: Record<string, unknown> = {};
-
-    if (typeof options.profile === 'string') {
-      profile = (VECTOR_PROFILES as Record<string, Record<string, unknown>>)[options.profile] || {};
-    } else if (typeof options.profile === 'object' && options.profile !== null) {
-      profile = options.profile as Record<string, unknown>;
-    }
-
-    this.opts = { ...profile, ...(config.options ?? {}) } as UniversalVectorDBOptions;
-
-    if (!this.opts.baseUrl) {
-      throw new Error('[UniversalVectorDBAdapter] options.baseUrl is required');
-    }
-
-    this.http = axios.create({
-      baseURL: this.opts.baseUrl,
-      headers: {
-        'Content-Type': 'application/json',
-        ...(this.opts.headers || {}),
-      },
-      timeout: this.opts.timeout ?? 30_000,
-    });
-  }
-
-  async initialize(): Promise<void> {
-    if (this.opts.pingPath) {
-      const ok = await this.ping();
-      if (!ok) {
-        throw new Error(`[UniversalVectorDBAdapter] Could not reach health endpoint: ${this.opts.pingPath}`);
-      }
-    }
-  }
-
-  async upsert(doc: UpsertDocument, namespace?: string): Promise<void> {
-    const path = this.opts.upsertPath ?? '/upsert';
-    
-    if (this.opts.upsertPayloadTemplate) {
-      const payload = buildPayload(this.opts.upsertPayloadTemplate, {
-        id: doc.id,
-        vector: doc.vector,
-        content: doc.content,
-        metadata: doc.metadata ?? {},
-        namespace: namespace ?? '',
-        index: this.indexName,
-      });
-      await this.http.post(path, payload);
-    } else {
-      // Fallback to standard generic shape
-      await this.http.post(path, {
-        index: this.indexName,
-        namespace,
-        ...doc,
-      });
-    }
-  }
-
-  async batchUpsert(docs: UpsertDocument[], namespace?: string): Promise<void> {
-    const path = this.opts.upsertPath ?? '/upsert';
-    
-    // For batch upsert with templating, if the user provided batchUpsertPayloadTemplate
-    if (this.opts.batchUpsertPayloadTemplate) {
-       const payload = buildPayload(this.opts.batchUpsertPayloadTemplate, {
-          vectors: docs,
-          namespace: namespace ?? '',
-          index: this.indexName,
-       });
-       await this.http.post(path, payload);
-    } else {
-       // Fallback to standard loop or generic shape
-       await Promise.all(docs.map((doc) => this.upsert(doc, namespace)));
-    }
-  }
-
-  async query(
-    vector: number[],
-    topK: number,
-    namespace?: string,
-    filter?: Record<string, unknown>
-  ): Promise<VectorMatch[]> {
-    const path = this.opts.queryPath ?? '/query';
-
-    let payload: unknown;
-    if (this.opts.queryPayloadTemplate) {
-      payload = buildPayload(this.opts.queryPayloadTemplate, {
-        vector,
-        topK,
-        namespace: namespace ?? '',
-        index: this.indexName,
-        filter: filter ?? {},
-      });
-    } else {
-      payload = { index: this.indexName, namespace, vector, topK, filter };
-    }
-
-    const { data } = await this.http.post(path, payload);
-
-    const extractPath = this.opts.responseExtractPath ?? 'matches';
-    const matchesRaw = resolvePath(data, extractPath);
-
-    if (!Array.isArray(matchesRaw)) {
-       throw new Error(`[UniversalVectorDBAdapter] Expected an array at responseExtractPath '${extractPath}', but got ${typeof matchesRaw}`);
-    }
-
-    const idPath = this.opts.idPath ?? 'id';
-    const scorePath = this.opts.scorePath ?? 'score';
-    const contentPath = this.opts.contentPath ?? 'content';
-    const metadataPath = this.opts.metadataPath ?? 'metadata';
-
-    return matchesRaw.map((match) => ({
-      id: String(resolvePath(match, idPath) || ''),
-      score: Number(resolvePath(match, scorePath) || 0),
-      content: String(resolvePath(match, contentPath) || ''),
-      metadata: (resolvePath(match, metadataPath) as Record<string, unknown> | undefined) || {},
-    }));
-  }
-
-  async delete(id: string, namespace?: string): Promise<void> {
-    const path = this.opts.deletePath ?? '/delete';
-    await this.http.post(path, { index: this.indexName, namespace, id });
-  }
-
-  async deleteNamespace(namespace: string): Promise<void> {
-    const path = this.opts.deleteNamespacePath ?? '/delete-namespace';
-    await this.http.post(path, { index: this.indexName, namespace });
-  }
-
-  async ping(): Promise<boolean> {
-    try {
-      if (this.opts.pingPath) {
-        await this.http.get(this.opts.pingPath);
-      }
-      return true;
-    } catch (err) {
-      console.error('[UniversalVectorDBAdapter] Ping failed:', err);
-      return false;
-    }
-  }
-
-  async disconnect(): Promise<void> {
-    // Axios based adapter is stateless per request.
-  }
-}