@retrivora-ai/rag-engine 0.1.3 → 0.1.5

package/src/handlers/index.ts

@@ -1,44 +1,14 @@
-/**
- * createChatHandler — factory that returns a Next.js App Router POST handler
- * for the RAG chat endpoint.
- *
- * Use this when you want to mount the RAG chat API inside your OWN Next.js app
- * (i.e. you installed @rag/ai-accelerator as a package and want the API route
- * to live at your project's /api/chat).
- *
- * Usage — in your project's `src/app/api/chat/route.ts`:
- *
- *   import { createChatHandler } from '@rag/ai-accelerator/handlers';
- *   import myConfig from '@/config/ragConfig';
- *   export const POST = createChatHandler(myConfig);
- *
- * Or with env-based config:
- *
- *   import { createChatHandler } from '@rag/ai-accelerator/handlers';
- *   import { getRagConfig } from '@rag/ai-accelerator';
- *   export const POST = createChatHandler(getRagConfig());
- */
-
 import { NextRequest, NextResponse } from 'next/server';
-import { RAGPipeline } from '../rag/RAGPipeline';
+import { VectorPlugin } from '../core/VectorPlugin';
 import { RagConfig } from '../config/RagConfig';
-import { ChatMessage, ILLMProvider } from '../llm/ILLMProvider';
-import { IVectorDB } from '../vectordb/IVectorDB';
+import { ChatMessage } from '../llm/ILLMProvider';
 import { DocumentParser } from '../utils/DocumentParser';
 
-export interface CustomAdapters {
-  vectorDB?: IVectorDB;
-  llmProvider?: ILLMProvider;
-  embeddingProvider?: ILLMProvider;
-}
-
-export function createChatHandler(config: RagConfig, adapters?: CustomAdapters) {
-  let pipeline: RAGPipeline | null = null;
-
-  function getPipeline(): RAGPipeline {
-    if (!pipeline) pipeline = new RAGPipeline(config, adapters);
-    return pipeline;
-  }
+/**
+ * createChatHandler — factory that returns a Next.js App Router POST handler
+ */
+export function createChatHandler(config?: Partial<RagConfig>) {
+  const plugin = new VectorPlugin(config);
 
   return async function POST(req: NextRequest) {
     try {
@@ -53,8 +23,7 @@ export function createChatHandler(config: RagConfig, adapters?: CustomAdapters)
         return NextResponse.json({ error: 'message is required' }, { status: 400 });
       }
 
-      const rag = getPipeline();
-      const result = await rag.ask(message, history, namespace);
+      const result = await plugin.chat(message, history, namespace);
       return NextResponse.json(result);
     } catch (err) {
       const message = err instanceof Error ? err.message : 'Internal server error';
@@ -65,19 +34,9 @@ export function createChatHandler(config: RagConfig, adapters?: CustomAdapters)
 
 /**
  * createIngestHandler — factory for the document ingestion endpoint.
- *
- * Usage — in your project's `src/app/api/ingest/route.ts`:
- *
- *   import { createIngestHandler } from '@rag/ai-accelerator/handlers';
- *   export const POST = createIngestHandler(myConfig, { vectorDB: new MyCustomDB() });
  */
-export function createIngestHandler(config: RagConfig, adapters?: CustomAdapters) {
-  let pipeline: RAGPipeline | null = null;
-
-  function getPipeline(): RAGPipeline {
-    if (!pipeline) pipeline = new RAGPipeline(config, adapters);
-    return pipeline;
-  }
+export function createIngestHandler(config?: Partial<RagConfig>) {
+  const plugin = new VectorPlugin(config);
 
   return async function POST(req: NextRequest) {
     try {
@@ -88,16 +47,11 @@ export function createIngestHandler(config: RagConfig, adapters?: CustomAdapters
       };
 
       if (!Array.isArray(documents) || documents.length === 0) {
-        return NextResponse.json(
-          { error: 'documents array is required and must not be empty' },
-          { status: 400 }
-        );
+        return NextResponse.json({ error: 'documents array is required' }, { status: 400 });
       }
 
-      const rag = getPipeline();
-      const results = await rag.ingest(documents, namespace);
-      const totalChunks = results.reduce((sum, r) => sum + r.chunksIngested, 0);
-      return NextResponse.json({ results, totalChunks });
+      const results = await plugin.ingest(documents, namespace);
+      return NextResponse.json({ results });
     } catch (err) {
       const message = err instanceof Error ? err.message : 'Internal server error';
       return NextResponse.json({ error: message }, { status: 500 });
@@ -107,56 +61,31 @@ export function createIngestHandler(config: RagConfig, adapters?: CustomAdapters
 
 /**
  * createHealthHandler — factory for the health-check endpoint.
- *
- * Usage — in your project's `src/app/api/health/route.ts`:
- *
- *   import { createHealthHandler } from '@rag/ai-accelerator/handlers';
- *   export const GET = createHealthHandler(myConfig, { vectorDB: new MyCustomDB() });
  */
-export function createHealthHandler(config: RagConfig, adapters?: CustomAdapters) {
-  let pipeline: RAGPipeline | null = null;
-
-  function getPipeline(): RAGPipeline {
-    if (!pipeline) pipeline = new RAGPipeline(config, adapters);
-    return pipeline;
-  }
+export function createHealthHandler(config?: Partial<RagConfig>) {
+  const plugin = new VectorPlugin(config);
 
   return async function GET() {
     try {
-      const rag = getPipeline();
-      const health = await rag.health();
+      const health = await plugin.health();
       const status = health.vectorDB && health.llm ? 'ok' : 'degraded';
       return NextResponse.json({
         status,
         ...health,
-        project: config.projectId,
         timestamp: new Date().toISOString(),
       });
     } catch (err) {
       const message = err instanceof Error ? err.message : 'Unknown error';
-      return NextResponse.json(
-        { status: 'error', error: message, timestamp: new Date().toISOString() },
-        { status: 500 }
-      );
+      return NextResponse.json({ status: 'error', error: message }, { status: 500 });
     }
   };
 }
 
 /**
  * createUploadHandler — factory for the file upload ingestion endpoint.
- *
- * Usage — in your project's `src/app/api/upload/route.ts`:
- *
- *   import { createUploadHandler } from '@rag/ai-accelerator/handlers';
- *   export const POST = createUploadHandler(myConfig);
  */
-export function createUploadHandler(config: RagConfig, adapters?: CustomAdapters) {
-  let pipeline: RAGPipeline | null = null;
-
-  function getPipeline(): RAGPipeline {
-    if (!pipeline) pipeline = new RAGPipeline(config, adapters);
-    return pipeline;
-  }
+export function createUploadHandler(config?: Partial<RagConfig>) {
+  const plugin = new VectorPlugin(config);
 
   return async function POST(req: NextRequest) {
     try {
@@ -184,16 +113,8 @@ export function createUploadHandler(config: RagConfig, adapters?: CustomAdapters
         })
       );
 
-      const rag = getPipeline();
-      const results = await rag.ingest(documents, namespace);
-      const totalChunks = results.reduce((sum, r) => sum + r.chunksIngested, 0);
-
-      return NextResponse.json({
-        message: 'Upload successful',
-        filesProcessed: files.length,
-        totalChunks,
-        results,
-      });
+      const results = await plugin.ingest(documents, namespace);
+      return NextResponse.json({ message: 'Upload successful', results });
     } catch (err) {
       const message = err instanceof Error ? err.message : 'Upload failed';
       return NextResponse.json({ error: message }, { status: 500 });

package/src/hooks/useRagChat.ts

@@ -15,7 +15,7 @@
 
 import { useState, useCallback, useRef, useEffect } from 'react';
 import axios from 'axios';
-import { VectorMatch } from '@/vectordb/IVectorDB';
+import { VectorMatch } from '@/types';
 import { useStoredMessages } from './useStoredMessages';
 
 // ─── Types ────────────────────────────────────────────────────────────────────

package/src/index.ts

@@ -1,9 +1,6 @@
 /**
  * Package entry point — public API exported to consumers of this npm package.
  * This entry point is CLIENT-SAFE and can be imported in React Client Components.
- *
- * Usage:
- *   import { ChatWidget, ConfigProvider, useRagChat } from '@retrivora-ai/rag-engine';
  */
 
 // ── React Components ─────────────────────────────────────────
@@ -18,10 +15,9 @@ export { useRagChat } from './hooks/useRagChat';
 
 // ── Types (Interfaces/Types only, no Node.js deps) ─────────────
 export type { RagConfig, VectorDBConfig, LLMConfig, EmbeddingConfig, UIConfig, RAGConfig } from './config/RagConfig';
-export type { IVectorDB, VectorMatch, UpsertDocument } from './vectordb/IVectorDB';
+export type { VectorMatch, UpsertDocument } from './types';
 export type { ILLMProvider, ChatMessage, ChatOptions, EmbedOptions } from './llm/ILLMProvider';
-export type { IngestDocument, ChatResponse } from './rag/RAGPipeline';
+export type { IngestDocument, ChatResponse } from './types';
 export type { Chunk, ChunkOptions } from './rag/DocumentChunker';
 export type { ClientConfig } from './components/ConfigProvider';
 export type { RagMessage, UseRagChatOptions, UseRagChatReturn } from './hooks/useRagChat';
-

package/src/providers/vectordb/BaseVectorProvider.ts

@@ -0,0 +1,61 @@
+import { VectorDBConfig } from '../../config/RagConfig';
+import { VectorMatch, UpsertDocument } from '../../types';
+
+/**
+ * BaseVectorProvider — Abstract base class for all vector database providers.
+ * Each provider (Pinecone, Milvus, Redis, etc.) must extend this class.
+ */
+export abstract class BaseVectorProvider {
+  protected readonly config: VectorDBConfig;
+  protected readonly indexName: string;
+
+  constructor(config: VectorDBConfig) {
+    this.config = config;
+    this.indexName = config.indexName || 'default';
+  }
+
+  /**
+   * Initialise the connection (create tables, verify index, etc.)
+   */
+  abstract initialize(): Promise<void>;
+
+  /**
+   * Insert or update a single vector + content pair.
+   */
+  abstract upsert(doc: UpsertDocument, namespace?: string): Promise<void>;
+
+  /**
+   * Batch upsert for efficient ingestion of many documents.
+   */
+  abstract batchUpsert(docs: UpsertDocument[], namespace?: string): Promise<void>;
+
+  /**
+   * Find the top-K most similar vectors to the query vector.
+   */
+  abstract query(
+    vector: number[],
+    topK: number,
+    namespace?: string,
+    filter?: Record<string, unknown>
+  ): Promise<VectorMatch[]>;
+
+  /**
+   * Delete a stored vector by ID.
+   */
+  abstract delete(id: string, namespace?: string): Promise<void>;
+
+  /**
+   * Delete all vectors in a namespace (full data reset for a project).
+   */
+  abstract deleteNamespace(namespace: string): Promise<void>;
+
+  /**
+   * Check if the underlying DB is reachable.
+   */
+  abstract ping(): Promise<boolean>;
+
+  /**
+   * Gracefully close the connection.
+   */
+  abstract disconnect(): Promise<void>;
+}

package/src/providers/vectordb/ChromaDBProvider.ts

@@ -0,0 +1,94 @@
+import axios, { AxiosInstance } from 'axios';
+import { VectorDBConfig } from '../../config/RagConfig';
+import { BaseVectorProvider } from './BaseVectorProvider';
+import { VectorMatch, UpsertDocument } from '../../types';
+
+/**
+ * ChromaDBProvider — implementation for ChromaDB using its REST API.
+ */
+export class ChromaDBProvider extends BaseVectorProvider {
+  private http: AxiosInstance;
+  private collectionId: string = '';
+
+  constructor(config: VectorDBConfig) {
+    super(config);
+    const opts = config.options as Record<string, unknown>;
+    const baseUrl = opts.baseUrl as string;
+    if (!baseUrl) throw new Error('[ChromaDBProvider] baseUrl is required');
+
+    this.http = axios.create({
+      baseURL: baseUrl,
+      headers: { 'Content-Type': 'application/json' },
+    });
+  }
+
+  async initialize(): Promise<void> {
+    // Get or create collection
+    const { data } = await this.http.get(`/api/v1/collections/${this.indexName}`);
+    this.collectionId = data.id;
+  }
+
+  async upsert(doc: UpsertDocument, namespace?: string): Promise<void> {
+    await this.batchUpsert([doc], namespace);
+  }
+
+  async batchUpsert(docs: UpsertDocument[], namespace?: string): Promise<void> {
+    const payload = {
+      ids: docs.map(d => d.id),
+      embeddings: docs.map(d => d.vector),
+      documents: docs.map(d => d.content),
+      metadatas: docs.map(d => ({
+        ...(d.metadata || {}),
+        ...(namespace ? { namespace } : {}),
+      })),
+    };
+    await this.http.post(`/api/v1/collections/${this.collectionId}/add`, payload);
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  async query(vector: number[], topK: number, namespace?: string, _filter?: Record<string, unknown>): Promise<VectorMatch[]> {
+    const payload = {
+      query_embeddings: [vector],
+      n_results: topK,
+      where: namespace ? { namespace: { $eq: namespace } } : undefined,
+    };
+    const { data } = await this.http.post(`/api/v1/collections/${this.collectionId}/query`, payload);
+    
+    const matches: VectorMatch[] = [];
+    if (data.ids && data.ids[0]) {
+      for (let i = 0; i < data.ids[0].length; i++) {
+        matches.push({
+          id: data.ids[0][i],
+          score: data.distances ? 1 - data.distances[0][i] : 0,
+          content: data.documents ? data.documents[0][i] : '',
+          metadata: data.metadatas ? data.metadatas[0][i] : {},
+        });
+      }
+    }
+    return matches;
+  }
+
+  async delete(id: string, _namespace?: string): Promise<void> {
+    await this.http.post(`/api/v1/collections/${this.collectionId}/delete`, {
+      ids: [id],
+      where: _namespace ? { namespace: { $eq: _namespace } } : undefined,
+    });
+  }
+
+  async deleteNamespace(_namespace: string): Promise<void> {
+    await this.http.post(`/api/v1/collections/${this.collectionId}/delete`, {
+      where: { namespace: { $eq: _namespace } },
+    });
+  }
+
+  async ping(): Promise<boolean> {
+    try {
+      await this.http.get('/api/v1/heartbeat');
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
+  async disconnect(): Promise<void> {}
+}

package/src/providers/vectordb/MilvusProvider.ts

@@ -0,0 +1,100 @@
+import axios, { AxiosInstance } from 'axios';
+import { VectorDBConfig } from '../../config/RagConfig';
+import { BaseVectorProvider } from './BaseVectorProvider';
+import { VectorMatch, UpsertDocument } from '../../types';
+
+/**
+ * MilvusProvider — implementation for Milvus using its REST API.
+ */
+export class MilvusProvider extends BaseVectorProvider {
+  private http: AxiosInstance;
+
+  constructor(config: VectorDBConfig) {
+    super(config);
+    const opts = config.options as Record<string, unknown>;
+    const baseUrl = (opts.baseUrl as string) || (opts.uri as string);
+    if (!baseUrl) throw new Error('[MilvusProvider] baseUrl/uri is required');
+
+    this.http = axios.create({
+      baseURL: baseUrl,
+      headers: {
+        'Content-Type': 'application/json',
+        ...(opts.headers as Record<string, string> || {}),
+        ...(opts.apiKey ? { Authorization: `Bearer ${opts.apiKey}` } : {}),
+      },
+    });
+  }
+
+  async initialize(): Promise<void> {
+    await this.ping();
+  }
+
+  async upsert(doc: UpsertDocument, namespace?: string): Promise<void> {
+    const payload = {
+      collectionName: this.indexName,
+      data: [{
+        vector: doc.vector,
+        content: doc.content,
+        metadata: doc.metadata || {},
+        ...(namespace ? { namespace } : {}),
+      }],
+    };
+    await this.http.post('/v1/vector/upsert', payload);
+  }
+
+  async batchUpsert(docs: UpsertDocument[], namespace?: string): Promise<void> {
+    const payload = {
+      collectionName: this.indexName,
+      data: docs.map(doc => ({
+        vector: doc.vector,
+        content: doc.content,
+        metadata: doc.metadata || {},
+        ...(namespace ? { namespace } : {}),
+      })),
+    };
+    await this.http.post('/v1/vector/upsert', payload);
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  async query(vector: number[], topK: number, namespace?: string, _filter?: Record<string, unknown>): Promise<VectorMatch[]> {
+    const payload = {
+      collectionName: this.indexName,
+      vector: vector,
+      limit: topK,
+      filter: namespace ? `namespace == "${namespace}"` : undefined,
+      outputFields: ['content', 'metadata'],
+    };
+    const { data } = await this.http.post('/v1/vector/search', payload);
+    return (data.data || []).map((res: Record<string, unknown>) => ({
+      id: String(res['id']),
+      score: res['distance'] as number,
+      content: res['content'] as string,
+      metadata: res['metadata'] as Record<string, unknown>,
+    }));
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  async delete(id: string, _namespace?: string): Promise<void> {
+    await this.http.post('/v1/vector/delete', {
+      collectionName: this.indexName,
+      id: id,
+    });
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  async deleteNamespace(_namespace: string): Promise<void> {
+    // Milvus doesn't have a direct "delete namespace" REST API easily
+    // usually handled by deleting with a filter
+  }
+
+  async ping(): Promise<boolean> {
+    try {
+      await this.http.get('/v1/health');
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
+  async disconnect(): Promise<void> {}
+}

package/src/providers/vectordb/MongoDBProvider.ts

@@ -0,0 +1,118 @@
+import { MongoClient, Collection, Db } from 'mongodb';
+import { VectorDBConfig } from '../../config/RagConfig';
+import { BaseVectorProvider } from './BaseVectorProvider';
+import { VectorMatch, UpsertDocument } from '../../types';
+
+/**
+ * MongoDBProvider — MongoDB Atlas Vector Search implementation.
+ */
+export class MongoDBProvider extends BaseVectorProvider {
+  private client: MongoClient;
+  private db?: Db;
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  private collection?: Collection<any>;
+  private dbName: string;
+  private collectionName: string;
+  private embeddingKey: string;
+  private contentKey: string;
+  private metadataKey: string;
+
+  constructor(config: VectorDBConfig) {
+    super(config);
+    const opts = config.options as Record<string, unknown>;
+    if (!opts.uri || !opts.database || !opts.collection) {
+      throw new Error('[MongoDBProvider] options uri, database, and collection are required.');
+    }
+    this.client = new MongoClient(opts.uri as string);
+    this.dbName = opts.database as string;
+    this.collectionName = opts.collection as string;
+    this.embeddingKey = (opts.embeddingKey as string) || 'embedding';
+    this.contentKey = (opts.contentKey as string) || 'content';
+    this.metadataKey = (opts.metadataKey as string) || 'metadata';
+  }
+
+  async initialize(): Promise<void> {
+    await this.client.connect();
+    this.db = this.client.db(this.dbName);
+    this.collection = this.db.collection(this.collectionName);
+  }
+
+  async upsert(doc: UpsertDocument, namespace?: string): Promise<void> {
+    const document = {
+      _id: doc.id,
+      [this.embeddingKey]: doc.vector,
+      [this.contentKey]: doc.content,
+      [this.metadataKey]: doc.metadata || {},
+      ...(namespace ? { namespace } : {}),
+    };
+    await this.collection!.updateOne({ _id: doc.id }, { $set: document }, { upsert: true });
+  }
+
+  async batchUpsert(docs: UpsertDocument[], namespace?: string): Promise<void> {
+    const operations = docs.map((doc) => ({
+      updateOne: {
+        filter: { _id: doc.id },
+        update: {
+          $set: {
+            [this.embeddingKey]: doc.vector,
+            [this.contentKey]: doc.content,
+            [this.metadataKey]: doc.metadata || {},
+            ...(namespace ? { namespace } : {}),
+          },
+        },
+        upsert: true,
+      },
+    }));
+    await this.collection!.bulkWrite(operations);
+  }
+
+  async query(vector: number[], topK: number, namespace?: string, filter?: Record<string, unknown>): Promise<VectorMatch[]> {
+    const pipeline: Record<string, unknown>[] = [
+      {
+        $vectorSearch: {
+          index: this.config.options.indexName || 'vector_index',
+          path: this.embeddingKey,
+          queryVector: vector,
+          numCandidates: Math.max(topK * 10, 100),
+          limit: topK,
+          ...(filter || namespace ? { filter: { ...(filter || {}), ...(namespace ? { namespace } : {}) } } : {}),
+        },
+      },
+      {
+        $project: {
+          score: { $meta: 'vectorSearchScore' },
+          content: `$${this.contentKey}`,
+          metadata: `$${this.metadataKey}`,
+        },
+      },
+    ];
+    const results = await this.collection!.aggregate(pipeline).toArray();
+    return results.map((res: Record<string, unknown>) => ({
+      id: res['_id']!.toString(),
+      score: res['score'] as number,
+      content: res['content'] as string,
+      metadata: res['metadata'] as Record<string, unknown>,
+    }));
+  }
+
+  async delete(id: string, namespace?: string): Promise<void> {
+    await this.collection!.deleteOne({ _id: id, ...(namespace ? { namespace } : {}) });
+  }
+
+  async deleteNamespace(namespace: string): Promise<void> {
+    await this.collection!.deleteMany({ namespace });
+  }
+
+  async ping(): Promise<boolean> {
+    try {
+      await this.db!.command({ ping: 1 });
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
+  async disconnect(): Promise<void> {
+    await this.client.close();
+  }
+}