@retrivora-ai/rag-engine 0.4.5 → 1.0.0

package/src/rag/DocumentChunker.ts

@@ -32,7 +32,11 @@ export class DocumentChunker {
   private readonly chunkOverlap: number;
   private readonly separators: string[];
 
-  constructor(chunkSize = 1000, chunkOverlap = 200, separators = ['\n\n', '\n', ' ', '']) {
+  constructor(
+    chunkSize = 1000, 
+    chunkOverlap = 200, 
+    separators = ['\n# ', '\n## ', '\n### ', '\n#### ', '\n\n', '\n', ' ', '']
+  ) {
     this.chunkSize = chunkSize;
     this.chunkOverlap = chunkOverlap;
     this.separators = separators;
@@ -40,6 +44,7 @@ export class DocumentChunker {
 
   /**
    * Split a single text string into overlapping chunks using a recursive strategy.
+   * Preserves structural boundaries (Markdown headers) where possible.
    */
   chunk(text: string, options: ChunkOptions = {}): Chunk[] {
     const {
@@ -58,19 +63,18 @@ export class DocumentChunker {
     let chunkIndex = 0;
 
     for (const split of splits) {
+      // If adding this split exceeds chunk size, and we already have content, close the chunk
       if (currentLength + split.length > chunkSize && currentChunk.length > 0) {
-        // Current chunk is full, push it
         finalChunks.push({
           id: `${docId}_chunk_${chunkIndex++}`,
           content: currentChunk.join('').trim(),
           metadata: { ...metadata, docId, chunkIndex: chunkIndex - 1 },
         });
 
-        // Start new chunk with overlap
-        // This is a simplified overlap: we keep items from the end of the previous chunk
-        // until we reach the overlap size.
+        // Sliding window overlap logic
         const overlapItems = [];
         let overlapLen = 0;
+        // Search backwards through current chunk to find segments that fit within the overlap window
         for (let i = currentChunk.length - 1; i >= 0; i--) {
           if (overlapLen + currentChunk[i].length <= chunkOverlap) {
             overlapItems.unshift(currentChunk[i]);
@@ -99,35 +103,30 @@ export class DocumentChunker {
     return finalChunks;
   }
 
-  /**
-   * Recursively split text based on separators.
-   */
   private recursiveSplit(text: string, separators: string[], chunkSize: number): string[] {
     const finalSplits: string[] = [];
     
-    // Find the first separator that exists in the text
-    let separator = separators[separators.length - 1]; // default to last (usually empty string)
+    let separator = separators[separators.length - 1];
     let nextSeparators: string[] = [];
 
     for (let i = 0; i < separators.length; i++) {
-      if (text.includes(separators[i])) {
-        separator = separators[i];
+      const sep = separators[i];
+      if (text.includes(sep)) {
+        separator = sep;
         nextSeparators = separators.slice(i + 1);
         break;
       }
     }
 
-    // Split the text
     const parts = text.split(separator);
     
     for (const part of parts) {
       if (part.length <= chunkSize) {
-        finalSplits.push(part + separator);
+        // Keep the separator to preserve formatting/meaning
+        finalSplits.push(part + (part === parts[parts.length - 1] ? '' : separator));
       } else if (nextSeparators.length > 0) {
-        // Part is too long, try splitting with next level separators
         finalSplits.push(...this.recursiveSplit(part, nextSeparators, chunkSize));
       } else {
-        // No more separators, just hard split (should not happen if empty string is the last separator)
         finalSplits.push(part);
       }
     }
@@ -135,9 +134,6 @@ export class DocumentChunker {
     return finalSplits;
   }
 
-  /**
-   * Chunk multiple documents at once.
-   */
   chunkMany(
     documents: Array<{ content: string; docId?: string | number; metadata?: Record<string, unknown> }>
   ): Chunk[] {

package/src/rag/LlamaIndexIngestor.ts

@@ -0,0 +1,61 @@
+import { Chunk, ChunkOptions } from './DocumentChunker';
+
+/**
+ * LlamaIndexIngestor — Advanced ingestion provider using LlamaIndex (TS).
+ * 
+ * Provides semantic chunking, metadata extraction, and node parsing 
+ * superior to basic recursive character splitting.
+ * 
+ * Note: Requires 'llamaindex' package to be installed.
+ */
+export class LlamaIndexIngestor {
+  /**
+   * Chunks document content using LlamaIndex SentenceSplitter.
+   * This respects sentence and paragraph boundaries much more effectively
+   * than standard character-count splitting.
+   */
+  async chunk(text: string, options: ChunkOptions = {}): Promise<Chunk[]> {
+    try {
+      // Dynamically import for optional dependency support using template string to hide from TS static analysis
+      const { Document, SentenceSplitter, MetadataMode } = await import(`${'llamaindex'}`);
+
+      const splitter = new SentenceSplitter({
+        chunkSize: options.chunkSize ?? 1000,
+        chunkOverlap: options.chunkOverlap ?? 200,
+      });
+
+      const doc = new Document({ text, metadata: options.metadata || {} });
+      const nodes = splitter.getNodesFromDocuments([doc]);
+
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      return (nodes as any[]).map((node: any, index: number) => ({
+        id: `${options.docId || 'doc'}_node_${index}`,
+        content: node.getContent(MetadataMode.ALL),
+        metadata: {
+          ...options.metadata,
+          ...node.metadata,
+          nodeId: node.id_,
+          chunkIndex: index
+        }
+      }));
+    } catch (error) {
+      console.warn('[LlamaIndexIngestor] LlamaIndex package not found or failed. Falling back to default chunker.');
+      throw error;
+    }
+  }
+
+  /**
+   * Batch processing for multiple documents.
+   */
+  async chunkMany(
+    documents: Array<{ content: string; docId?: string | number; metadata?: Record<string, unknown> }>,
+    options: ChunkOptions = {}
+  ): Promise<Chunk[]> {
+    const allChunks: Chunk[] = [];
+    for (const doc of documents) {
+      const chunks = await this.chunk(doc.content, { ...options, docId: doc.docId, metadata: doc.metadata });
+      allChunks.push(...chunks);
+    }
+    return allChunks;
+  }
+}

package/src/rag/Reranker.ts

@@ -0,0 +1,20 @@
+import { VectorMatch } from '../types';
+
+export class Reranker {
+  /**
+   * Re-ranks matches based on a secondary relevance score.
+   * In a production environment, this would call a Cross-Encoder model.
+   * Here we implement a placeholder that filters by score and limits count.
+   */
+  async rerank(
+    matches: VectorMatch[], 
+    query: string, 
+    limit: number = 5
+  ): Promise<VectorMatch[]> {
+    // Current implementation: sort by original score and apply a strict threshold
+    // and semantic match filtering (placeholder)
+    return matches
+      .sort((a, b) => b.score - a.score)
+      .slice(0, limit);
+  }
+}

package/src/server.ts

@@ -10,7 +10,7 @@ export type { RagConfig, VectorDBConfig, LLMConfig, EmbeddingConfig, UIConfig, R
 export type { VectorMatch, UpsertDocument, IngestDocument, ChatResponse } from './types';
 export type { ILLMProvider, ChatMessage, ChatOptions, EmbedOptions } from './llm/ILLMProvider';
 export type { Chunk, ChunkOptions } from './rag/DocumentChunker';
-export type { HealthCheckResult } from './core/ProviderHealthCheck';
+export type { HealthCheckResult, IProviderValidator, IProviderHealthChecker } from './core/ProviderInterfaces';
 export type { ValidationError } from './core/ConfigValidator';
 export type { BatchOptions, BatchResult } from './core/BatchProcessor';
 

package/src/types/index.ts

@@ -41,3 +41,12 @@ export interface GraphSearchResult {
   nodes: GraphNode[];
   edges: Edge[];
 }
+
+export interface RetrievalResult {
+  sources: VectorMatch[];
+  graphData?: GraphSearchResult;
+}
+
+export interface IRetriever {
+  retrieve(query: string, options?: Record<string, unknown>): Promise<RetrievalResult>;
+}

package/dist/chunk-FWCSY2DS.mjs

@@ -1,37 +0,0 @@
-var __defProp = Object.defineProperty;
-var __defProps = Object.defineProperties;
-var __getOwnPropDescs = Object.getOwnPropertyDescriptors;
-var __getOwnPropSymbols = Object.getOwnPropertySymbols;
-var __hasOwnProp = Object.prototype.hasOwnProperty;
-var __propIsEnum = Object.prototype.propertyIsEnumerable;
-var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
-var __spreadValues = (a, b) => {
-  for (var prop in b || (b = {}))
-    if (__hasOwnProp.call(b, prop))
-      __defNormalProp(a, prop, b[prop]);
-  if (__getOwnPropSymbols)
-    for (var prop of __getOwnPropSymbols(b)) {
-      if (__propIsEnum.call(b, prop))
-        __defNormalProp(a, prop, b[prop]);
-    }
-  return a;
-};
-var __spreadProps = (a, b) => __defProps(a, __getOwnPropDescs(b));
-var __objRest = (source, exclude) => {
-  var target = {};
-  for (var prop in source)
-    if (__hasOwnProp.call(source, prop) && exclude.indexOf(prop) < 0)
-      target[prop] = source[prop];
-  if (source != null && __getOwnPropSymbols)
-    for (var prop of __getOwnPropSymbols(source)) {
-      if (exclude.indexOf(prop) < 0 && __propIsEnum.call(source, prop))
-        target[prop] = source[prop];
-    }
-  return target;
-};
-
-export {
-  __spreadValues,
-  __spreadProps,
-  __objRest
-};

package/dist/index-7qeLTPBL.d.mts

@@ -1,114 +0,0 @@
-import { e as VectorDBConfig, L as LLMConfig, E as EmbeddingConfig, c as RagConfig, C as ChatResponse } from './RagConfig-BgRDL9Vy.mjs';
-import { NextRequest, NextResponse } from 'next/server';
-
-/**
- * ProviderHealthCheck.ts — Pre-flight validation for vector DB and LLM providers.
- *
- * Performs connectivity tests, credential validation, and capability checks
- * before initializing providers.
- *
- * Note: Option keys used here mirror each Provider constructor's expected keys.
- */
-
-interface HealthCheckResult {
-    healthy: boolean;
-    provider: string;
-    version?: string;
-    capabilities?: Record<string, unknown>;
-    error?: string;
-    timestamp: number;
-}
-declare class ProviderHealthCheck {
-    /**
-     * Validates vector database configuration before initialization.
-     */
-    static checkVectorProvider(config: VectorDBConfig): Promise<HealthCheckResult>;
-    private static checkPinecone;
-    private static checkPostgres;
-    private static checkMongoDB;
-    /**
-     * Milvus health check — uses baseUrl/uri (matching MilvusProvider constructor).
-     */
-    private static checkMilvus;
-    /**
-     * Qdrant health check — uses baseUrl (matching QdrantProvider constructor).
-     */
-    private static checkQdrant;
-    /**
-     * ChromaDB health check — uses baseUrl/host (matching ChromaDBProvider constructor).
-     */
-    private static checkChromaDB;
-    /**
-     * Redis health check — Redis is TCP-only (no HTTP endpoint).
-     * We report healthy=true with a note; actual connectivity is validated at first operation.
-     */
-    private static checkRedis;
-    /**
-     * Weaviate health check — uses baseUrl/url (matching WeaviateProvider constructor).
-     */
-    private static checkWeaviate;
-    private static checkRestAPI;
-    /**
-     * Validates LLM provider configuration.
-     */
-    static checkLLMProvider(config: LLMConfig): Promise<HealthCheckResult>;
-    private static checkOpenAI;
-    private static checkAnthropic;
-    private static checkOllama;
-    private static checkLLMRestAPI;
-    /**
-     * Runs comprehensive health checks on all configured providers in parallel.
-     */
-    static checkAll(vectorDbConfig: VectorDBConfig, llmConfig: LLMConfig, embeddingConfig?: EmbeddingConfig): Promise<{
-        vectorDb: HealthCheckResult;
-        llm: HealthCheckResult;
-        embedding?: HealthCheckResult;
-        allHealthy: boolean;
-    }>;
-}
-
-/**
- * createChatHandler — factory that returns a Next.js App Router POST handler
- */
-declare function createChatHandler(config?: Partial<RagConfig>): (req: NextRequest) => Promise<NextResponse<{
-    error: string;
-}> | NextResponse<ChatResponse>>;
-/**
- * createIngestHandler — factory for the document ingestion endpoint.
- */
-declare function createIngestHandler(config?: Partial<RagConfig>): (req: NextRequest) => Promise<NextResponse<{
-    error: string;
-}> | NextResponse<{
-    results: {
-        docId: string | number;
-        chunksIngested: number;
-    }[];
-}>>;
-/**
- * createHealthHandler — factory for the health-check endpoint.
- */
-declare function createHealthHandler(config?: Partial<RagConfig>): () => Promise<NextResponse<{
-    timestamp: string;
-    vectorDb: HealthCheckResult;
-    llm: HealthCheckResult;
-    embedding?: HealthCheckResult;
-    allHealthy: boolean;
-    status: string;
-}> | NextResponse<{
-    status: string;
-    error: string;
-}>>;
-/**
- * createUploadHandler — factory for the file upload ingestion endpoint.
- */
-declare function createUploadHandler(config?: Partial<RagConfig>): (req: NextRequest) => Promise<NextResponse<{
-    error: string;
-}> | NextResponse<{
-    message: string;
-    results: {
-        docId: string | number;
-        chunksIngested: number;
-    }[];
-}>>;
-
-export { type HealthCheckResult as H, ProviderHealthCheck as P, createHealthHandler as a, createIngestHandler as b, createChatHandler as c, createUploadHandler as d };

package/dist/index-DowY4_K0.d.ts

@@ -1,114 +0,0 @@
-import { e as VectorDBConfig, L as LLMConfig, E as EmbeddingConfig, c as RagConfig, C as ChatResponse } from './RagConfig-BgRDL9Vy.js';
-import { NextRequest, NextResponse } from 'next/server';
-
-/**
- * ProviderHealthCheck.ts — Pre-flight validation for vector DB and LLM providers.
- *
- * Performs connectivity tests, credential validation, and capability checks
- * before initializing providers.
- *
- * Note: Option keys used here mirror each Provider constructor's expected keys.
- */
-
-interface HealthCheckResult {
-    healthy: boolean;
-    provider: string;
-    version?: string;
-    capabilities?: Record<string, unknown>;
-    error?: string;
-    timestamp: number;
-}
-declare class ProviderHealthCheck {
-    /**
-     * Validates vector database configuration before initialization.
-     */
-    static checkVectorProvider(config: VectorDBConfig): Promise<HealthCheckResult>;
-    private static checkPinecone;
-    private static checkPostgres;
-    private static checkMongoDB;
-    /**
-     * Milvus health check — uses baseUrl/uri (matching MilvusProvider constructor).
-     */
-    private static checkMilvus;
-    /**
-     * Qdrant health check — uses baseUrl (matching QdrantProvider constructor).
-     */
-    private static checkQdrant;
-    /**
-     * ChromaDB health check — uses baseUrl/host (matching ChromaDBProvider constructor).
-     */
-    private static checkChromaDB;
-    /**
-     * Redis health check — Redis is TCP-only (no HTTP endpoint).
-     * We report healthy=true with a note; actual connectivity is validated at first operation.
-     */
-    private static checkRedis;
-    /**
-     * Weaviate health check — uses baseUrl/url (matching WeaviateProvider constructor).
-     */
-    private static checkWeaviate;
-    private static checkRestAPI;
-    /**
-     * Validates LLM provider configuration.
-     */
-    static checkLLMProvider(config: LLMConfig): Promise<HealthCheckResult>;
-    private static checkOpenAI;
-    private static checkAnthropic;
-    private static checkOllama;
-    private static checkLLMRestAPI;
-    /**
-     * Runs comprehensive health checks on all configured providers in parallel.
-     */
-    static checkAll(vectorDbConfig: VectorDBConfig, llmConfig: LLMConfig, embeddingConfig?: EmbeddingConfig): Promise<{
-        vectorDb: HealthCheckResult;
-        llm: HealthCheckResult;
-        embedding?: HealthCheckResult;
-        allHealthy: boolean;
-    }>;
-}
-
-/**
- * createChatHandler — factory that returns a Next.js App Router POST handler
- */
-declare function createChatHandler(config?: Partial<RagConfig>): (req: NextRequest) => Promise<NextResponse<{
-    error: string;
-}> | NextResponse<ChatResponse>>;
-/**
- * createIngestHandler — factory for the document ingestion endpoint.
- */
-declare function createIngestHandler(config?: Partial<RagConfig>): (req: NextRequest) => Promise<NextResponse<{
-    error: string;
-}> | NextResponse<{
-    results: {
-        docId: string | number;
-        chunksIngested: number;
-    }[];
-}>>;
-/**
- * createHealthHandler — factory for the health-check endpoint.
- */
-declare function createHealthHandler(config?: Partial<RagConfig>): () => Promise<NextResponse<{
-    timestamp: string;
-    vectorDb: HealthCheckResult;
-    llm: HealthCheckResult;
-    embedding?: HealthCheckResult;
-    allHealthy: boolean;
-    status: string;
-}> | NextResponse<{
-    status: string;
-    error: string;
-}>>;
-/**
- * createUploadHandler — factory for the file upload ingestion endpoint.
- */
-declare function createUploadHandler(config?: Partial<RagConfig>): (req: NextRequest) => Promise<NextResponse<{
-    error: string;
-}> | NextResponse<{
-    message: string;
-    results: {
-        docId: string | number;
-        chunksIngested: number;
-    }[];
-}>>;
-
-export { type HealthCheckResult as H, ProviderHealthCheck as P, createHealthHandler as a, createIngestHandler as b, createChatHandler as c, createUploadHandler as d };