@retrivora-ai/rag-engine 1.9.2 → 1.9.6

package/dist/index-C9v7-tWd.d.mts

@@ -0,0 +1,183 @@
+import { k as RagConfig, I as ILLMProvider, g as IngestDocument, C as ChatMessage, d as ChatResponse, u as RetrievalResult, n as UniversalRagConfig } from './ILLMProvider-DNhyOYoK.mjs';
+
+/**
+ * DocumentChunker — splits long text into overlapping chunks
+ * suitable for vector embedding and retrieval.
+ *
+ * Strategy: sentence-aware sliding window
+ *   1. Split on sentence boundaries
+ *   2. Accumulate sentences into chunks up to `chunkSize` characters
+ *   3. Carry over `chunkOverlap` characters from the previous chunk
+ */
+interface Chunk {
+    id: string;
+    content: string;
+    metadata?: Record<string, unknown>;
+}
+interface ChunkOptions {
+    /** Target chunk size in characters (default 1000) */
+    chunkSize?: number;
+    /** Overlap between consecutive chunks in characters (default 200) */
+    chunkOverlap?: number;
+    /** Source document identifier used as ID prefix */
+    docId?: string | number;
+    /** Extra metadata to attach to every chunk */
+    metadata?: Record<string, unknown>;
+    /** Characters used to split text, in order of priority */
+    separators?: string[];
+}
+declare class DocumentChunker {
+    private readonly chunkSize;
+    private readonly chunkOverlap;
+    private readonly separators;
+    constructor(chunkSize?: number, chunkOverlap?: number, separators?: string[]);
+    /**
+     * Split a single text string into overlapping chunks using a recursive strategy.
+     * Preserves structural boundaries (Markdown headers) where possible.
+     */
+    chunk(text: string, options?: ChunkOptions): Chunk[];
+    private recursiveSplit;
+    chunkMany(documents: Array<{
+        content: string;
+        docId?: string | number;
+        metadata?: Record<string, unknown>;
+    }>): Chunk[];
+}
+
+/**
+ * Pipeline — orchestrates the RAG flow: Embed → Search → Augment → Generate.
+ *
+ * Features:
+ * - Lazy initialization of providers
+ * - Batch processing with retry logic
+ * - Smart embedding strategy (integrated / separate / external)
+ * - Error recovery for transient failures
+ * - Multi-tenancy support via namespacing
+ * - LRU-bounded embedding cache (max 500 entries, prevents memory leaks)
+ * - Full observability tracing (latency, tokens, hallucination scoring)
+ */
+declare class Pipeline {
+    private config;
+    private vectorDB;
+    private graphDB?;
+    private llmProvider;
+    private embeddingProvider;
+    private llmRouter;
+    private chunker;
+    private llamaIngestor?;
+    private entityExtractor?;
+    private reranker;
+    private agent?;
+    /** LRU-bounded cache: avoids re-embedding identical queries within the same process. */
+    private embeddingCache;
+    private initialised;
+    constructor(config: RagConfig);
+    /**
+     * Expose the underlying LLM provider (set after initialize()).
+     * Used by the stream handler to pass to UITransformer.analyzeAndDecide().
+     */
+    getLLMProvider(): ILLMProvider | undefined;
+    initialize(): Promise<void>;
+    /**
+     * Ingest documents with automatic chunking, embedding, and batch upsert.
+     */
+    ingest(documents: IngestDocument[], namespace?: string): Promise<Array<{
+        docId: string | number;
+        chunksIngested: number;
+    }>>;
+    /** Step 1: Chunk the document content. */
+    private prepareChunks;
+    /**
+     * Step 2: Generate embeddings for chunks with retry logic.
+     * Uses batchEmbed when available for efficiency; falls back to sequential embedding.
+     */
+    private processEmbeddings;
+    /** Step 3: Upsert chunks to vector database with retry logic. */
+    private processUpserts;
+    /** Step 4: Optional graph-based entity extraction and ingestion. */
+    private processGraphIngestion;
+    ask(question: string, history?: ChatMessage[], namespace?: string): Promise<ChatResponse>;
+    /**
+     * High-performance streaming RAG flow.
+     * Yields text chunks first, then the retrieval metadata + observability trace at the end.
+     *
+     * Latency optimizations:
+     * - Strategy classification runs in parallel with query embedding (saves ~400ms)
+     * - Hallucination scoring is fire-and-forget (doesn't block metadata yield)
+     * - UITransformation is computed after text streaming and emitted with metadata
+     * - SchemaMapper.train runs while answer generation streams
+     */
+    askStream(question: string, history?: ChatMessage[], namespace?: string): AsyncIterable<string | ChatResponse>;
+    /**
+     * Universal retrieval method combining all enabled providers.
+     * Uses an LRU-bounded embedding cache to avoid re-embedding the same query.
+     */
+    private generateUiTransformation;
+    private applyStructuredFilters;
+    private resolveNumericPredicateValue;
+    private extractNumericValueFromContent;
+    private matchesNumericPredicate;
+    private normalizeComparableField;
+    private fieldSimilarityScore;
+    private fieldTokens;
+    private toFiniteNumber;
+    retrieve(query: string, options: {
+        namespace?: string;
+        topK?: number;
+        filter?: Record<string, unknown>;
+    }): Promise<RetrievalResult>;
+    /** Rewrite the user query for better retrieval performance. */
+    private rewriteQuery;
+    /** Generate 3-5 short, relevant questions based on the vector database content. */
+    getSuggestions(query: string, namespace?: string): Promise<string[]>;
+}
+
+/**
+ * Public SDK facade matching the prompt-level Retrivora API.
+ *
+ * It keeps provider details behind configuration and delegates execution to the
+ * existing Pipeline implementation.
+ */
+declare class Retrivora {
+    private readonly pipeline;
+    readonly config: RagConfig;
+    constructor(config?: UniversalRagConfig);
+    initialize(): Promise<void>;
+    ingest(documents: IngestDocument[], namespace?: string): Promise<Array<{
+        docId: string | number;
+        chunksIngested: number;
+    }>>;
+    ask(question: string, history?: ChatMessage[], namespace?: string): Promise<ChatResponse>;
+    askStream(question: string, history?: ChatMessage[], namespace?: string): AsyncIterable<string | ChatResponse>;
+    getPipeline(): Pipeline;
+}
+
+/**
+ * Named SDK exceptions make failures machine-readable for host applications.
+ */
+type RetrivoraErrorCode = 'PROVIDER_NOT_FOUND' | 'EMBEDDING_FAILED' | 'RETRIEVAL_FAILED' | 'RATE_LIMITED' | 'CONFIGURATION_ERROR' | 'AUTHENTICATION_ERROR';
+declare class RetrivoraError extends Error {
+    readonly code: RetrivoraErrorCode;
+    readonly details?: unknown;
+    constructor(message: string, code: RetrivoraErrorCode, details?: unknown);
+}
+declare class ProviderNotFoundException extends RetrivoraError {
+    constructor(providerType: string, provider: string, details?: unknown);
+}
+declare class EmbeddingFailedException extends RetrivoraError {
+    constructor(message?: string, details?: unknown);
+}
+declare class RetrievalException extends RetrivoraError {
+    constructor(message?: string, details?: unknown);
+}
+declare class RateLimitException extends RetrivoraError {
+    constructor(message?: string, details?: unknown);
+}
+declare class ConfigurationException extends RetrivoraError {
+    constructor(message: string, details?: unknown);
+}
+declare class AuthenticationException extends RetrivoraError {
+    constructor(message?: string, details?: unknown);
+}
+
+export { AuthenticationException as A, type Chunk as C, DocumentChunker as D, EmbeddingFailedException as E, ProviderNotFoundException as P, RateLimitException as R, type ChunkOptions as a, ConfigurationException as b, RetrievalException as c, Retrivora as d, RetrivoraError as e, type RetrivoraErrorCode as f, Pipeline as g };

package/dist/{index-B70ZLkfG.d.mts → index-CHL1jdYm.d.mts}

@@ -1,4 +1,4 @@
-import { k as RagConfig, I as ILLMProvider, C as ChatMessage, d as ChatResponse, g as IngestDocument } from './ILLMProvider-Bw2A28nU.mjs';
+import { k as RagConfig, I as ILLMProvider, C as ChatMessage, d as ChatResponse, g as IngestDocument } from './ILLMProvider-DNhyOYoK.mjs';
 import { NextRequest, NextResponse } from 'next/server';
 
 interface ValidationError {

package/dist/index-CjQdL0cX.d.ts

@@ -0,0 +1,183 @@
+import { k as RagConfig, I as ILLMProvider, g as IngestDocument, C as ChatMessage, d as ChatResponse, u as RetrievalResult, n as UniversalRagConfig } from './ILLMProvider-DNhyOYoK.js';
+
+/**
+ * DocumentChunker — splits long text into overlapping chunks
+ * suitable for vector embedding and retrieval.
+ *
+ * Strategy: sentence-aware sliding window
+ *   1. Split on sentence boundaries
+ *   2. Accumulate sentences into chunks up to `chunkSize` characters
+ *   3. Carry over `chunkOverlap` characters from the previous chunk
+ */
+interface Chunk {
+    id: string;
+    content: string;
+    metadata?: Record<string, unknown>;
+}
+interface ChunkOptions {
+    /** Target chunk size in characters (default 1000) */
+    chunkSize?: number;
+    /** Overlap between consecutive chunks in characters (default 200) */
+    chunkOverlap?: number;
+    /** Source document identifier used as ID prefix */
+    docId?: string | number;
+    /** Extra metadata to attach to every chunk */
+    metadata?: Record<string, unknown>;
+    /** Characters used to split text, in order of priority */
+    separators?: string[];
+}
+declare class DocumentChunker {
+    private readonly chunkSize;
+    private readonly chunkOverlap;
+    private readonly separators;
+    constructor(chunkSize?: number, chunkOverlap?: number, separators?: string[]);
+    /**
+     * Split a single text string into overlapping chunks using a recursive strategy.
+     * Preserves structural boundaries (Markdown headers) where possible.
+     */
+    chunk(text: string, options?: ChunkOptions): Chunk[];
+    private recursiveSplit;
+    chunkMany(documents: Array<{
+        content: string;
+        docId?: string | number;
+        metadata?: Record<string, unknown>;
+    }>): Chunk[];
+}
+
+/**
+ * Pipeline — orchestrates the RAG flow: Embed → Search → Augment → Generate.
+ *
+ * Features:
+ * - Lazy initialization of providers
+ * - Batch processing with retry logic
+ * - Smart embedding strategy (integrated / separate / external)
+ * - Error recovery for transient failures
+ * - Multi-tenancy support via namespacing
+ * - LRU-bounded embedding cache (max 500 entries, prevents memory leaks)
+ * - Full observability tracing (latency, tokens, hallucination scoring)
+ */
+declare class Pipeline {
+    private config;
+    private vectorDB;
+    private graphDB?;
+    private llmProvider;
+    private embeddingProvider;
+    private llmRouter;
+    private chunker;
+    private llamaIngestor?;
+    private entityExtractor?;
+    private reranker;
+    private agent?;
+    /** LRU-bounded cache: avoids re-embedding identical queries within the same process. */
+    private embeddingCache;
+    private initialised;
+    constructor(config: RagConfig);
+    /**
+     * Expose the underlying LLM provider (set after initialize()).
+     * Used by the stream handler to pass to UITransformer.analyzeAndDecide().
+     */
+    getLLMProvider(): ILLMProvider | undefined;
+    initialize(): Promise<void>;
+    /**
+     * Ingest documents with automatic chunking, embedding, and batch upsert.
+     */
+    ingest(documents: IngestDocument[], namespace?: string): Promise<Array<{
+        docId: string | number;
+        chunksIngested: number;
+    }>>;
+    /** Step 1: Chunk the document content. */
+    private prepareChunks;
+    /**
+     * Step 2: Generate embeddings for chunks with retry logic.
+     * Uses batchEmbed when available for efficiency; falls back to sequential embedding.
+     */
+    private processEmbeddings;
+    /** Step 3: Upsert chunks to vector database with retry logic. */
+    private processUpserts;
+    /** Step 4: Optional graph-based entity extraction and ingestion. */
+    private processGraphIngestion;
+    ask(question: string, history?: ChatMessage[], namespace?: string): Promise<ChatResponse>;
+    /**
+     * High-performance streaming RAG flow.
+     * Yields text chunks first, then the retrieval metadata + observability trace at the end.
+     *
+     * Latency optimizations:
+     * - Strategy classification runs in parallel with query embedding (saves ~400ms)
+     * - Hallucination scoring is fire-and-forget (doesn't block metadata yield)
+     * - UITransformation is computed after text streaming and emitted with metadata
+     * - SchemaMapper.train runs while answer generation streams
+     */
+    askStream(question: string, history?: ChatMessage[], namespace?: string): AsyncIterable<string | ChatResponse>;
+    /**
+     * Universal retrieval method combining all enabled providers.
+     * Uses an LRU-bounded embedding cache to avoid re-embedding the same query.
+     */
+    private generateUiTransformation;
+    private applyStructuredFilters;
+    private resolveNumericPredicateValue;
+    private extractNumericValueFromContent;
+    private matchesNumericPredicate;
+    private normalizeComparableField;
+    private fieldSimilarityScore;
+    private fieldTokens;
+    private toFiniteNumber;
+    retrieve(query: string, options: {
+        namespace?: string;
+        topK?: number;
+        filter?: Record<string, unknown>;
+    }): Promise<RetrievalResult>;
+    /** Rewrite the user query for better retrieval performance. */
+    private rewriteQuery;
+    /** Generate 3-5 short, relevant questions based on the vector database content. */
+    getSuggestions(query: string, namespace?: string): Promise<string[]>;
+}
+
+/**
+ * Public SDK facade matching the prompt-level Retrivora API.
+ *
+ * It keeps provider details behind configuration and delegates execution to the
+ * existing Pipeline implementation.
+ */
+declare class Retrivora {
+    private readonly pipeline;
+    readonly config: RagConfig;
+    constructor(config?: UniversalRagConfig);
+    initialize(): Promise<void>;
+    ingest(documents: IngestDocument[], namespace?: string): Promise<Array<{
+        docId: string | number;
+        chunksIngested: number;
+    }>>;
+    ask(question: string, history?: ChatMessage[], namespace?: string): Promise<ChatResponse>;
+    askStream(question: string, history?: ChatMessage[], namespace?: string): AsyncIterable<string | ChatResponse>;
+    getPipeline(): Pipeline;
+}
+
+/**
+ * Named SDK exceptions make failures machine-readable for host applications.
+ */
+type RetrivoraErrorCode = 'PROVIDER_NOT_FOUND' | 'EMBEDDING_FAILED' | 'RETRIEVAL_FAILED' | 'RATE_LIMITED' | 'CONFIGURATION_ERROR' | 'AUTHENTICATION_ERROR';
+declare class RetrivoraError extends Error {
+    readonly code: RetrivoraErrorCode;
+    readonly details?: unknown;
+    constructor(message: string, code: RetrivoraErrorCode, details?: unknown);
+}
+declare class ProviderNotFoundException extends RetrivoraError {
+    constructor(providerType: string, provider: string, details?: unknown);
+}
+declare class EmbeddingFailedException extends RetrivoraError {
+    constructor(message?: string, details?: unknown);
+}
+declare class RetrievalException extends RetrivoraError {
+    constructor(message?: string, details?: unknown);
+}
+declare class RateLimitException extends RetrivoraError {
+    constructor(message?: string, details?: unknown);
+}
+declare class ConfigurationException extends RetrivoraError {
+    constructor(message: string, details?: unknown);
+}
+declare class AuthenticationException extends RetrivoraError {
+    constructor(message?: string, details?: unknown);
+}
+
+export { AuthenticationException as A, type Chunk as C, DocumentChunker as D, EmbeddingFailedException as E, ProviderNotFoundException as P, RateLimitException as R, type ChunkOptions as a, ConfigurationException as b, RetrievalException as c, Retrivora as d, RetrivoraError as e, type RetrivoraErrorCode as f, Pipeline as g };

package/dist/{index-DVu-mkAM.d.ts → index-Hgbwl9X4.d.ts}

@@ -1,4 +1,4 @@
-import { k as RagConfig, I as ILLMProvider, C as ChatMessage, d as ChatResponse, g as IngestDocument } from './ILLMProvider-Bw2A28nU.js';
+import { k as RagConfig, I as ILLMProvider, C as ChatMessage, d as ChatResponse, g as IngestDocument } from './ILLMProvider-DNhyOYoK.js';
 import { NextRequest, NextResponse } from 'next/server';
 
 interface ValidationError {

package/dist/index.css

@@ -409,6 +409,9 @@
   .mx-auto {
     margin-inline: auto;
   }
+  .my-1\.5 {
+    margin-block: calc(var(--spacing) * 1.5);
+  }
   .my-2 {
     margin-block: calc(var(--spacing) * 2);
   }
@@ -607,6 +610,9 @@
   .h-16 {
     height: calc(var(--spacing) * 16);
   }
+  .h-28 {
+    height: calc(var(--spacing) * 28);
+  }
   .h-32 {
     height: calc(var(--spacing) * 32);
   }
@@ -637,6 +643,9 @@
   .max-h-56 {
     max-height: calc(var(--spacing) * 56);
   }
+  .max-h-60 {
+    max-height: calc(var(--spacing) * 60);
+  }
   .max-h-64 {
     max-height: calc(var(--spacing) * 64);
   }
@@ -712,9 +721,21 @@
   .w-16 {
     width: calc(var(--spacing) * 16);
   }
+  .w-20 {
+    width: calc(var(--spacing) * 20);
+  }
+  .w-28 {
+    width: calc(var(--spacing) * 28);
+  }
+  .w-36 {
+    width: calc(var(--spacing) * 36);
+  }
   .w-40 {
     width: calc(var(--spacing) * 40);
   }
+  .w-44 {
+    width: calc(var(--spacing) * 44);
+  }
   .w-\[400\%\] {
     width: 400%;
   }
@@ -1234,6 +1255,9 @@
       background-color: color-mix(in oklab, var(--color-emerald-50) 50%, transparent);
     }
   }
+  .bg-emerald-100 {
+    background-color: var(--color-emerald-100);
+  }
   .bg-emerald-500 {
     background-color: var(--color-emerald-500);
   }
@@ -1270,6 +1294,18 @@
   .bg-slate-50 {
     background-color: var(--color-slate-50);
   }
+  .bg-slate-50\/20 {
+    background-color: color-mix(in srgb, oklch(98.4% 0.003 247.858) 20%, transparent);
+    @supports (color: color-mix(in lab, red, red)) {
+      background-color: color-mix(in oklab, var(--color-slate-50) 20%, transparent);
+    }
+  }
+  .bg-slate-50\/50 {
+    background-color: color-mix(in srgb, oklch(98.4% 0.003 247.858) 50%, transparent);
+    @supports (color: color-mix(in lab, red, red)) {
+      background-color: color-mix(in oklab, var(--color-slate-50) 50%, transparent);
+    }
+  }
   .bg-slate-100 {
     background-color: var(--color-slate-100);
   }
@@ -1815,6 +1851,10 @@
     -webkit-user-select: none;
     user-select: none;
   }
+  .select-text {
+    -webkit-user-select: text;
+    user-select: text;
+  }
   .\[animation-delay\:0ms\] {
     animation-delay: 0ms;
   }

package/dist/index.d.mts

@@ -1,8 +1,8 @@
 import * as react_jsx_runtime from 'react/jsx-runtime';
 import { CSSProperties, MouseEvent, ReactNode } from 'react';
-import { P as Product, U as UIConfig, R as RagMessage, V as VectorMatch, O as ObservabilityTrace, a as UseRagChatOptions, b as UseRagChatReturn } from './ILLMProvider-Bw2A28nU.mjs';
-export { C as ChatMessage, c as ChatOptions, d as ChatResponse, E as EmbedOptions, e as EmbeddingConfig, f as EmbeddingProvider, I as ILLMProvider, g as IngestDocument, L as LLMConfig, h as LLMProvider, i as LatencyBreakdown, j as RAGConfig, k as RagConfig, l as RetrievedChunk, T as TokenUsage, m as UpsertDocument, n as VectorDBConfig, o as VectorDBProvider } from './ILLMProvider-Bw2A28nU.mjs';
-export { C as Chunk, a as ChunkOptions } from './DocumentChunker-Dh9TvmGG.mjs';
+import { P as Product, U as UIConfig, R as RagMessage, V as VectorMatch, O as ObservabilityTrace, a as UseRagChatOptions, b as UseRagChatReturn } from './ILLMProvider-DNhyOYoK.mjs';
+export { C as ChatMessage, c as ChatOptions, d as ChatResponse, E as EmbedOptions, e as EmbeddingConfig, f as EmbeddingProvider, I as ILLMProvider, g as IngestDocument, L as LLMConfig, h as LLMProvider, i as LatencyBreakdown, j as RAGConfig, k as RagConfig, l as RetrievalConfig, m as RetrievedChunk, T as TokenUsage, n as UniversalRagConfig, o as UpsertDocument, p as VectorDBConfig, q as VectorDBProvider, W as WorkflowConfig } from './ILLMProvider-DNhyOYoK.mjs';
+export { A as AuthenticationException, C as Chunk, a as ChunkOptions, b as ConfigurationException, E as EmbeddingFailedException, P as ProviderNotFoundException, R as RateLimitException, c as RetrievalException, d as Retrivora, e as RetrivoraError, f as RetrivoraErrorCode } from './index-C9v7-tWd.mjs';
 
 type ChatViewportSize = 'compact' | 'medium' | 'large';
 interface ChatWindowProps {

package/dist/index.d.ts

@@ -1,8 +1,8 @@
 import * as react_jsx_runtime from 'react/jsx-runtime';
 import { CSSProperties, MouseEvent, ReactNode } from 'react';
-import { P as Product, U as UIConfig, R as RagMessage, V as VectorMatch, O as ObservabilityTrace, a as UseRagChatOptions, b as UseRagChatReturn } from './ILLMProvider-Bw2A28nU.js';
-export { C as ChatMessage, c as ChatOptions, d as ChatResponse, E as EmbedOptions, e as EmbeddingConfig, f as EmbeddingProvider, I as ILLMProvider, g as IngestDocument, L as LLMConfig, h as LLMProvider, i as LatencyBreakdown, j as RAGConfig, k as RagConfig, l as RetrievedChunk, T as TokenUsage, m as UpsertDocument, n as VectorDBConfig, o as VectorDBProvider } from './ILLMProvider-Bw2A28nU.js';
-export { C as Chunk, a as ChunkOptions } from './DocumentChunker-Dh9TvmGG.js';
+import { P as Product, U as UIConfig, R as RagMessage, V as VectorMatch, O as ObservabilityTrace, a as UseRagChatOptions, b as UseRagChatReturn } from './ILLMProvider-DNhyOYoK.js';
+export { C as ChatMessage, c as ChatOptions, d as ChatResponse, E as EmbedOptions, e as EmbeddingConfig, f as EmbeddingProvider, I as ILLMProvider, g as IngestDocument, L as LLMConfig, h as LLMProvider, i as LatencyBreakdown, j as RAGConfig, k as RagConfig, l as RetrievalConfig, m as RetrievedChunk, T as TokenUsage, n as UniversalRagConfig, o as UpsertDocument, p as VectorDBConfig, q as VectorDBProvider, W as WorkflowConfig } from './ILLMProvider-DNhyOYoK.js';
+export { A as AuthenticationException, C as Chunk, a as ChunkOptions, b as ConfigurationException, E as EmbeddingFailedException, P as ProviderNotFoundException, R as RateLimitException, c as RetrievalException, d as Retrivora, e as RetrivoraError, f as RetrivoraErrorCode } from './index-CjQdL0cX.js';
 
 type ChatViewportSize = 'compact' | 'medium' | 'large';
 interface ChatWindowProps {