@retrivora-ai/rag-engine 1.8.0 → 1.8.2

package/src/core/ProviderRegistry.ts

@@ -77,8 +77,8 @@ export class ProviderRegistry {
       }
       case 'pgvector':
       case 'postgresql': {
-        const { PostgreSQLProvider } = await import('../providers/vectordb/PostgreSQLProvider');
-        return PostgreSQLProvider;
+        const { MultiTablePostgresProvider } = await import('../providers/vectordb/MultiTablePostgresProvider');
+        return MultiTablePostgresProvider as unknown as VectorProviderClass;
       }
       case 'mongodb': {
         const { MongoDBProvider } = await import('../providers/vectordb/MongoDBProvider');

package/src/core/VectorPlugin.ts

@@ -38,6 +38,15 @@ export class VectorPlugin {
     return this.config;
   }
 
+  /**
+   * Get the initialized LLM provider (available after the first request).
+   * Used by handlers to pass to UITransformer.analyzeAndDecide() for
+   * LLM-driven visualization decisions.
+   */
+  getLLMProvider() {
+    return this.pipeline.getLLMProvider();
+  }
+
   /**
    * Perform pre-flight health checks on all configured providers.
    * Useful to verify connectivity before running operations.

package/src/handlers/index.ts

@@ -27,7 +27,12 @@ function sseMetaFrame(meta: unknown): string {
 
 /** Encode the UI transformation result as an SSE frame. */
 function sseUIFrame(uiTransformation: unknown): string {
-  return `data: ${JSON.stringify({ type: 'ui_transformation', ...((uiTransformation as object) ?? {}) })}\n\n`;
+  return `data: ${JSON.stringify({ type: 'ui_transformation', data: uiTransformation })}\n\n`;
+}
+
+/** Encode the observability trace as an SSE frame. */
+function sseObservabilityFrame(trace: unknown): string {
+  return `data: ${JSON.stringify({ type: 'observability', data: trace })}\n\n`;
 }
 
 /** Encode a stream error as an SSE frame. */
@@ -152,15 +157,32 @@ export function createStreamHandler(configOrPlugin?: Partial<RagConfig> | Vector
               // Retrieval metadata object — always the final frame
               enqueue(sseMetaFrame(chunk));
 
-              // Transform retrieved sources to UI representation
-              const sources = (chunk as ChatResponse)?.sources || [];
-              if (sources && sources.length > 0) {
+              // ── LLM-driven visualization decision ─────────────────────────
+              const responseChunk = chunk as ChatResponse;
+              const sources = responseChunk?.sources || [];
+
+              // Emit observability trace if present
+              if (responseChunk?.trace) {
+                enqueue(sseObservabilityFrame(responseChunk.trace));
+              }
+
+              if (sources.length > 0) {
                 try {
-                  const uiTransformation = UITransformer.transform(message, sources);
-                  enqueue(sseUIFrame(uiTransformation));
+                  const llmProvider = plugin.getLLMProvider?.();
+                  const uiTransformation = responseChunk?.ui_transformation ??
+                    (llmProvider
+                      ? await UITransformer.analyzeAndDecide(message, sources, llmProvider)
+                      : UITransformer.transform(message, sources, plugin.getConfig()));
+
+                  if (uiTransformation) {
+                    enqueue(sseUIFrame(uiTransformation));
+                  }
                 } catch (transformError) {
                   console.warn('[createStreamHandler] UI transformation warning:', transformError);
-                  // Don't fail the stream if transformation fails, just skip it
+                  try {
+                    const fallback = UITransformer.transform(message, sources, plugin.getConfig());
+                    if (fallback) enqueue(sseUIFrame(fallback));
+                  } catch { /* silent */ }
                 }
               }
             }
@@ -249,15 +271,68 @@ export function createUploadHandler(configOrPlugin?: Partial<RagConfig> | Vector
       const formData = await req.formData();
       const files = formData.getAll('files') as File[];
       const namespace = (formData.get('namespace') as string) || undefined;
+      const dimensionRaw = formData.get('dimension');
+      const dimension = dimensionRaw ? parseInt(dimensionRaw as string, 10) : undefined;
 
       if (!files || files.length === 0) {
         return NextResponse.json({ error: 'No files provided' }, { status: 400 });
       }
 
-      const documents = await Promise.all(
-        files.map(async (file) => {
+      const documents: { docId: string; content: string; metadata?: Record<string, unknown> }[] = [];
+
+      for (const file of files) {
+        if (file.name.toLowerCase().endsWith('.csv') || file.type === 'text/csv') {
+          const text = await file.text();
+          const Papa = await import('papaparse');
+          const parsed = Papa.default.parse(text, { header: true, skipEmptyLines: true });
+          
+          if (parsed.data && parsed.data.length > 0) {
+            let i = 0;
+            let lastRowData: Record<string, string> | null = null;
+            
+            for (const row of parsed.data) {
+              i++;
+              const rowData = row as Record<string, string>;
+              
+              // Dynamically determine the grouping key (typically the first column like ID or Handle)
+              const groupingKey = Object.keys(rowData)[0];
+              
+              // Forward-fill ALL empty columns for variants belonging to the same group
+              if (lastRowData && groupingKey && rowData[groupingKey] && rowData[groupingKey] === lastRowData[groupingKey]) {
+                for (const key of Object.keys(rowData)) {
+                  // Forward-fill if the current row lacks the field but the previous row has it
+                  if (!rowData[key] && lastRowData[key]) {
+                    rowData[key] = lastRowData[key];
+                  }
+                }
+              }
+              // Save a copy of the filled rowData for the next iteration
+              lastRowData = { ...rowData };
+              
+              const contentParts: string[] = [];
+              for (const [key, val] of Object.entries(rowData)) {
+                if (key && val) {
+                  contentParts.push(`${key}: ${val}`);
+                }
+              }
+              
+              documents.push({
+                docId: `${file.name}-row-${i}`,
+                content: contentParts.join(', '),
+                metadata: {
+                  fileName: file.name,
+                  fileSize: file.size,
+                  fileType: file.type,
+                  uploadedAt: new Date().toISOString(),
+                  ...(dimension ? { dimension } : {}),
+                  ...rowData
+                }
+              });
+            }
+          }
+        } else {
           const content = await DocumentParser.parse(file, file.name, file.type);
-          return {
+          documents.push({
             docId: file.name,
             content,
             metadata: {
@@ -265,10 +340,11 @@ export function createUploadHandler(configOrPlugin?: Partial<RagConfig> | Vector
               fileSize: file.size,
               fileType: file.type,
               uploadedAt: new Date().toISOString(),
+              ...(dimension ? { dimension } : {}),
             },
-          };
-        })
-      );
+          });
+        }
+      }
 
       const results = await plugin.ingest(documents, namespace);
       return NextResponse.json({ message: 'Upload successful', results });
@@ -309,5 +385,5 @@ export function createSuggestionsHandler(configOrPlugin?: Partial<RagConfig> | V
   };
 }
 
-// Re-export SSE helper so host apps can use it in custom handlers
-export { sseFrame, sseTextFrame, sseMetaFrame, sseErrorFrame, sseUIFrame };
+// Re-export SSE helpers so host apps can use them in custom handlers
+export { sseFrame, sseTextFrame, sseMetaFrame, sseErrorFrame, sseUIFrame, sseObservabilityFrame };

package/src/hooks/useRagChat.ts

@@ -19,16 +19,17 @@
  */
 
 import { useState, useCallback, useRef, useEffect } from 'react';
-import { VectorMatch, RagMessage, UseRagChatOptions, UseRagChatReturn } from '../types';
+import { VectorMatch, RagMessage, UseRagChatOptions, UseRagChatReturn, ObservabilityTrace } from '../types';
 import { useStoredMessages } from './useStoredMessages';
 
 // ─── SSE Frame Parser ──────────────────────────────────────────────────────────
 
-interface SseTextFrame  { type: 'text';     text: string }
-interface SseMetaFrame  { type: 'metadata'; sources?: VectorMatch[] }
-interface SseErrorFrame { type: 'error';    error: string }
-interface SseUiFrame    { type: 'ui_transformation'; data: unknown }
-type SseFrame = SseTextFrame | SseMetaFrame | SseErrorFrame | SseUiFrame;
+interface SseTextFrame         { type: 'text';             text: string }
+interface SseMetaFrame         { type: 'metadata';         sources?: VectorMatch[] }
+interface SseErrorFrame        { type: 'error';            error: string }
+interface SseUiFrame           { type: 'ui_transformation'; data: unknown }
+interface SseObservabilityFrame { type: 'observability';   data: ObservabilityTrace }
+type SseFrame = SseTextFrame | SseMetaFrame | SseErrorFrame | SseUiFrame | SseObservabilityFrame;
 
 /**
  * Parse all complete SSE frames from a raw chunk string.
@@ -126,6 +127,7 @@ export function useRagChat(
         let assistantContent = '';
         let sources: VectorMatch[] = [];
         let uiTransformation: unknown = null;
+        let trace: ObservabilityTrace | undefined;
         // Buffer for partial SSE frames that arrive split across reads
         let buffer = '';
 
@@ -155,12 +157,14 @@ export function useRagChat(
           buffer = buffer.slice(lastBoundary + 2);
 
           for (const frame of parseSseChunk(complete)) {
-            if (frame.type === 'text') {
+            if (frame.type === 'text' && frame.text) {
               assistantContent += frame.text;
             } else if (frame.type === 'metadata') {
               sources = frame.sources ?? [];
             } else if (frame.type === 'ui_transformation') {
               uiTransformation = frame.data;
+            } else if (frame.type === 'observability') {
+              trace = (frame as SseObservabilityFrame).data;
             } else if (frame.type === 'error') {
               setError(frame.error || 'Stream error');
             }
@@ -184,20 +188,26 @@ export function useRagChat(
         // Process any remaining buffered data
         if (buffer.trim()) {
           for (const frame of parseSseChunk(buffer)) {
-            if (frame.type === 'text') assistantContent += frame.text;
+            if (frame.type === 'text' && frame.text) assistantContent += frame.text;
             else if (frame.type === 'metadata') sources = frame.sources ?? [];
+            else if (frame.type === 'observability') trace = (frame as SseObservabilityFrame).data;
           }
         }
 
-        const finalAssistantMessage: RagMessage = {
+        // Final message with all accumulated data
+        const finalMsg: RagMessage = {
           id: assistantMessageId,
           role: 'assistant',
           content: assistantContent,
           sources: sources.length > 0 ? sources : undefined,
-          uiTransformation: uiTransformation || undefined,
+          uiTransformation: uiTransformation ?? undefined,
+          trace,
           createdAt: new Date().toISOString(),
         };
-        onReply?.(finalAssistantMessage);
+        setMessages((prev) =>
+          prev.map((msg) => (msg.id === assistantMessageId ? finalMsg : msg))
+        );
+        onReply?.(finalMsg);
       } catch (err) {
         const msg = err instanceof Error ? err.message : 'Something went wrong. Please try again.';
         setError(msg);

package/src/index.ts

@@ -9,11 +9,15 @@ export { ChatWindow } from './components/ChatWindow';
 export { DocumentUpload } from './components/DocumentUpload';
 export { MessageBubble } from './components/MessageBubble';
 export { SourceCard } from './components/SourceCard';
+export { ObservabilityPanel } from './components/ObservabilityPanel';
 export { ConfigProvider, useConfig } from './components/ConfigProvider';
 
 // ── Headless Hooks ────────────────────────────────────────────
 export { useRagChat } from './hooks/useRagChat';
 
+// ── Utils ─────────────────────────────────────────────────────
+export { addSynonyms } from './utils/synonyms';
+
 // ── Types (Interfaces/Types only, no Node.js deps) ─────────────
 export type { RagConfig, VectorDBConfig, VectorDBProvider, LLMConfig, LLMProvider, EmbeddingConfig, EmbeddingProvider, UIConfig, RAGConfig } from './config/RagConfig';
 export type { ILLMProvider, EmbedOptions } from './llm/ILLMProvider';
@@ -35,5 +39,9 @@ export type {
   ProductCardProps,
   ProductCarouselProps,
   DocumentUploadProps,
-  SourceCardProps
+  SourceCardProps,
+  ObservabilityTrace,
+  LatencyBreakdown,
+  TokenUsage,
+  RetrievedChunk
 } from './types';

package/src/llm/LLMFactory.ts

@@ -12,10 +12,50 @@ interface LLMProviderStatic {
   getHealthChecker?: () => IProviderHealthChecker;
 }
 
+/** Registry for custom provider factories registered at runtime. */
+const customProviders = new Map<string, (config: LLMConfig) => ILLMProvider>();
+
 /**
  * LLMFactory — instantiates the correct ILLMProvider based on llmConfig.provider.
  */
 export class LLMFactory {
+  /**
+   * Register a custom LLM provider factory at runtime.
+   *
+   * Use this to add support for any LLM backend (Azure OpenAI, Cohere, Mistral,
+   * Bedrock, etc.) without modifying this library's source code.
+   *
+   * @example
+   * // In your Next.js app initialization:
+   * import { LLMFactory } from '@retrivora-ai/rag-engine/server';
+   * import { MyCustomProvider } from './providers/MyCustomProvider';
+   *
+   * LLMFactory.register('my-provider', (config) => new MyCustomProvider(config));
+   *
+   * // Then set in your .env.local:
+   * // LLM_PROVIDER=my-provider
+   */
+  static register(name: string, factory: (config: LLMConfig) => ILLMProvider): void {
+    customProviders.set(name.toLowerCase(), factory);
+    console.log(`[LLMFactory] Registered custom provider: "${name}"`);
+  }
+
+  /**
+   * Unregister a previously registered custom provider.
+   */
+  static unregister(name: string): void {
+    customProviders.delete(name.toLowerCase());
+  }
+
+  /**
+   * List all registered provider names (built-in + custom).
+   */
+  static listProviders(): string[] {
+    return [
+      'openai', 'anthropic', 'ollama', 'gemini', 'rest', 'universal_rest', 'custom',
+      ...Array.from(customProviders.keys()),
+    ];
+  }
   static create(llmConfig: LLMConfig, embeddingConfig?: EmbeddingConfig): ILLMProvider {
     switch (llmConfig.provider) {
       case 'openai':
@@ -30,11 +70,23 @@ export class LLMFactory {
       case 'universal_rest':
       case 'custom':
         return new UniversalLLMAdapter(llmConfig);
-      default:
+      default: {
+        // Check the runtime custom provider registry first
+        const providerName = String(llmConfig.provider ?? '').toLowerCase();
+        const customFactory = customProviders.get(providerName);
+        if (customFactory) {
+          return customFactory(llmConfig);
+        }
+        // Fall back to UniversalLLMAdapter if a baseUrl is provided
         if (llmConfig.baseUrl || (llmConfig.options as Record<string, unknown>)?.baseUrl) {
           return new UniversalLLMAdapter(llmConfig);
         }
-        throw new Error(`[LLMFactory] Unknown provider "${llmConfig.provider}"`);
+        throw new Error(
+          `[LLMFactory] Unknown provider "${llmConfig.provider}". ` +
+          `Built-in providers: ${LLMFactory.listProviders().join(', ')}. ` +
+          `Register a custom provider with LLMFactory.register().`
+        );
+      }
     }
   }
 

package/src/llm/providers/AnthropicProvider.ts

@@ -75,13 +75,15 @@ export class AnthropicProvider implements ILLMProvider {
   }
 
   async chat(messages: ChatMessage[], context: string, options?: ChatOptions): Promise<string> {
-    const systemPrompt =
+    const resolvedPrompt = options?.systemPrompt ??
       this.llmConfig.systemPrompt ??
       `You are a helpful assistant. Use the context below to answer the user's question accurately.\n\nContext:\n${context}`;
 
-    const system = systemPrompt.includes('{{context}}')
-      ? systemPrompt.replace('{{context}}', context)
-      : `${systemPrompt}\n\nContext:\n${context}`;
+    const system = resolvedPrompt.includes('{{context}}')
+      ? resolvedPrompt.replace('{{context}}', context)
+      : options?.systemPrompt
+        ? resolvedPrompt // per-call: context is already in the user message
+        : `${resolvedPrompt}\n\nContext:\n${context}`;
 
     const anthropicMessages: Anthropic.MessageParam[] = messages.map((m) => ({
       role: m.role === 'assistant' ? 'assistant' : 'user',
@@ -100,13 +102,15 @@ export class AnthropicProvider implements ILLMProvider {
   }
 
   async *chatStream(messages: ChatMessage[], context: string, options?: ChatOptions): AsyncIterable<string> {
-    const systemPrompt =
+    const resolvedPrompt = options?.systemPrompt ??
       this.llmConfig.systemPrompt ??
       `You are a helpful assistant. Use the context below to answer the user's question accurately.\n\nContext:\n${context}`;
 
-    const system = systemPrompt.includes('{{context}}')
-      ? systemPrompt.replace('{{context}}', context)
-      : `${systemPrompt}\n\nContext:\n${context}`;
+    const system = resolvedPrompt.includes('{{context}}')
+      ? resolvedPrompt.replace('{{context}}', context)
+      : options?.systemPrompt
+        ? resolvedPrompt
+        : `${resolvedPrompt}\n\nContext:\n${context}`;
 
     const anthropicMessages: Anthropic.MessageParam[] = messages.map((m) => ({
       role: m.role === 'assistant' ? 'assistant' : 'user',