@retrivora-ai/rag-engine 1.8.1 → 1.8.2

package/src/core/Pipeline.ts

@@ -12,7 +12,11 @@ import { ProviderRegistry } from './ProviderRegistry';
 import { BatchProcessor, BatchOptions } from './BatchProcessor';
 import { EmbeddingStrategyResolver } from '../config/EmbeddingStrategy';
 import { QueryProcessor } from './QueryProcessor';
-import { IngestDocument, ChatResponse, VectorMatch, GraphSearchResult, RetrievalResult, UpsertDocument } from '../types';
+import {
+  IngestDocument, ChatResponse, VectorMatch, GraphSearchResult,
+  RetrievalResult, UpsertDocument, ObservabilityTrace, LatencyBreakdown,
+  TokenUsage, RetrievedChunk,
+} from '../types';
 import { UITransformer } from '../utils/UITransformer';
 import { SchemaMapper, SchemaMap } from '../utils/SchemaMapper';
 
@@ -60,6 +64,89 @@ class LRUEmbeddingCache {
   }
 }
 
+// ─── Token estimation helpers ──────────────────────────────────────────────────
+
+/** Rough token count estimate when the provider doesn't expose usage data. */
+function estimateTokens(text: string): number {
+  return Math.ceil(text.length / 4);
+}
+
+/**
+ * Model pricing table (cost per 1k tokens in USD).
+ * Used for estimating cost when providers don't return billing data.
+ */
+const MODEL_COST_PER_1K: Record<string, { input: number; output: number }> = {
+  'gpt-4o':               { input: 0.0025, output: 0.010  },
+  'gpt-4o-mini':          { input: 0.00015, output: 0.0006 },
+  'gpt-4-turbo':          { input: 0.01,   output: 0.03   },
+  'gpt-3.5-turbo':        { input: 0.0005, output: 0.0015 },
+  'claude-3-5-sonnet':    { input: 0.003,  output: 0.015  },
+  'claude-3-haiku':       { input: 0.00025, output: 0.00125 },
+  'gemini-1.5-flash':     { input: 0.000075, output: 0.0003 },
+  'gemini-1.5-pro':       { input: 0.00125, output: 0.005 },
+};
+
+function estimateCostUsd(
+  promptTokens: number,
+  completionTokens: number,
+  model?: string,
+): number | undefined {
+  if (!model) return undefined;
+  const key = Object.keys(MODEL_COST_PER_1K).find((k) => model.toLowerCase().includes(k));
+  if (!key) return undefined;
+  const prices = MODEL_COST_PER_1K[key];
+  return (promptTokens / 1000) * prices.input + (completionTokens / 1000) * prices.output;
+}
+
+// ─── Hallucination scorer ──────────────────────────────────────────────────────
+
+/**
+ * Ask the LLM to self-critique: rate how well the answer is supported by the context.
+ * Returns a score 0–1 and a brief reason. Silently returns undefined on failure.
+ */
+async function scoreHallucination(
+  llm: ILLMProvider,
+  answer: string,
+  context: string,
+): Promise<{ score: number; reason: string } | undefined> {
+  const maxContextChars = 3000;
+  const truncatedContext = context.length > maxContextChars
+    ? context.slice(0, maxContextChars) + '\n...[truncated]'
+    : context;
+
+  const prompt = `You are an AI quality checker. Given the CONTEXT retrieved from a knowledge base and the ANSWER generated from it, rate how well-grounded the answer is.
+
+CONTEXT:
+${truncatedContext}
+
+ANSWER:
+${answer}
+
+Return ONLY a valid JSON object with no markdown fences:
+{"score": <float 0-1>, "reason": "<one sentence>"}
+
+Where score 0 = fully grounded in context, score 1 = likely hallucinated or unsupported.`;
+
+  try {
+    const raw = await llm.chat(
+      [{ role: 'user', content: prompt }],
+      '',
+      { temperature: 0, maxTokens: 120 },
+    );
+
+    const jsonMatch = raw.match(/\{[\s\S]*\}/);
+    if (!jsonMatch) return undefined;
+
+    const parsed = JSON.parse(jsonMatch[0]);
+    if (typeof parsed.score === 'number' && typeof parsed.reason === 'string') {
+      return { score: Math.min(1, Math.max(0, parsed.score)), reason: parsed.reason };
+    }
+  } catch {
+    // Non-critical — fail silently
+  }
+  return undefined;
+}
+
 // ─── Pipeline ─────────────────────────────────────────────────────────────────
 
 /**
@@ -72,6 +159,7 @@ class LRUEmbeddingCache {
  * - Error recovery for transient failures
  * - Multi-tenancy support via namespacing
  * - LRU-bounded embedding cache (max 500 entries, prevents memory leaks)
+ * - Full observability tracing (latency, tokens, hallucination scoring)
  */
 export class Pipeline {
   private vectorDB!: BaseVectorProvider;
@@ -109,10 +197,7 @@ export class Pipeline {
   async initialize(): Promise<void> {
     if (this.initialised) return;
 
-    // Augment system prompt with a Universal Visualization Protocol
-    // We use a unique marker to ensure this is only injected once but is ALWAYS present.
-    //const CHART_MARKER = '<!-- UI_PROTOCOL_V7 -->';
-    const chartInstruction = `
+    const chartInstruction = `\
 You are a helpful product assistant. Use the provided context to answer questions accurately.
 
 ### UI STYLE RULES (CRITICAL):
@@ -129,11 +214,8 @@ You are a helpful product assistant. Use the provided context to answer question
 
     this.config.llm.systemPrompt = chartInstruction;
 
-    console.log(`[Pipeline] Initializing with provider: ${this.config.vectorDb.provider}...`);
-    // Resolve vector DB provider
     this.vectorDB = await ProviderRegistry.createVectorProvider(this.config.vectorDb);
 
-    // Resolve LLM + embedding providers
     const { llmProvider, embeddingProvider } = await EmbeddingStrategyResolver.resolve(
       this.config.llm,
       this.config.embedding
@@ -150,7 +232,6 @@ You are a helpful product assistant. Use the provided context to answer question
 
     await this.vectorDB.initialize();
 
-    // Initialize Agentic Layer if configured
     if (this.config.rag?.architecture === 'agentic') {
       this.agent = new LangChainAgent(this, this.config);
       await this.agent.initialize(this.llmProvider as unknown);
@@ -161,7 +242,6 @@ You are a helpful product assistant. Use the provided context to answer question
 
   /**
    * Ingest documents with automatic chunking, embedding, and batch upsert.
-   * Handles retries for transient failures.
    */
   async ingest(
     documents: IngestDocument[],
@@ -185,10 +265,7 @@ You are a helpful product assistant. Use the provided context to answer question
 
         const totalProcessed = await this.processUpserts(upsertDocs, ns);
 
-        results.push({
-          docId: doc.docId,
-          chunksIngested: totalProcessed,
-        });
+        results.push({ docId: doc.docId, chunksIngested: totalProcessed });
 
         if (this.graphDB && this.entityExtractor) {
           await this.processGraphIngestion(doc.docId, chunks);
@@ -202,23 +279,17 @@ You are a helpful product assistant. Use the provided context to answer question
     return results;
   }
 
-  /**
-   * Step 1: Chunk the document content.
-   */
+  /** Step 1: Chunk the document content. */
   private async prepareChunks(doc: IngestDocument): Promise<Chunk[]> {
     if (this.llamaIngestor) {
-      return await this.llamaIngestor.chunk(doc.content, {
+      return this.llamaIngestor.chunk(doc.content, {
         docId: doc.docId,
         metadata: doc.metadata,
         chunkSize: this.config.rag?.chunkSize,
         chunkOverlap: this.config.rag?.chunkOverlap,
       });
     }
-
-    return this.chunker.chunk(doc.content, {
-      docId: doc.docId,
-      metadata: doc.metadata,
-    });
+    return this.chunker.chunk(doc.content, { docId: doc.docId, metadata: doc.metadata });
   }
 
   /**
@@ -238,7 +309,6 @@ You are a helpful product assistant. Use the provided context to answer question
       embedBatchOptions
     );
 
-    // mapWithRetry filters out undefined on failure; guard against mismatch explicitly
     if (vectors.length !== chunks.length) {
       throw new Error(
         `[Pipeline] Embedding mismatch: got ${vectors.length} vectors for ${chunks.length} chunks. ` +
@@ -249,9 +319,7 @@ You are a helpful product assistant. Use the provided context to answer question
     return vectors;
   }
 
-  /**
-   * Step 3: Upsert chunks to vector database with retry logic.
-   */
+  /** Step 3: Upsert chunks to vector database with retry logic. */
   private async processUpserts(upsertDocs: UpsertDocument[], namespace: string): Promise<number> {
     const upsertBatchOptions: BatchOptions = {
       batchSize: 100,
@@ -272,14 +340,10 @@ You are a helpful product assistant. Use the provided context to answer question
     return upsertResult.totalProcessed;
   }
 
-  /**
-   * Step 4: Optional graph-based entity extraction and ingestion.
-   */
+  /** Step 4: Optional graph-based entity extraction and ingestion. */
   private async processGraphIngestion(docId: string | number, chunks: Chunk[]): Promise<void> {
-    console.log(`[Pipeline] Extracting entities for doc ${docId} (${chunks.length} chunks)...`);
-
     const extractionOptions: BatchOptions = {
-      batchSize: 2, // Low concurrency for LLM extraction
+      batchSize: 2,
       maxRetries: 1,
       initialDelayMs: 500,
     };
@@ -293,7 +357,7 @@ You are a helpful product assistant. Use the provided context to answer question
             if (nodes.length > 0) await this.graphDB!.addNodes(nodes);
             if (edges.length > 0) await this.graphDB!.addEdges(edges);
           } catch (err) {
-            console.warn(`[Pipeline] Entity extraction failed for chunk:`, err);
+            console.warn(`[Pipeline] Entity extraction failed for chunk in doc ${docId}:`, err);
           }
         }
       },
@@ -305,7 +369,6 @@ You are a helpful product assistant. Use the provided context to answer question
     await this.initialize();
 
     if (this.config.rag?.architecture === 'agentic' && this.agent) {
-      console.log('[Pipeline] 🤖 Executing in Agentic Mode...');
       const agentReply = await this.agent.run(question, history);
       return { reply: agentReply, sources: [] };
     }
@@ -314,58 +377,68 @@ You are a helpful product assistant. Use the provided context to answer question
     let reply = '';
     let sources: VectorMatch[] = [];
     let graphData: GraphSearchResult | undefined;
-
     let uiTransformation: unknown;
+    let trace: ObservabilityTrace | undefined;
+
     for await (const chunk of stream) {
       if (typeof chunk === 'string') {
         reply += chunk;
       } else if (typeof chunk === 'object' && chunk !== null) {
-        if ('sources' in chunk) sources = chunk.sources;
-        if ('graphData' in chunk) graphData = chunk.graphData;
-        if ('ui_transformation' in chunk) uiTransformation = chunk.ui_transformation;
+        if ('sources' in chunk) sources = (chunk as ChatResponse).sources;
+        if ('graphData' in chunk) graphData = (chunk as ChatResponse).graphData;
+        if ('ui_transformation' in chunk) uiTransformation = (chunk as ChatResponse).ui_transformation;
+        if ('trace' in chunk) trace = (chunk as { trace: ObservabilityTrace }).trace;
       }
     }
 
-    return { reply, sources, graphData, ui_transformation: uiTransformation };
+    return { reply, sources, graphData, ui_transformation: uiTransformation, trace };
   }
 
   /**
    * High-performance streaming RAG flow.
-   * Yields text chunks first, then the retrieval metadata at the end.
+   * Yields text chunks first, then the retrieval metadata + observability trace at the end.
    */
   async *askStream(question: string, history: ChatMessage[] = [], namespace?: string): AsyncIterable<string | ChatResponse> {
     await this.initialize();
     const ns = namespace ?? this.config.projectId;
     const topK = this.config.rag?.topK ?? 5;
     const scoreThreshold = this.config.rag?.scoreThreshold ?? 0.0;
+    const requestId = `req_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
+    const requestStart = performance.now();
 
     try {
       let searchQuery = question;
+      let rewrittenQuery: string | undefined;
 
       // 1. Query Pre-processing
       if (this.config.rag?.useQueryTransformation) {
         searchQuery = await this.rewriteQuery(question, history);
+        rewrittenQuery = searchQuery !== question ? searchQuery : undefined;
       }
 
-      // 2. Parallel Retrieval
+      // 2. Parallel Retrieval (with latency tracking)
       const hints = QueryProcessor.extractQueryFieldHints(question, this.config.rag?.filterableFields);
       const filter = QueryProcessor.buildQueryFilter(question, hints);
 
-      console.log(`[Pipeline] 🧩 Extracted filters:`, JSON.stringify(filter));
-
+      const embedStart = performance.now();
       const { sources: rawSources, graphData } = await this.retrieve(searchQuery, {
         namespace: ns,
         topK: topK * 2,
         filter
       });
+      const retrieveEnd = performance.now();
+      const embedMs = retrieveEnd - embedStart; // embed + retrieve combined (cache may make embed ~0)
+      const retrieveMs = retrieveEnd - embedStart;
 
       // 3. Reranking
+      const rerankStart = performance.now();
       let sources = rawSources.filter((m) => m.score >= scoreThreshold);
       if (this.config.rag?.useReranking) {
         sources = await this.reranker.rerank(sources, question, topK);
       } else {
         sources = sources.slice(0, topK);
       }
+      const rerankMs = performance.now() - rerankStart;
 
       // 4. Context Augmentation
       let context = sources.length
@@ -379,44 +452,98 @@ You are a helpful product assistant. Use the provided context to answer question
         context = `GRAPH KNOWLEDGE:\n${graphContext}\n\nVECTOR CONTEXT:\n${context}`;
       }
 
-      // 4.5 Schema Training (Dynamic Discovery) - Start training but don't block the stream yet
+      // 4.5 Schema Training (non-blocking)
       const allMetadataKeys = Array.from(new Set(sources.flatMap(s => Object.keys(s.metadata || {}))));
       const trainingPromise = allMetadataKeys.length > 0
         ? SchemaMapper.train(this.llmProvider, ns, allMetadataKeys)
         : Promise.resolve(undefined);
 
-      // 5. Generation (Streaming)
-      const restrictionSuffix = "\n\n(IMPORTANT: Use plain text only. NEVER generate tables, HTML figures, or text charts. If listing products, use simple bullet points.)";
+      // 5. Generation (Streaming) — capture the full reply for observability
+      const restrictionSuffix = '\n\n(IMPORTANT: Use plain text only. NEVER generate tables, HTML figures, or text charts. If listing products, use simple bullet points.)';
       const hardenedHistory = [...history];
       const userQuestion = { role: 'user', content: question + restrictionSuffix } as ChatMessage;
       const messages: ChatMessage[] = [...hardenedHistory, userQuestion];
 
+      // Build prompt strings for tracing
+      const systemPrompt = this.config.llm.systemPrompt ?? '';
+      const userPrompt = messages.map(m => `${m.role}: ${m.content}`).join('\n');
+
+      let fullReply = '';
+      const generateStart = performance.now();
+
       if (this.llmProvider.chatStream) {
         const stream = this.llmProvider.chatStream(messages, context);
         if (!stream) {
           throw new Error(`[Pipeline] ${this.config.llm.provider} chatStream returned undefined`);
         }
         for await (const chunk of stream) {
+          fullReply += chunk;
           yield chunk;
         }
       } else {
         const reply = await this.llmProvider.chat(messages, context);
+        fullReply = reply;
         yield reply;
       }
 
-      // 6. Automated UI Fallback (UITransformer)
-      // Wait for training to finish if it hasn't already
+      const generateMs = performance.now() - generateStart;
+      const totalMs = performance.now() - requestStart;
+
+      const latency: LatencyBreakdown = {
+        embedMs: Math.round(embedMs),
+        retrieveMs: Math.round(retrieveMs),
+        rerankMs: this.config.rag?.useReranking ? Math.round(rerankMs) : undefined,
+        generateMs: Math.round(generateMs),
+        totalMs: Math.round(totalMs),
+      };
+
+      // 6. Token estimation
+      const promptText = systemPrompt + '\n' + context + '\n' + userPrompt;
+      const promptTokens = estimateTokens(promptText);
+      const completionTokens = estimateTokens(fullReply);
+      const tokens: TokenUsage = {
+        promptTokens,
+        completionTokens,
+        totalTokens: promptTokens + completionTokens,
+        estimatedCostUsd: estimateCostUsd(promptTokens, completionTokens, this.config.llm.model),
+      };
+
+      // 7. Hallucination scoring (async, non-blocking on generation yield)
+      const hallucinationResult = await scoreHallucination(this.llmProvider, fullReply, context).catch(() => undefined);
+
+      // 8. Automated UI transformation
       const trainedSchema = await trainingPromise;
-
       const uiTransformation = await this.generateUiTransformation(question, sources, context, trainedSchema);
 
-      // Yield retrieval metadata and automated UI transformation at the end
+      // 9. Build observability trace
+      const trace: ObservabilityTrace = {
+        requestId,
+        query: question,
+        rewrittenQuery,
+        systemPrompt,
+        userPrompt: question + restrictionSuffix,
+        chunks: sources.map((s) => ({
+          id: s.id,
+          score: s.score,
+          content: s.content,
+          metadata: s.metadata ?? {},
+          namespace: ns,
+        } satisfies RetrievedChunk)),
+        latency,
+        tokens,
+        hallucinationScore: hallucinationResult?.score,
+        hallucinationReason: hallucinationResult?.reason,
+        timestamp: new Date().toISOString(),
+      };
+
+      // Yield metadata + trace + UI transformation
       yield {
         reply: '',
         sources,
         graphData,
-        ui_transformation: uiTransformation
-      } as ChatResponse;
+        ui_transformation: uiTransformation,
+        trace,
+      } as ChatResponse & { trace: ObservabilityTrace };
 
     } catch (error) {
       throw new Error(`[Pipeline] Stream failed: ${error instanceof Error ? error.message : String(error)}`);
@@ -438,9 +565,6 @@ You are a helpful product assistant. Use the provided context to answer question
     }
 
     try {
-      // Delegate to the centralized LLM-driven visualization decision.
-      // UITransformer.analyzeAndDecide() internally falls back to the keyword
-      // heuristic if the LLM call fails or returns unparseable JSON.
       return await UITransformer.analyzeAndDecide(question, sources, this.llmProvider);
     } catch (err) {
       console.warn('[Pipeline] generateUiTransformation failed, using heuristic fallback:', err);
@@ -472,12 +596,9 @@ You are a helpful product assistant. Use the provided context to answer question
     return { sources, graphData };
   }
 
-  /**
-   * Rewrite the user query for better retrieval performance.
-   */
+  /** Rewrite the user query for better retrieval performance. */
   private async rewriteQuery(question: string, history: ChatMessage[]): Promise<string> {
-    const prompt = `
-Given the following conversation history and a new question, rewrite the question to be a better search query for a vector database. 
+    const prompt = `Given the following conversation history and a new question, rewrite the question to be a better search query for a vector database. 
 Focus on extracting the core intent and entities. Do not answer the question, just rewrite it.
 
 History:
@@ -498,29 +619,19 @@ Optimized Search Query:`;
     return rewrite.trim() || question;
   }
 
-  /**
-   * Generate 3-5 short, relevant questions based on the vector database content.
-   */
+  /** Generate 3-5 short, relevant questions based on the vector database content. */
   async getSuggestions(query: string, namespace?: string): Promise<string[]> {
-    if (!query || query.trim().length < 3) {
-      return [];
-    }
+    if (!query || query.trim().length < 3) return [];
 
     await this.initialize();
     const ns = namespace ?? this.config.projectId;
 
     try {
-      // 1. Retrieve relevant context (top 3 matches)
       const { sources } = await this.retrieve(query, { namespace: ns, topK: 3 });
+      if (sources.length === 0) return [];
 
-      if (sources.length === 0) {
-        return [];
-      }
-
-      // 2. Generate suggestions using LLM
       const context = sources.map((s) => s.content).join('\n\n---\n\n');
-      const prompt = `
-Based on the following snippets from a document, what are 3 short, relevant questions a user might ask? 
+      const prompt = `Based on the following snippets from a document, what are 3 short, relevant questions a user might ask? 
 Focus on questions that can be answered by the context.
 Keep each question under 10 words and make them very specific to the content.
 Return ONLY a JSON array of strings like ["Question 1", "Question 2", "Question 3"].
@@ -538,7 +649,6 @@ Suggestions:`;
         ''
       );
 
-      // Simple parsing of JSON array from LLM response
       const match = response.match(/\[[\s\S]*\]/);
       if (match) {
         const suggestions = JSON.parse(match[0]);

package/src/handlers/index.ts

@@ -30,6 +30,11 @@ function sseUIFrame(uiTransformation: unknown): string {
   return `data: ${JSON.stringify({ type: 'ui_transformation', data: uiTransformation })}\n\n`;
 }
 
+/** Encode the observability trace as an SSE frame. */
+function sseObservabilityFrame(trace: unknown): string {
+  return `data: ${JSON.stringify({ type: 'observability', data: trace })}\n\n`;
+}
+
 /** Encode a stream error as an SSE frame. */
 function sseErrorFrame(message: string): string {
   return `data: ${JSON.stringify({ type: 'error', error: message })}\n\n`;
@@ -153,12 +158,14 @@ export function createStreamHandler(configOrPlugin?: Partial<RagConfig> | Vector
               enqueue(sseMetaFrame(chunk));
 
               // ── LLM-driven visualization decision ─────────────────────────
-              // Sends question + RAG context to the LLM and asks it to choose
-              // the best visualization format (bar_chart, pie_chart, table…).
-              // Falls back to keyword heuristic if the LLM call fails.
               const responseChunk = chunk as ChatResponse;
               const sources = responseChunk?.sources || [];
 
+              // Emit observability trace if present
+              if (responseChunk?.trace) {
+                enqueue(sseObservabilityFrame(responseChunk.trace));
+              }
+
               if (sources.length > 0) {
                 try {
                   const llmProvider = plugin.getLLMProvider?.();
@@ -172,7 +179,6 @@ export function createStreamHandler(configOrPlugin?: Partial<RagConfig> | Vector
                   }
                 } catch (transformError) {
                   console.warn('[createStreamHandler] UI transformation warning:', transformError);
-                  // Last-resort fallback: heuristic transform
                   try {
                     const fallback = UITransformer.transform(message, sources, plugin.getConfig());
                     if (fallback) enqueue(sseUIFrame(fallback));
@@ -265,15 +271,68 @@ export function createUploadHandler(configOrPlugin?: Partial<RagConfig> | Vector
       const formData = await req.formData();
       const files = formData.getAll('files') as File[];
       const namespace = (formData.get('namespace') as string) || undefined;
+      const dimensionRaw = formData.get('dimension');
+      const dimension = dimensionRaw ? parseInt(dimensionRaw as string, 10) : undefined;
 
       if (!files || files.length === 0) {
         return NextResponse.json({ error: 'No files provided' }, { status: 400 });
       }
 
-      const documents = await Promise.all(
-        files.map(async (file) => {
+      const documents: { docId: string; content: string; metadata?: Record<string, unknown> }[] = [];
+
+      for (const file of files) {
+        if (file.name.toLowerCase().endsWith('.csv') || file.type === 'text/csv') {
+          const text = await file.text();
+          const Papa = await import('papaparse');
+          const parsed = Papa.default.parse(text, { header: true, skipEmptyLines: true });
+          
+          if (parsed.data && parsed.data.length > 0) {
+            let i = 0;
+            let lastRowData: Record<string, string> | null = null;
+            
+            for (const row of parsed.data) {
+              i++;
+              const rowData = row as Record<string, string>;
+              
+              // Dynamically determine the grouping key (typically the first column like ID or Handle)
+              const groupingKey = Object.keys(rowData)[0];
+              
+              // Forward-fill ALL empty columns for variants belonging to the same group
+              if (lastRowData && groupingKey && rowData[groupingKey] && rowData[groupingKey] === lastRowData[groupingKey]) {
+                for (const key of Object.keys(rowData)) {
+                  // Forward-fill if the current row lacks the field but the previous row has it
+                  if (!rowData[key] && lastRowData[key]) {
+                    rowData[key] = lastRowData[key];
+                  }
+                }
+              }
+              // Save a copy of the filled rowData for the next iteration
+              lastRowData = { ...rowData };
+              
+              const contentParts: string[] = [];
+              for (const [key, val] of Object.entries(rowData)) {
+                if (key && val) {
+                  contentParts.push(`${key}: ${val}`);
+                }
+              }
+              
+              documents.push({
+                docId: `${file.name}-row-${i}`,
+                content: contentParts.join(', '),
+                metadata: {
+                  fileName: file.name,
+                  fileSize: file.size,
+                  fileType: file.type,
+                  uploadedAt: new Date().toISOString(),
+                  ...(dimension ? { dimension } : {}),
+                  ...rowData
+                }
+              });
+            }
+          }
+        } else {
           const content = await DocumentParser.parse(file, file.name, file.type);
-          return {
+          documents.push({
             docId: file.name,
             content,
             metadata: {
@@ -281,10 +340,11 @@ export function createUploadHandler(configOrPlugin?: Partial<RagConfig> | Vector
               fileSize: file.size,
               fileType: file.type,
               uploadedAt: new Date().toISOString(),
+              ...(dimension ? { dimension } : {}),
             },
-          };
-        })
-      );
+          });
+        }
+      }
 
       const results = await plugin.ingest(documents, namespace);
       return NextResponse.json({ message: 'Upload successful', results });
@@ -325,5 +385,5 @@ export function createSuggestionsHandler(configOrPlugin?: Partial<RagConfig> | V
   };
 }
 
-// Re-export SSE helper so host apps can use it in custom handlers
-export { sseFrame, sseTextFrame, sseMetaFrame, sseErrorFrame, sseUIFrame };
+// Re-export SSE helpers so host apps can use them in custom handlers
+export { sseFrame, sseTextFrame, sseMetaFrame, sseErrorFrame, sseUIFrame, sseObservabilityFrame };