@retrivora-ai/rag-engine 2.0.6 → 2.0.8

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@retrivora-ai/rag-engine",
-  "version": "2.0.6",
+  "version": "2.0.8",
   "description": "Retrivora AI is a plug-and-play AI engine for RAG chat experiences — generic vector DB + LLM provider, embeddable or standalone.",
   "author": "Abhinav Alkuchi",
   "license": "UNLICENSED",

package/src/core/Pipeline.ts

@@ -580,58 +580,67 @@ You are a helpful assistant. Use the provided context to answer questions accura
 
       const hasMetadataFilter = filter.metadata && Object.keys(filter.metadata).length > 0;
       const hasNumericPredicates = Array.isArray(filter.__numericPredicates) && filter.__numericPredicates.length > 0;
-      const wantsExhaustiveList = hasNumericPredicates || hasMetadataFilter || /\b(list|all|show|provide|give|get|browse|find|search|display)\b/i.test(question);
-      const retrievalLimit = wantsExhaustiveList ? Math.max(topK * 20, 100) : topK * 2;
+      const wantsExhaustiveList = true; // Always query all available resources across the database
+      const retrievalLimit = Math.max(topK * 50, 1000);
 
-      const rawSources = (strategyResult === 'vector' || strategyResult === 'both') && queryVector && queryVector.length > 0
+      const rawSources = ((strategyResult === 'vector' || strategyResult === 'both') && queryVector && queryVector.length > 0
         ? await this.vectorDB.query(queryVector, retrievalLimit, ns, filter)
-        : [];
+        : []).filter((s): s is VectorMatch => Boolean(s && typeof s === 'object'));
+
+      console.log('[Pipeline] Raw vectorDB.query response:', {
+        rawCount: rawSources.length,
+        sample: rawSources.slice(0, 2).map((s) => ({
+          id: s?.id,
+          score: s?.score,
+          contentType: typeof s?.content,
+          contentSnippet: String(s?.content ?? '').substring(0, 150),
+          metadataKeys: Object.keys(s?.metadata || {}),
+        })),
+      });
 
       const retrieveEnd = performance.now();
       const embedMs = retrieveEnd - embedStart;
       const retrieveMs = retrieveEnd - embedStart;
 
-      // 3. Reranking
+      // 3. Reranking & Filtering across ALL matching resources
       const rerankStart = performance.now();
-      const structuredSources = this.applyStructuredFilters(rawSources, filter);
+      const structuredSources = this.applyStructuredFilters(rawSources, filter).filter((s): s is VectorMatch => Boolean(s && typeof s === 'object'));
       let fullSources = hasNumericPredicates
         ? structuredSources
-        : structuredSources.filter((m) => m.score >= scoreThreshold);
-      const rerankLimit = wantsExhaustiveList ? retrievalLimit : topK;
+        : structuredSources.filter((m) => (m?.score ?? 0) >= scoreThreshold);
+      const rerankLimit = Math.max(retrievalLimit, fullSources.length);
       const useReranking = arch !== 'naive' && (arch === 'retrieve-and-rerank' || this.config.rag?.useReranking);
       if (!hasNumericPredicates && useReranking) {
-        fullSources = await this.reranker.rerank(fullSources, question, rerankLimit);
-      } else if (!wantsExhaustiveList) {
-        fullSources = fullSources.slice(0, topK);
+        fullSources = (await this.reranker.rerank(fullSources, question, rerankLimit)).filter((s): s is VectorMatch => Boolean(s && typeof s === 'object'));
       }
       const rerankMs = performance.now() - rerankStart;
 
-      // 4. Context Augmentation - Review all retrieved/reranked sources for full comprehension
-      let context = fullSources.length
-        ? fullSources.map((m, i) => `[Source ${i + 1}]\n${m?.content ?? ''}`).join('\n\n---\n\n')
+      // 4. Context Augmentation — Cap context sent to LLM to top 15 sources (~3000 tokens)
+      // to keep prompt under Groq's 6000 token limit while preserving ALL sources for UI citations.
+      const totalCount = fullSources.length;
+      const promptSources = fullSources.slice(0, 15);
+      let context = promptSources.length
+        ? `[SEARCH SUMMARY: Found ${totalCount} matching record(s) out of ${rawSources.length} total resources searched in database. Context includes top ${promptSources.length} relevant records]\n\n` +
+          promptSources.map((m, i) => `[Source ${i + 1}]\n${m?.content ?? ''}`).join('\n\n---\n\n')
         : 'No relevant context found.';
 
-      // Count the records of relevant sources to pass into the slice dynamically
-      let displayCount = 15; // default fallback
-
-      if (hasMetadataFilter) {
-        // If a metadata filter matches (e.g. brand, category), all of those matches are highly relevant
-        displayCount = fullSources.length;
-      } else {
-        // For general semantic queries, count the number of matches meeting high relevance threshold
-        const baseThreshold = Math.max(scoreThreshold, 0.4);
-        const highlyRelevant = fullSources.filter(m => m.score >= baseThreshold);
-        displayCount = Math.max(highlyRelevant.length, topK);
-        // Cap non-metadata semantic searches to a maximum of 15 to keep the UI clean
-        if (displayCount > 15) {
-          displayCount = 15;
-        }
-      }
-
-      // Ensure they are sorted from highest to lowest score, then slice using our dynamic count
+      // Do NOT truncate UI sources — keep all qualified matches for UI citations & tables
       const sources = [...fullSources]
-        .sort((a, b) => b.score - a.score)
-        .slice(0, displayCount);
+        .filter((s): s is VectorMatch => Boolean(s && typeof s === 'object'))
+        .sort((a, b) => (b?.score ?? 0) - (a?.score ?? 0));
+
+      console.log('[Pipeline] Final sources for prompt & UI:', {
+        count: sources.length,
+        totalRawCount: rawSources.length,
+        sources: sources.slice(0, 10).map((s, idx) => ({
+          rank: idx + 1,
+          id: s?.id,
+          score: s?.score,
+          contentType: typeof s?.content,
+          contentSnippet: String(s?.content ?? '').substring(0, 150),
+          metadata: s?.metadata,
+        })),
+      });
 
       if (graphData && graphData.nodes.length > 0) {
         const graphContext = graphData.nodes.map(n =>
@@ -653,17 +662,17 @@ You are a helpful assistant. Use the provided context to answer questions accura
       // Yield sources immediately to support early UI citation rendering (highly responsive)
       yield {
         reply: '',
-        sources: sources.map((s) => ({
+        sources: (sources || []).filter((s): s is VectorMatch => Boolean(s && typeof s === 'object')).map((s) => ({
           id: s.id,
-          score: s.score,
-          content: s.content,
-          metadata: s.metadata ?? {},
+          score: s.score ?? 0,
+          content: s?.content ?? '',
+          metadata: s?.metadata ?? {},
           namespace: ns,
         } satisfies RetrievedChunk)),
       } as ChatResponse;
 
       // 4.5 Schema Training — true fire-and-forget, never gates anything.
-      const allMetadataKeys = Array.from(new Set(sources.flatMap(s => Object.keys(s.metadata || {}))));
+      const allMetadataKeys = Array.from(new Set(sources.filter(Boolean).flatMap(s => Object.keys(s?.metadata || {}))));
       const cachedSchema = allMetadataKeys.length > 0
         ? SchemaMapper.getCached(ns, allMetadataKeys)
         : undefined;
@@ -840,10 +849,8 @@ You are a helpful assistant. Use the provided context to answer questions accura
         yield fullReply;
       }
 
-      // Kick off hallucination scoring now that we have the full reply.
-      // It runs in parallel with the remaining post-processing below.
-      const runHallucination = this.config.llm.options?.hallucinationScoring === true ||
-        (this.config.llm.provider !== 'ollama' && this.config.llm.options?.hallucinationScoring !== false);
+      // Kick off hallucination scoring ONLY when explicitly enabled by options
+      const runHallucination = this.config.llm.options?.hallucinationScoring === true;
       if (runHallucination) {
         hallucinationScoringPromise = scoreHallucination(this.llmProvider, fullReply, context)
           .catch(() => undefined);
@@ -979,9 +986,8 @@ You are a helpful assistant. Use the provided context to answer questions accura
       );
     }
 
-    const isLocalProvider = this.config.llm.provider === 'ollama';
-    const disableLlmUiTransform = this.config.llm.options?.disableLlmUiTransform === true;
-    if (forceDeterministic || isLocalProvider || disableLlmUiTransform) {
+    const enableLlmUiTransform = this.config.llm.options?.enableLlmUiTransform === true;
+    if (forceDeterministic || !enableLlmUiTransform) {
       return UITransformer.transform(question, sources, this.config, cachedSchema);
     }
 
@@ -1015,6 +1021,7 @@ You are a helpful assistant. Use the provided context to answer questions accura
   }
 
   private resolveNumericPredicateValue(source: VectorMatch, predicate: NumericPredicate): number | null {
+    if (!source) return null;
     const meta = source.metadata || {};
     const field = predicate.field;
     const entries = Object.entries(meta)
@@ -1032,7 +1039,7 @@ You are a helpful assistant. Use the provided context to answer questions accura
         if (value !== null) return value;
       }
 
-      const contentValue = this.extractNumericValueFromContent(source.content, field);
+      const contentValue = this.extractNumericValueFromContent(source.content ?? '', field);
       if (contentValue !== null) return contentValue;
     }
 
@@ -1046,6 +1053,7 @@ You are a helpful assistant. Use the provided context to answer questions accura
   }
 
   private extractNumericValueFromContent(content: string, field: string): number | null {
+    if (!content || typeof content !== 'string') return null;
     const escapedWords = field
       .split(/\s+|_+|-+/)
       .filter(Boolean)
@@ -1164,6 +1172,18 @@ You are a helpful assistant. Use the provided context to answer questions accura
       }
     }
 
+    console.log('[Pipeline] retrieve() response:', {
+      query,
+      namespace: ns,
+      count: resolvedSources.length,
+      sample: resolvedSources.slice(0, 2).map((s) => ({
+        id: s?.id,
+        score: s?.score,
+        contentSnippet: String(s?.content ?? '').substring(0, 150),
+        metadata: s?.metadata,
+      })),
+    });
+
     return { sources: resolvedSources, graphData };
   }
 

package/src/core/QueryProcessor.ts

@@ -360,10 +360,10 @@ Return ONLY 'vector', 'graph', or 'both'. No explanation.`;
           { temperature: 0, maxTokens: 10 }
         );
 
-        const cleanResponse = response.trim().toLowerCase();
-        if (cleanResponse.includes('vector')) return 'vector';
-        if (cleanResponse.includes('graph')) return 'graph';
+        const cleanResponse = (response ?? '').trim().toLowerCase();
         if (cleanResponse.includes('both')) return 'both';
+        if (cleanResponse.includes('graph')) return 'graph';
+        if (cleanResponse.includes('vector')) return 'vector';
       } catch (error) {
         console.warn('[QueryProcessor] LLM strategy classification failed, falling back to keywords:', error);
       }

package/src/llm/providers/UniversalLLMAdapter.ts

@@ -18,6 +18,17 @@ interface UniversalLLMOptions {
   timeout?: number;
 }
 
+export function extractContent(obj: unknown): string | undefined {
+  if (!obj || typeof obj !== 'object') return undefined;
+  const choice = (obj as any).choices?.[0];
+  if (!choice) return undefined;
+  if (typeof choice.message?.content === 'string') return choice.message.content;
+  if (typeof choice.delta?.content === 'string') return choice.delta.content;
+  if (typeof choice.text === 'string') return choice.text;
+  if (typeof choice.content === 'string') return choice.content;
+  return undefined;
+}
+
 export class UniversalLLMAdapter implements ILLMProvider {
   private readonly http: AxiosInstance;
   private readonly model: string;
@@ -96,29 +107,45 @@ export class UniversalLLMAdapter implements ILLMProvider {
 
     const isSelfHost = this.baseUrl.includes('retrivora.com') || this.baseUrl.includes('localhost');
     if (isSelfHost || Boolean(process.env.VERCEL)) {
-      try {
-        const _g = globalThis as any;
-        if (typeof _g.__retrivoraDispatchChat === 'function') {
-          const res = await _g.__retrivoraDispatchChat({
+      const _g = globalThis as any;
+      let dispatch = _g.__retrivoraDispatchChat;
+      if (typeof dispatch !== 'function') {
+        try {
+          const gateway = await import('../../../../../services/llm-gateway/router');
+          if (gateway?.dispatchChatCompletion) {
+            _g.__retrivoraDispatchChat = gateway.dispatchChatCompletion;
+            _g.__retrivoraDispatchEmbedding = gateway.dispatchEmbedding;
+            dispatch = gateway.dispatchChatCompletion;
+          }
+        } catch { /* proceed */ }
+      }
+
+      if (typeof dispatch === 'function') {
+        try {
+          const res = await dispatch({
             model: this.model,
             messages: formattedMessages as any,
             max_tokens: this.maxTokens,
             temperature: this.temperature,
           }, this.apiKey);
 
-          if (res?.response?.choices?.[0]?.message?.content) {
-            return res.response.choices[0].message.content;
+          const content = extractContent(res?.response);
+          if (content !== undefined) {
+            return content;
           }
+          throw new Error(`[UniversalLLMAdapter] In-process dispatch returned empty content for model: ${this.model}`);
+        } catch (dispatchErr) {
+          throw dispatchErr;
         }
-      } catch {
-        /* proceed to HTTP fetch */
+      } else {
+        throw new Error(`[UniversalLLMAdapter] In-process gateway dispatch not registered. Direct self-referential HTTP calls to ${this.baseUrl} are disabled on Vercel to prevent HTTP 508 Loop Detected.`);
       }
     }
 
     const { data } = await this.http.post(path, payload);
 
     const extractPath = this.opts.responseExtractPath ?? 'choices[0].message.content';
-    const result = resolvePath(data, extractPath);
+    const result = resolvePath(data, extractPath) ?? extractContent(data);
 
     if (result === undefined) {
       throw new Error(`[UniversalLLMAdapter] Could not extract text from path '${extractPath}' in response.`);
@@ -170,10 +197,22 @@ export class UniversalLLMAdapter implements ILLMProvider {
     let streamBody: ReadableStream<Uint8Array> | null = null;
 
     if (isSelfHost || Boolean(process.env.VERCEL)) {
-      try {
-        const _g = globalThis as any;
-        if (typeof _g.__retrivoraDispatchChat === 'function') {
-          const res = await _g.__retrivoraDispatchChat({
+      const _g = globalThis as any;
+      let dispatch = _g.__retrivoraDispatchChat;
+      if (typeof dispatch !== 'function') {
+        try {
+          const gateway = await import('../../../../../services/llm-gateway/router');
+          if (gateway?.dispatchChatCompletion) {
+            _g.__retrivoraDispatchChat = gateway.dispatchChatCompletion;
+            _g.__retrivoraDispatchEmbedding = gateway.dispatchEmbedding;
+            dispatch = gateway.dispatchChatCompletion;
+          }
+        } catch { /* proceed */ }
+      }
+
+      if (typeof dispatch === 'function') {
+        try {
+          const res = await dispatch({
             model: this.model,
             messages: formattedMessages as any,
             max_tokens: this.maxTokens,
@@ -183,13 +222,19 @@ export class UniversalLLMAdapter implements ILLMProvider {
 
           if (res?.stream) {
             streamBody = res.stream as ReadableStream<Uint8Array>;
-          } else if (res?.response?.choices?.[0]?.message?.content) {
-            yield res.response.choices[0].message.content;
-            return;
+          } else {
+            const content = extractContent(res?.response);
+            if (content !== undefined) {
+              yield content;
+              return;
+            }
+            throw new Error(`[UniversalLLMAdapter] In-process dispatch stream returned empty response for model: ${this.model}`);
           }
+        } catch (dispatchErr) {
+          throw dispatchErr;
         }
-      } catch {
-        /* proceed to HTTP fetch */
+      } else {
+        throw new Error(`[UniversalLLMAdapter] In-process gateway dispatch not registered. Direct self-referential HTTP calls to ${this.baseUrl} are disabled on Vercel to prevent HTTP 508 Loop Detected.`);
       }
     }
 
@@ -231,7 +276,7 @@ export class UniversalLLMAdapter implements ILLMProvider {
 
           try {
             const json = JSON.parse(trimmed.slice(5).trim());
-            const text = resolvePath(json, extractPath);
+            const text = (resolvePath(json, extractPath) as string | undefined) ?? extractContent(json);
             if (text && typeof text === 'string') yield text;
           } catch {
             // Malformed SSE line — skip
@@ -244,7 +289,7 @@ export class UniversalLLMAdapter implements ILLMProvider {
         const jsonStr = buffer.replace(/^data:\s*/, '').trim();
         try {
           const json = JSON.parse(jsonStr);
-          const text = resolvePath(json, extractPath);
+          const text = (resolvePath(json, extractPath) as string | undefined) ?? extractContent(json);
           if (text && typeof text === 'string') yield text;
         } catch { /* ignore final partial line */ }
       }
@@ -271,10 +316,22 @@ export class UniversalLLMAdapter implements ILLMProvider {
 
     const isSelfHost = this.baseUrl.includes('retrivora.com') || this.baseUrl.includes('localhost');
     if (isSelfHost || Boolean(process.env.VERCEL)) {
-      try {
-        const _g = globalThis as any;
-        if (typeof _g.__retrivoraDispatchEmbedding === 'function') {
-          const res = await _g.__retrivoraDispatchEmbedding({
+      const _g = globalThis as any;
+      let dispatch = _g.__retrivoraDispatchEmbedding;
+      if (typeof dispatch !== 'function') {
+        try {
+          const gateway = await import('../../../../../services/llm-gateway/router');
+          if (gateway?.dispatchEmbedding) {
+            _g.__retrivoraDispatchChat = gateway.dispatchChatCompletion;
+            _g.__retrivoraDispatchEmbedding = gateway.dispatchEmbedding;
+            dispatch = gateway.dispatchEmbedding;
+          }
+        } catch { /* proceed */ }
+      }
+
+      if (typeof dispatch === 'function') {
+        try {
+          const res = await dispatch({
             model: this.model,
             input: text,
           }, this.apiKey);
@@ -282,9 +339,12 @@ export class UniversalLLMAdapter implements ILLMProvider {
           if (res?.data?.[0]?.embedding) {
             return res.data[0].embedding;
           }
+          throw new Error(`[UniversalLLMAdapter] In-process embedding dispatch returned invalid vector for model: ${this.model}`);
+        } catch (dispatchErr) {
+          throw dispatchErr;
         }
-      } catch {
-        /* proceed to HTTP fetch */
+      } else {
+        throw new Error(`[UniversalLLMAdapter] In-process gateway dispatch not registered. Direct self-referential HTTP calls to ${this.baseUrl} are disabled on Vercel to prevent HTTP 508 Loop Detected.`);
       }
     }
 

package/src/rag/Reranker.ts

@@ -25,7 +25,7 @@ export class Reranker {
       }
 
       const scoredMatches = matches.map(match => {
-        const contentLower = match.content.toLowerCase();
+        const contentLower = (match?.content ?? '').toLowerCase();
         let keywordScore = 0;
 
         keywords.forEach(keyword => {
@@ -57,7 +57,7 @@ export class Reranker {
       const topN = matches.slice(0, 10);
 
       const response = await llm.chat(
-        [{ role: 'user', content: `Query: "${query}"\n\nDocuments:\n${topN.map((m, i) => `[${i}] ${m.content.replace(/\n/g, ' ')}`).join('\n')}` }],
+        [{ role: 'user', content: `Query: "${query}"\n\nDocuments:\n${topN.map((m, i) => `[${i}] ${(m?.content ?? '').replace(/\n/g, ' ')}`).join('\n')}` }],
         '',
         {
           systemPrompt: 'You are a relevance ranking expert. Given the user query and a list of retrieved document chunks, rank the chunks by relevance to the query. Return the result as a comma-separated list of document indices in order of relevance, from most relevant to least relevant. Use the indices provided in brackets like [0], [1], etc. Only return the indices (e.g., "2,0,1"), nothing else. Do not include brackets in the output.',

package/src/rendering/IntentClassifier.ts

@@ -107,7 +107,7 @@ export class IntentClassifier {
       numericFieldCount = numericKeys.size;
       categoricalFieldCount = catKeys.size;
 
-      if (productKeyMatches >= 2 || docs.some(d => d.content.toLowerCase().includes('price') || d.content.toLowerCase().includes('brand'))) {
+      if (productKeyMatches >= 2 || docs.some(d => (d?.content ?? '').toLowerCase().includes('price') || (d?.content ?? '').toLowerCase().includes('brand'))) {
         isProductLike = true;
       }
     }

package/src/rendering/RendererRegistry.ts

@@ -13,7 +13,7 @@ export class TextRendererStrategy implements IRendererStrategy {
       return { type: 'text', title, data: { content: data } };
     }
     const docs = Array.isArray(data) ? (data as VectorMatch[]) : [];
-    const text = docs.map(d => d.content).join('\n\n');
+    const text = docs.map(d => d?.content ?? '').join('\n\n');
     return { type: 'text', title, data: { content: text || 'No detailed content available.' } };
   }
 }

package/src/utils/ProductExtractor.ts

@@ -216,7 +216,7 @@ export function extractProductsFromSources(sources: VectorMatch[] | undefined, i
 
       return {
         id: s.id,
-        name: name ?? s.content.split('\n')[0] ?? 'Unknown Product',
+        name: name ?? (s?.content ?? '').split('\n')[0] ?? 'Unknown Product',
         brand: brand,
         price: price,
         image: resolveImage(m),

package/src/utils/UITransformer.ts

@@ -128,6 +128,17 @@ export class UITransformer {
       return this.createTextResponse('No data available', 'No relevant data found for your query.');
     }
 
+    console.log('[UITransformer.transform] Processing retrievedData:', {
+      userQuery,
+      retrievedCount: retrievedData.length,
+      sample: retrievedData.slice(0, 2).map((item) => ({
+        id: item?.id,
+        contentType: typeof item?.content,
+        contentSnippet: String(item?.content ?? '').substring(0, 150),
+        metadataKeys: Object.keys(item?.metadata || {}),
+      })),
+    });
+
     const resolvedIntent = intent ?? this.detectIntentHeuristic(userQuery);
 
     const filteredData = resolvedIntent.filterInStockOnly
@@ -549,8 +560,7 @@ RULES:
   ): UITransformationResponse {
     const products: CarouselProduct[] = data
       .map(item => this.extractProductInfo(item, config, trainedSchema))
-      .filter((p): p is CarouselProduct => p !== null)
-      .slice(0, 15);
+      .filter((p): p is CarouselProduct => p !== null);
 
     return {
       type: 'product_carousel',
@@ -760,7 +770,7 @@ RULES:
       type: 'radar_chart' as VisualizationType,
       title: 'Product Comparison',
       description: `Comparing ${data.length} items across ${radarData.length} attributes`,
-      data: radarData.length > 0 ? radarData : data.map(d => ({ attribute: d.content.substring(0, 40) })),
+      data: radarData.length > 0 ? radarData : data.map(d => ({ attribute: (d?.content ?? '').substring(0, 40) })),
     };
   }
 
@@ -780,7 +790,7 @@ RULES:
   private static transformToText(data: VectorMatch[]): UITransformationResponse {
     return this.createTextResponse(
       'Retrieved Context',
-      data.map(item => item.content).join('\n\n'),
+      data.map(item => item?.content ?? '').join('\n\n'),
       `Found ${data.length} relevant results`,
     );
   }
@@ -983,6 +993,7 @@ RULES:
 
 
   private static isProductData(item: VectorMatch): boolean {
+    if (!item) return false;
     const content = (item.content || '').toLowerCase();
     const productKeywords = [
       'product', 'price', 'stock', 'sku', 'brand', 'msrp',
@@ -1023,7 +1034,7 @@ RULES:
   }
 
   private static profileData(data: VectorMatch[]): DataProfile {
-    const records = data.map((item): DataRecord => {
+    const records = (data || []).filter((item): item is VectorMatch => Boolean(item && typeof item === 'object')).map((item): DataRecord => {
       const fields: Record<string, PrimitiveValue> = {};
       Object.entries(item.metadata || {}).forEach(([key, value]) => {
         const primitive = this.toPrimitive(value);
@@ -1032,8 +1043,8 @@ RULES:
       if (!fields.content && item.content) fields.content = item.content.substring(0, 500);
       return {
         id: item.id,
-        content: item.content,
-        score: item.score,
+        content: item.content ?? '',
+        score: item.score ?? 0,
         fields,
         source: item,
       };
@@ -1297,7 +1308,7 @@ RULES:
       return {
         timestamp: (meta.timestamp ?? meta.date ?? new Date().toISOString()) as string | number,
         value: (meta.value ?? item.score ?? 0) as number,
-        label: (meta.label ?? item.content.substring(0, 50)) as string,
+        label: (meta.label ?? (item?.content ?? '').substring(0, 50)) as string,
       };
     });
   }
@@ -1429,7 +1440,7 @@ RULES:
   }
 
   private static resolveTableCellValue(item: VectorMatch, column: string): string | number | boolean {
-    if (column === 'Content') return item.content.substring(0, 100);
+    if (column === 'Content') return (item?.content ?? '').substring(0, 100);
 
     const meta = item.metadata || {};
     const normalizedColumn = this.normalizeComparableField(column);
@@ -1444,10 +1455,10 @@ RULES:
     const aliasValue = this.resolveAliasedTableCell(meta, column);
     if (aliasValue !== undefined) return this.toDisplayValue(aliasValue);
 
-    const contentValue = this.extractContentFieldValue(item.content, column);
+    const contentValue = this.extractContentFieldValue(item?.content ?? '', column);
     if (contentValue !== null) return contentValue;
 
-    const aliasContentValue = this.extractAliasedContentFieldValue(item.content, column);
+    const aliasContentValue = this.extractAliasedContentFieldValue(item?.content ?? '', column);
     return aliasContentValue ?? '';
   }
 
@@ -1519,6 +1530,7 @@ RULES:
   }
 
   private static determineStockStatus(item: VectorMatch): boolean {
+    if (!item) return true;
     const meta = item.metadata || {};
     const stockValue = resolveMetadataValue(meta, 'stock');
     if (stockValue !== undefined) {
@@ -1583,42 +1595,74 @@ RULES:
     config?: import('../config/RagConfig').RagConfig,
     trainedSchema?: import('./SchemaMapper').SchemaMap,
   ): CarouselProduct | null {
+    if (!item) return null;
     const meta = item.metadata || {};
+    const content = item.content || '';
     const name = this.getDynamicVal(meta, 'name', config, trainedSchema);
     const price = this.getDynamicVal(meta, 'price', config, trainedSchema);
     const brand = this.getDynamicVal(meta, 'brand', config, trainedSchema);
     const description = this.cleanProductDescription(
-      this.extractProductDescriptionFromContent(item.content) ??
+      this.extractProductDescriptionFromContent(content) ??
       this.getProductDescriptionValue(meta, config, trainedSchema),
     );
 
     if (name || this.isProductData(item)) {
       let finalName = name ? String(name) : undefined;
+      if (!finalName && content) {
+        const nameMatch = content.match(/(?:Product|Item|Name):\s*([^\n]+)/i);
+        finalName = nameMatch ? nameMatch[1].trim() : content.split('\n')[0]?.substring(0, 60);
+      }
+
       if (!finalName) {
-        const nameMatch = item.content.match(/(?:Product|Item|Name):\s*([^\n]+)/i);
-        finalName = nameMatch ? nameMatch[1].trim() : item.content.split('\n')[0].substring(0, 60);
+        console.warn(
+          `[UITransformer] Unable to read product name from metadata or content for item ${item.id}. Defaulting to 'Unnamed Product'.`,
+          { metadataKeys: Object.keys(meta), contentLength: content.length },
+        );
       }
 
       let finalPrice: string | number | undefined =
         typeof price === 'number' || typeof price === 'string' ? price : undefined;
-      if (!finalPrice) {
+      if (!finalPrice && content) {
         const priceMatch =
-          item.content.match(/(?:Price|Cost):\s*\$?([\d,.]+)/i) ||
-          item.content.match(/\$\s*([\d,.]+)/);
+          content.match(/(?:Price|Cost):\s*\$?([\d,.]+)/i) ||
+          content.match(/\$\s*([\d,.]+)/);
         if (priceMatch) finalPrice = priceMatch[1].replace(/,/g, '');
       }
 
+      if (finalPrice === undefined) {
+        console.warn(
+          `[UITransformer] Unable to read product price from metadata or content for item ${item.id}.`,
+          { metadataKeys: Object.keys(meta), contentSnippet: content.substring(0, 100) },
+        );
+      }
+
       const imageValue = this.getDynamicVal(meta, 'image', config, trainedSchema);
+      if (!imageValue) {
+        console.debug(
+          `[UITransformer] Unable to read product image URL from metadata for item ${item.id}.`,
+        );
+      }
+
+      if (!description) {
+        console.debug(
+          `[UITransformer] Unable to read product description from metadata or content for item ${item.id}.`,
+        );
+      }
 
       return {
         id: item.id,
-        name: finalName,
+        name: finalName || 'Unnamed Product',
         price: finalPrice,
         image: typeof imageValue === 'string' ? imageValue : undefined,
         brand: brand ? String(brand) : undefined,
         description,
         inStock: this.determineStockStatus(item),
       };
+    } else {
+      console.warn(
+        `[UITransformer] Item ${item.id} skipped: unable to read name or identify as product data from metadata/content.`,
+        { metadataKeys: Object.keys(meta), contentSnippet: content.substring(0, 100) },
+      );
     }
 
     return null;
@@ -1733,11 +1777,11 @@ RULES:
   }
 
   private static buildContextSummary(sources: VectorMatch[], maxChars = 6000): string {
-    const items = sources.map((s, i) => ({
+    const items = (sources || []).filter(Boolean).map((s, i) => ({
       index: i + 1,
-      content: s.content?.substring(0, 400) ?? '',
-      metadata: s.metadata ?? {},
-      score: s.score ?? 0,
+      content: s?.content?.substring(0, 400) ?? '',
+      metadata: s?.metadata ?? {},
+      score: s?.score ?? 0,
     }));
 
     const full = JSON.stringify(items, null, 2);