@retrivora-ai/rag-engine 2.0.5 → 2.0.7

package/src/core/DatabaseStorage.ts

@@ -1,5 +1,6 @@
 import * as fs from 'fs';
 import * as path from 'path';
+import * as os from 'os';
 import { RagConfig } from '../config/RagConfig';
 import { LicenseVerifier } from './LicenseVerifier';
 
@@ -35,7 +36,10 @@ export class DatabaseStorage {
   private feedbackTableName: string;
 
   // Filesystem fallback configuration
-  private fallbackDir = path.join(process.cwd(), '.retrivora');
+  private isServerless = Boolean(process.env.VERCEL || process.env.AWS_LAMBDA_FUNCTION_NAME || process.env.LAMBDA_TASK_ROOT);
+  private fallbackDir = this.isServerless
+    ? path.join(os.tmpdir(), '.retrivora')
+    : path.join(process.cwd(), '.retrivora');
   private historyFile = path.join(this.fallbackDir, 'history.json');
   private feedbackFile = path.join(this.fallbackDir, 'feedback.json');
 
@@ -191,10 +195,28 @@ export class DatabaseStorage {
 
   // Helper for FS fallback writes
   private writeLocalFile(filePath: string, data: any[]): void {
-    if (!fs.existsSync(this.fallbackDir)) {
-      fs.mkdirSync(this.fallbackDir, { recursive: true });
+    try {
+      const dir = path.dirname(filePath);
+      if (!fs.existsSync(dir)) {
+        fs.mkdirSync(dir, { recursive: true });
+      }
+      fs.writeFileSync(filePath, JSON.stringify(data, null, 2), 'utf-8');
+    } catch (err: any) {
+      if (err.code === 'EROFS') {
+        try {
+          const tmpDir = path.join(os.tmpdir(), '.retrivora');
+          if (!fs.existsSync(tmpDir)) {
+            fs.mkdirSync(tmpDir, { recursive: true });
+          }
+          const tmpFilePath = path.join(tmpDir, path.basename(filePath));
+          fs.writeFileSync(tmpFilePath, JSON.stringify(data, null, 2), 'utf-8');
+        } catch {
+          /* ignore write failure in read-only environment */
+        }
+      } else {
+        console.warn(`[DatabaseStorage] Failed to write local fallback file: ${err.message}`);
+      }
     }
-    fs.writeFileSync(filePath, JSON.stringify(data, null, 2), 'utf-8');
   }
 
   // ─── Message History Operations ──────────────────────────────────────────

package/src/core/Pipeline.ts

@@ -583,9 +583,20 @@ You are a helpful assistant. Use the provided context to answer questions accura
       const wantsExhaustiveList = hasNumericPredicates || hasMetadataFilter || /\b(list|all|show|provide|give|get|browse|find|search|display)\b/i.test(question);
       const retrievalLimit = wantsExhaustiveList ? Math.max(topK * 20, 100) : topK * 2;
 
-      const rawSources = (strategyResult === 'vector' || strategyResult === 'both') && queryVector && queryVector.length > 0
+      const rawSources = ((strategyResult === 'vector' || strategyResult === 'both') && queryVector && queryVector.length > 0
         ? await this.vectorDB.query(queryVector, retrievalLimit, ns, filter)
-        : [];
+        : []).filter((s): s is VectorMatch => Boolean(s && typeof s === 'object'));
+
+      console.log('[Pipeline] Raw vectorDB.query response:', {
+        rawCount: rawSources.length,
+        sample: rawSources.slice(0, 2).map((s) => ({
+          id: s?.id,
+          score: s?.score,
+          contentType: typeof s?.content,
+          contentSnippet: String(s?.content ?? '').substring(0, 150),
+          metadataKeys: Object.keys(s?.metadata || {}),
+        })),
+      });
 
       const retrieveEnd = performance.now();
       const embedMs = retrieveEnd - embedStart;
@@ -593,14 +604,14 @@ You are a helpful assistant. Use the provided context to answer questions accura
 
       // 3. Reranking
       const rerankStart = performance.now();
-      const structuredSources = this.applyStructuredFilters(rawSources, filter);
+      const structuredSources = this.applyStructuredFilters(rawSources, filter).filter((s): s is VectorMatch => Boolean(s && typeof s === 'object'));
       let fullSources = hasNumericPredicates
         ? structuredSources
-        : structuredSources.filter((m) => m.score >= scoreThreshold);
+        : structuredSources.filter((m) => (m?.score ?? 0) >= scoreThreshold);
       const rerankLimit = wantsExhaustiveList ? retrievalLimit : topK;
       const useReranking = arch !== 'naive' && (arch === 'retrieve-and-rerank' || this.config.rag?.useReranking);
       if (!hasNumericPredicates && useReranking) {
-        fullSources = await this.reranker.rerank(fullSources, question, rerankLimit);
+        fullSources = (await this.reranker.rerank(fullSources, question, rerankLimit)).filter((s): s is VectorMatch => Boolean(s && typeof s === 'object'));
       } else if (!wantsExhaustiveList) {
         fullSources = fullSources.slice(0, topK);
       }
@@ -608,7 +619,7 @@ You are a helpful assistant. Use the provided context to answer questions accura
 
       // 4. Context Augmentation - Review all retrieved/reranked sources for full comprehension
       let context = fullSources.length
-        ? fullSources.map((m, i) => `[Source ${i + 1}]\n${m.content}`).join('\n\n---\n\n')
+        ? fullSources.map((m, i) => `[Source ${i + 1}]\n${m?.content ?? ''}`).join('\n\n---\n\n')
         : 'No relevant context found.';
 
       // Count the records of relevant sources to pass into the slice dynamically
@@ -620,7 +631,7 @@ You are a helpful assistant. Use the provided context to answer questions accura
       } else {
         // For general semantic queries, count the number of matches meeting high relevance threshold
         const baseThreshold = Math.max(scoreThreshold, 0.4);
-        const highlyRelevant = fullSources.filter(m => m.score >= baseThreshold);
+        const highlyRelevant = fullSources.filter(m => (m?.score ?? 0) >= baseThreshold);
         displayCount = Math.max(highlyRelevant.length, topK);
         // Cap non-metadata semantic searches to a maximum of 15 to keep the UI clean
         if (displayCount > 15) {
@@ -630,9 +641,22 @@ You are a helpful assistant. Use the provided context to answer questions accura
 
       // Ensure they are sorted from highest to lowest score, then slice using our dynamic count
       const sources = [...fullSources]
-        .sort((a, b) => b.score - a.score)
+        .filter((s): s is VectorMatch => Boolean(s && typeof s === 'object'))
+        .sort((a, b) => (b?.score ?? 0) - (a?.score ?? 0))
         .slice(0, displayCount);
 
+      console.log('[Pipeline] Final sources for prompt & UI:', {
+        count: sources.length,
+        sources: sources.map((s, idx) => ({
+          rank: idx + 1,
+          id: s?.id,
+          score: s?.score,
+          contentType: typeof s?.content,
+          contentSnippet: String(s?.content ?? '').substring(0, 150),
+          metadata: s?.metadata,
+        })),
+      });
+
       if (graphData && graphData.nodes.length > 0) {
         const graphContext = graphData.nodes.map(n =>
           `Entity: ${n.label} (${n.id})${n.properties ? ' - ' + JSON.stringify(n.properties) : ''}`
@@ -653,17 +677,17 @@ You are a helpful assistant. Use the provided context to answer questions accura
       // Yield sources immediately to support early UI citation rendering (highly responsive)
       yield {
         reply: '',
-        sources: sources.map((s) => ({
+        sources: (sources || []).filter((s): s is VectorMatch => Boolean(s && typeof s === 'object')).map((s) => ({
           id: s.id,
-          score: s.score,
-          content: s.content,
-          metadata: s.metadata ?? {},
+          score: s.score ?? 0,
+          content: s?.content ?? '',
+          metadata: s?.metadata ?? {},
           namespace: ns,
         } satisfies RetrievedChunk)),
       } as ChatResponse;
 
       // 4.5 Schema Training — true fire-and-forget, never gates anything.
-      const allMetadataKeys = Array.from(new Set(sources.flatMap(s => Object.keys(s.metadata || {}))));
+      const allMetadataKeys = Array.from(new Set(sources.filter(Boolean).flatMap(s => Object.keys(s?.metadata || {}))));
       const cachedSchema = allMetadataKeys.length > 0
         ? SchemaMapper.getCached(ns, allMetadataKeys)
         : undefined;
@@ -722,7 +746,12 @@ You are a helpful assistant. Use the provided context to answer questions accura
         finalRestrictionSuffix += '\n\n(IMPORTANT: You must think step-by-step before answering. Write your thinking process inside a `<think>...</think>` block. Write the `<think>` block first, then follow it with your final response. Keep the thinking process thorough and detailed. Example: `<think>Thinking details here...</think>Final answer text here.`)';
       }
 
-      const hardenedHistory = [...history];
+      const hardenedHistory = (Array.isArray(history) ? history : [])
+        .filter((m): m is ChatMessage => Boolean(m && typeof m === 'object'))
+        .map((m) => ({
+          role: m.role || 'user',
+          content: typeof m.content === 'string' ? m.content : (m.content != null ? String(m.content) : ''),
+        }));
       const userQuestion = { role: 'user', content: question + finalRestrictionSuffix } as ChatMessage;
       const messages: ChatMessage[] = [...hardenedHistory, userQuestion];
 
@@ -880,11 +909,11 @@ You are a helpful assistant. Use the provided context to answer questions accura
         rewrittenQuery,
         systemPrompt,
         userPrompt: question + finalRestrictionSuffix,
-        chunks: sources.map((s) => ({
+        chunks: (sources || []).filter(Boolean).map((s) => ({
           id: s.id,
           score: s.score,
-          content: s.content,
-          metadata: s.metadata ?? {},
+          content: s?.content ?? '',
+          metadata: s?.metadata ?? {},
           namespace: ns,
         } satisfies RetrievedChunk)),
         latency,
@@ -1010,6 +1039,7 @@ You are a helpful assistant. Use the provided context to answer questions accura
   }
 
   private resolveNumericPredicateValue(source: VectorMatch, predicate: NumericPredicate): number | null {
+    if (!source) return null;
     const meta = source.metadata || {};
     const field = predicate.field;
     const entries = Object.entries(meta)
@@ -1027,7 +1057,7 @@ You are a helpful assistant. Use the provided context to answer questions accura
         if (value !== null) return value;
       }
 
-      const contentValue = this.extractNumericValueFromContent(source.content, field);
+      const contentValue = this.extractNumericValueFromContent(source.content ?? '', field);
       if (contentValue !== null) return contentValue;
     }
 
@@ -1041,6 +1071,7 @@ You are a helpful assistant. Use the provided context to answer questions accura
   }
 
   private extractNumericValueFromContent(content: string, field: string): number | null {
+    if (!content || typeof content !== 'string') return null;
     const escapedWords = field
       .split(/\s+|_+|-+/)
       .filter(Boolean)
@@ -1159,6 +1190,18 @@ You are a helpful assistant. Use the provided context to answer questions accura
       }
     }
 
+    console.log('[Pipeline] retrieve() response:', {
+      query,
+      namespace: ns,
+      count: resolvedSources.length,
+      sample: resolvedSources.slice(0, 2).map((s) => ({
+        id: s?.id,
+        score: s?.score,
+        contentSnippet: String(s?.content ?? '').substring(0, 150),
+        metadata: s?.metadata,
+      })),
+    });
+
     return { sources: resolvedSources, graphData };
   }
 

package/src/core/QueryProcessor.ts

@@ -360,10 +360,10 @@ Return ONLY 'vector', 'graph', or 'both'. No explanation.`;
           { temperature: 0, maxTokens: 10 }
         );
 
-        const cleanResponse = response.trim().toLowerCase();
-        if (cleanResponse.includes('vector')) return 'vector';
-        if (cleanResponse.includes('graph')) return 'graph';
+        const cleanResponse = (response ?? '').trim().toLowerCase();
         if (cleanResponse.includes('both')) return 'both';
+        if (cleanResponse.includes('graph')) return 'graph';
+        if (cleanResponse.includes('vector')) return 'vector';
       } catch (error) {
         console.warn('[QueryProcessor] LLM strategy classification failed, falling back to keywords:', error);
       }

package/src/llm/providers/UniversalLLMAdapter.ts

@@ -94,6 +94,27 @@ export class UniversalLLMAdapter implements ILLMProvider {
       };
     }
 
+    const isSelfHost = this.baseUrl.includes('retrivora.com') || this.baseUrl.includes('localhost');
+    if (isSelfHost || Boolean(process.env.VERCEL)) {
+      const _g = globalThis as any;
+      if (typeof _g.__retrivoraDispatchChat === 'function') {
+        try {
+          const res = await _g.__retrivoraDispatchChat({
+            model: this.model,
+            messages: formattedMessages as any,
+            max_tokens: this.maxTokens,
+            temperature: this.temperature,
+          }, this.apiKey);
+
+          if (res?.response?.choices?.[0]?.message?.content) {
+            return res.response.choices[0].message.content;
+          }
+        } catch (dispatchErr) {
+          throw dispatchErr;
+        }
+      }
+    }
+
     const { data } = await this.http.post(path, payload);
 
     const extractPath = this.opts.responseExtractPath ?? 'choices[0].message.content';
@@ -145,22 +166,54 @@ export class UniversalLLMAdapter implements ILLMProvider {
       };
     }
 
-    const response = await fetch(url, {
-      method: 'POST',
-      headers: this.resolvedHeaders,
-      body: JSON.stringify(payload),
-    });
+    const isSelfHost = this.baseUrl.includes('retrivora.com') || this.baseUrl.includes('localhost');
+    let streamBody: ReadableStream<Uint8Array> | null = null;
 
-    if (!response.ok) {
-      const errorText = await response.text().catch(() => response.statusText);
-      throw new Error(`[UniversalLLMAdapter] Streaming request failed (${response.status}): ${errorText}`);
+    if (isSelfHost || Boolean(process.env.VERCEL)) {
+      const _g = globalThis as any;
+      if (typeof _g.__retrivoraDispatchChat === 'function') {
+        try {
+          const res = await _g.__retrivoraDispatchChat({
+            model: this.model,
+            messages: formattedMessages as any,
+            max_tokens: this.maxTokens,
+            temperature: this.temperature,
+            stream: true,
+          }, this.apiKey);
+
+          if (res?.stream) {
+            streamBody = res.stream as ReadableStream<Uint8Array>;
+          } else if (res?.response?.choices?.[0]?.message?.content) {
+            yield res.response.choices[0].message.content;
+            return;
+          }
+        } catch (dispatchErr) {
+          // If in-process dispatch threw a real error (Rate limit, Auth, API error), rethrow it
+          // rather than silently falling back to external HTTP fetch which masks the error.
+          throw dispatchErr;
+        }
+      }
     }
 
-    if (!response.body) {
-      throw new Error('[UniversalLLMAdapter] Response body is null — server did not send a streaming response.');
+    if (!streamBody) {
+      const response = await fetch(url, {
+        method: 'POST',
+        headers: this.resolvedHeaders,
+        body: JSON.stringify(payload),
+      });
+
+      if (!response.ok) {
+        const errorText = await response.text().catch(() => response.statusText);
+        throw new Error(`[UniversalLLMAdapter] Streaming request failed (${response.status}): ${errorText}`);
+      }
+
+      if (!response.body) {
+        throw new Error('[UniversalLLMAdapter] Response body is null — server did not send a streaming response.');
+      }
+      streamBody = response.body;
     }
 
-    const reader = response.body.getReader();
+    const reader = streamBody.getReader();
     const decoder = new TextDecoder('utf-8');
     let buffer = '';
 
@@ -218,6 +271,25 @@ export class UniversalLLMAdapter implements ILLMProvider {
       };
     }
 
+    const isSelfHost = this.baseUrl.includes('retrivora.com') || this.baseUrl.includes('localhost');
+    if (isSelfHost || Boolean(process.env.VERCEL)) {
+      const _g = globalThis as any;
+      if (typeof _g.__retrivoraDispatchEmbedding === 'function') {
+        try {
+          const res = await _g.__retrivoraDispatchEmbedding({
+            model: this.model,
+            input: text,
+          }, this.apiKey);
+
+          if (res?.data?.[0]?.embedding) {
+            return res.data[0].embedding;
+          }
+        } catch (dispatchErr) {
+          throw dispatchErr;
+        }
+      }
+    }
+
     const { data } = await this.http.post(path, payload);
 
     const extractPath = this.opts.embedExtractPath ?? 'data[0].embedding';

package/src/rag/Reranker.ts

@@ -25,7 +25,7 @@ export class Reranker {
       }
 
       const scoredMatches = matches.map(match => {
-        const contentLower = match.content.toLowerCase();
+        const contentLower = (match?.content ?? '').toLowerCase();
         let keywordScore = 0;
 
         keywords.forEach(keyword => {
@@ -57,7 +57,7 @@ export class Reranker {
       const topN = matches.slice(0, 10);
 
       const response = await llm.chat(
-        [{ role: 'user', content: `Query: "${query}"\n\nDocuments:\n${topN.map((m, i) => `[${i}] ${m.content.replace(/\n/g, ' ')}`).join('\n')}` }],
+        [{ role: 'user', content: `Query: "${query}"\n\nDocuments:\n${topN.map((m, i) => `[${i}] ${(m?.content ?? '').replace(/\n/g, ' ')}`).join('\n')}` }],
         '',
         {
           systemPrompt: 'You are a relevance ranking expert. Given the user query and a list of retrieved document chunks, rank the chunks by relevance to the query. Return the result as a comma-separated list of document indices in order of relevance, from most relevant to least relevant. Use the indices provided in brackets like [0], [1], etc. Only return the indices (e.g., "2,0,1"), nothing else. Do not include brackets in the output.',

package/src/rendering/IntentClassifier.ts

@@ -107,7 +107,7 @@ export class IntentClassifier {
       numericFieldCount = numericKeys.size;
       categoricalFieldCount = catKeys.size;
 
-      if (productKeyMatches >= 2 || docs.some(d => d.content.toLowerCase().includes('price') || d.content.toLowerCase().includes('brand'))) {
+      if (productKeyMatches >= 2 || docs.some(d => (d?.content ?? '').toLowerCase().includes('price') || (d?.content ?? '').toLowerCase().includes('brand'))) {
         isProductLike = true;
       }
     }

package/src/rendering/RendererRegistry.ts

@@ -13,7 +13,7 @@ export class TextRendererStrategy implements IRendererStrategy {
       return { type: 'text', title, data: { content: data } };
     }
     const docs = Array.isArray(data) ? (data as VectorMatch[]) : [];
-    const text = docs.map(d => d.content).join('\n\n');
+    const text = docs.map(d => d?.content ?? '').join('\n\n');
     return { type: 'text', title, data: { content: text || 'No detailed content available.' } };
   }
 }

package/src/utils/ProductExtractor.ts

@@ -216,7 +216,7 @@ export function extractProductsFromSources(sources: VectorMatch[] | undefined, i
 
       return {
         id: s.id,
-        name: name ?? s.content.split('\n')[0] ?? 'Unknown Product',
+        name: name ?? (s?.content ?? '').split('\n')[0] ?? 'Unknown Product',
         brand: brand,
         price: price,
         image: resolveImage(m),

package/src/utils/UITransformer.ts

@@ -128,6 +128,17 @@ export class UITransformer {
       return this.createTextResponse('No data available', 'No relevant data found for your query.');
     }
 
+    console.log('[UITransformer.transform] Processing retrievedData:', {
+      userQuery,
+      retrievedCount: retrievedData.length,
+      sample: retrievedData.slice(0, 2).map((item) => ({
+        id: item?.id,
+        contentType: typeof item?.content,
+        contentSnippet: String(item?.content ?? '').substring(0, 150),
+        metadataKeys: Object.keys(item?.metadata || {}),
+      })),
+    });
+
     const resolvedIntent = intent ?? this.detectIntentHeuristic(userQuery);
 
     const filteredData = resolvedIntent.filterInStockOnly
@@ -760,7 +771,7 @@ RULES:
       type: 'radar_chart' as VisualizationType,
       title: 'Product Comparison',
       description: `Comparing ${data.length} items across ${radarData.length} attributes`,
-      data: radarData.length > 0 ? radarData : data.map(d => ({ attribute: d.content.substring(0, 40) })),
+      data: radarData.length > 0 ? radarData : data.map(d => ({ attribute: (d?.content ?? '').substring(0, 40) })),
     };
   }
 
@@ -780,7 +791,7 @@ RULES:
   private static transformToText(data: VectorMatch[]): UITransformationResponse {
     return this.createTextResponse(
       'Retrieved Context',
-      data.map(item => item.content).join('\n\n'),
+      data.map(item => item?.content ?? '').join('\n\n'),
       `Found ${data.length} relevant results`,
     );
   }
@@ -983,6 +994,7 @@ RULES:
 
 
   private static isProductData(item: VectorMatch): boolean {
+    if (!item) return false;
     const content = (item.content || '').toLowerCase();
     const productKeywords = [
       'product', 'price', 'stock', 'sku', 'brand', 'msrp',
@@ -1297,7 +1309,7 @@ RULES:
       return {
         timestamp: (meta.timestamp ?? meta.date ?? new Date().toISOString()) as string | number,
         value: (meta.value ?? item.score ?? 0) as number,
-        label: (meta.label ?? item.content.substring(0, 50)) as string,
+        label: (meta.label ?? (item?.content ?? '').substring(0, 50)) as string,
       };
     });
   }
@@ -1429,7 +1441,7 @@ RULES:
   }
 
   private static resolveTableCellValue(item: VectorMatch, column: string): string | number | boolean {
-    if (column === 'Content') return item.content.substring(0, 100);
+    if (column === 'Content') return (item?.content ?? '').substring(0, 100);
 
     const meta = item.metadata || {};
     const normalizedColumn = this.normalizeComparableField(column);
@@ -1444,10 +1456,10 @@ RULES:
     const aliasValue = this.resolveAliasedTableCell(meta, column);
     if (aliasValue !== undefined) return this.toDisplayValue(aliasValue);
 
-    const contentValue = this.extractContentFieldValue(item.content, column);
+    const contentValue = this.extractContentFieldValue(item?.content ?? '', column);
     if (contentValue !== null) return contentValue;
 
-    const aliasContentValue = this.extractAliasedContentFieldValue(item.content, column);
+    const aliasContentValue = this.extractAliasedContentFieldValue(item?.content ?? '', column);
     return aliasContentValue ?? '';
   }
 
@@ -1519,6 +1531,7 @@ RULES:
   }
 
   private static determineStockStatus(item: VectorMatch): boolean {
+    if (!item) return true;
     const meta = item.metadata || {};
     const stockValue = resolveMetadataValue(meta, 'stock');
     if (stockValue !== undefined) {
@@ -1583,42 +1596,74 @@ RULES:
     config?: import('../config/RagConfig').RagConfig,
     trainedSchema?: import('./SchemaMapper').SchemaMap,
   ): CarouselProduct | null {
+    if (!item) return null;
     const meta = item.metadata || {};
+    const content = item.content || '';
     const name = this.getDynamicVal(meta, 'name', config, trainedSchema);
     const price = this.getDynamicVal(meta, 'price', config, trainedSchema);
     const brand = this.getDynamicVal(meta, 'brand', config, trainedSchema);
     const description = this.cleanProductDescription(
-      this.extractProductDescriptionFromContent(item.content) ??
+      this.extractProductDescriptionFromContent(content) ??
       this.getProductDescriptionValue(meta, config, trainedSchema),
     );
 
     if (name || this.isProductData(item)) {
       let finalName = name ? String(name) : undefined;
+      if (!finalName && content) {
+        const nameMatch = content.match(/(?:Product|Item|Name):\s*([^\n]+)/i);
+        finalName = nameMatch ? nameMatch[1].trim() : content.split('\n')[0]?.substring(0, 60);
+      }
+
       if (!finalName) {
-        const nameMatch = item.content.match(/(?:Product|Item|Name):\s*([^\n]+)/i);
-        finalName = nameMatch ? nameMatch[1].trim() : item.content.split('\n')[0].substring(0, 60);
+        console.warn(
+          `[UITransformer] Unable to read product name from metadata or content for item ${item.id}. Defaulting to 'Unnamed Product'.`,
+          { metadataKeys: Object.keys(meta), contentLength: content.length },
+        );
       }
 
       let finalPrice: string | number | undefined =
         typeof price === 'number' || typeof price === 'string' ? price : undefined;
-      if (!finalPrice) {
+      if (!finalPrice && content) {
         const priceMatch =
-          item.content.match(/(?:Price|Cost):\s*\$?([\d,.]+)/i) ||
-          item.content.match(/\$\s*([\d,.]+)/);
+          content.match(/(?:Price|Cost):\s*\$?([\d,.]+)/i) ||
+          content.match(/\$\s*([\d,.]+)/);
         if (priceMatch) finalPrice = priceMatch[1].replace(/,/g, '');
       }
 
+      if (finalPrice === undefined) {
+        console.warn(
+          `[UITransformer] Unable to read product price from metadata or content for item ${item.id}.`,
+          { metadataKeys: Object.keys(meta), contentSnippet: content.substring(0, 100) },
+        );
+      }
+
       const imageValue = this.getDynamicVal(meta, 'image', config, trainedSchema);
+      if (!imageValue) {
+        console.debug(
+          `[UITransformer] Unable to read product image URL from metadata for item ${item.id}.`,
+        );
+      }
+
+      if (!description) {
+        console.debug(
+          `[UITransformer] Unable to read product description from metadata or content for item ${item.id}.`,
+        );
+      }
 
       return {
         id: item.id,
-        name: finalName,
+        name: finalName || 'Unnamed Product',
         price: finalPrice,
         image: typeof imageValue === 'string' ? imageValue : undefined,
         brand: brand ? String(brand) : undefined,
         description,
         inStock: this.determineStockStatus(item),
       };
+    } else {
+      console.warn(
+        `[UITransformer] Item ${item.id} skipped: unable to read name or identify as product data from metadata/content.`,
+        { metadataKeys: Object.keys(meta), contentSnippet: content.substring(0, 100) },
+      );
     }
 
     return null;
@@ -1733,11 +1778,11 @@ RULES:
   }
 
   private static buildContextSummary(sources: VectorMatch[], maxChars = 6000): string {
-    const items = sources.map((s, i) => ({
+    const items = (sources || []).filter(Boolean).map((s, i) => ({
       index: i + 1,
-      content: s.content?.substring(0, 400) ?? '',
-      metadata: s.metadata ?? {},
-      score: s.score ?? 0,
+      content: s?.content?.substring(0, 400) ?? '',
+      metadata: s?.metadata ?? {},
+      score: s?.score ?? 0,
     }));
 
     const full = JSON.stringify(items, null, 2);