@retrivora-ai/rag-engine 1.9.3 → 1.9.7

package/src/utils/UITransformer.ts

@@ -217,7 +217,7 @@ export class UITransformer {
     // ── Step 1: Detect intent ───────────────────────────────────────────────
     let intent: QueryIntent;
     try {
-      intent = await this.detectIntent(query, llm);
+      intent = await this.detectIntent(query, llm, sources);
       console.debug('[UITransformer] Detected intent:', intent);
     } catch (err) {
       console.warn('[UITransformer] Intent detection failed; using heuristic.', err);
@@ -239,11 +239,24 @@ export class UITransformer {
       const context = this.buildContextSummary(sources);
       const systemPrompt = this.buildVisualizationSystemPrompt();
 
+      const profile = this.profileData(sources);
+      const schemaContext = `
+RETRIEVED DATA SCHEMA:
+- Numeric fields (measures): ${profile.numericFields.map(f => `${f.key} (unique values: ${f.uniqueCount})`).join(', ') || 'None'}
+- Category fields (dimensions): ${profile.categoricalFields.map(f => `${f.key} (unique values: ${f.uniqueCount})`).join(', ') || 'None'}
+- Date fields (temporal): ${profile.dateFields.map(f => `${f.key} (unique values: ${f.uniqueCount})`).join(', ') || 'None'}
+- Boolean fields: ${profile.booleanFields.map(f => `${f.key} (unique values: ${f.uniqueCount})`).join(', ') || 'None'}
+- Text/Other fields: ${profile.fields.filter(f => f.kind === 'text').map(f => f.key).join(', ') || 'None'}
+`;
+
       const userPrompt = [
         `USER QUESTION: ${query}`,
         '',
         `DETECTED INTENT (JSON): ${JSON.stringify(intent)}`,
         '',
+        `RETRIEVED DATA SCHEMA:`,
+        schemaContext,
+        '',
         'RETRIEVED DATA (JSON):',
         context,
       ].join('\n');
@@ -256,6 +269,26 @@ export class UITransformer {
 
       const parsed = this.parseTransformationResponse(rawResponse);
       if (parsed) {
+        // Validate suitability of selected chart type based on schema
+        let isCompatible = true;
+        if (parsed.type === 'line_chart' && (profile.dateFields.length === 0 || profile.numericFields.length === 0)) {
+          isCompatible = false;
+        } else if (
+          ['bar_chart', 'horizontal_bar', 'pie_chart', 'donut_chart'].includes(parsed.type) &&
+          (profile.categoricalFields.length === 0 && profile.numericFields.length === 0)
+        ) {
+          isCompatible = false;
+        } else if (parsed.type === 'scatter_plot' && profile.numericFields.length < 2) {
+          isCompatible = false;
+        } else if (parsed.type === 'metric_card' && profile.numericFields.length === 0) {
+          isCompatible = false;
+        }
+
+        if (!isCompatible) {
+          console.warn(`[UITransformer] LLM chose incompatible visualization type "${parsed.type}" for retrieved schema. Falling back to heuristic.`);
+          return this.transform(query, sources, undefined, undefined, intent);
+        }
+
         const intentAllowsTable =
           intent.wantsExplicitTable ||
           ['tabular', 'table', 'geographic'].includes(intent.visualizationHint);
@@ -295,7 +328,27 @@ export class UITransformer {
    * - The intent object can be reused across both the heuristic and LLM paths.
    * - It is easy to unit-test intent detection in isolation.
    */
-  static async detectIntent(query: string, llm: ILLMProvider): Promise<QueryIntent> {
+  static async detectIntent(query: string, llm: ILLMProvider, sources?: VectorMatch[]): Promise<QueryIntent> {
+    let schemaProfileText = '';
+    if (sources && sources.length > 0) {
+      const profile = this.profileData(sources);
+      const hasProducts = sources.some(item => this.isProductData(item));
+      schemaProfileText = `
+RETRIEVED DATA SCHEMA:
+- Numeric fields (measures): ${profile.numericFields.map(f => `${f.key} (unique values: ${f.uniqueCount})`).join(', ') || 'None'}
+- Category fields (dimensions): ${profile.categoricalFields.map(f => `${f.key} (unique values: ${f.uniqueCount})`).join(', ') || 'None'}
+- Date fields (temporal): ${profile.dateFields.map(f => `${f.key} (unique values: ${f.uniqueCount})`).join(', ') || 'None'}
+- Boolean fields: ${profile.booleanFields.map(f => `${f.key} (unique values: ${f.uniqueCount})`).join(', ') || 'None'}
+- Text fields: ${profile.fields.filter(f => f.kind === 'text').map(f => f.key).join(', ') || 'None'}
+
+Please choose visualizationHint and recommendedChart dynamically based on the available schema.
+- Do NOT recommend "trend" or "line_chart" if there are no Date fields.
+- Do NOT recommend numeric-dependent visualizations (like bar_chart, line_chart, pie_chart, scatter_plot, histogram, metric_card) if there are no Numeric fields. Fall back to tabular/table or text.
+- If there are multiple metadata fields but no clear categories or numbers, "tabular"/"table" is a highly accurate default.
+${hasProducts ? `- Note: The retrieved data matches product/catalog structure. If the user query is looking for, exploring, searching, browsing, or listing products/items, you MUST recommend visualizationHint: "product_browse" and recommendedChart: "text".` : ''}
+`;
+    }
+
     const systemPrompt = `You are an intent classifier for a product-search RAG system.
 Given a user query, return ONLY a valid JSON object with this exact shape — no prose, no markdown:
 
@@ -332,8 +385,10 @@ RULES:
 - isComparison: true if user compares, ranks, or contrasts entities or categories.
 - language: detect from the query text itself; default "en" if uncertain.`;
 
+    const userPrompt = `QUERY: ${query}${schemaProfileText ? `\n\n${schemaProfileText}` : ''}`;
+
     const rawResponse = await llm.chat(
-      [{ role: 'user', content: `QUERY: ${query}` }],
+      [{ role: 'user', content: userPrompt }],
       '',
       { systemPrompt, temperature: 0 },
     );
@@ -894,14 +949,28 @@ RULES:
     return asksForRecords && (hasNumericPredicate || hasEntityTerms);
   }
 
+  /** @internal kept private for transform() internal logic */
   private static isProductQuery(query: string): boolean {
+    return UITransformer._productQueryTest(query);
+  }
+
+  /**
+   * Public entry-point for external callers (e.g. Pipeline)
+   * to check whether a query maps to product_browse without triggering an LLM call.
+   */
+  static isProductQueryPublic(query: string): boolean {
+    return UITransformer._productQueryTest(query);
+  }
+
+  private static _productQueryTest(query: string): boolean {
     const q = query.toLowerCase();
     const productTerms = /\b(product|products|item|items|sku|catalog|catalogue|price|prices|brand|model|stock|inventory|in stock|out of stock|buy|shop)\b/.test(q);
-    const productAction = /\b(recommend|suggest|describe|description|detail|details|about|show|find|search|browse|list)\b/.test(q);
+    const productAction = /\b(recommend|suggest|describe|description|detail|details|about|show|find|search|browse|list|explore|exploring)\b/.test(q);
     const nonProductEntityTerms = /\b(organization|organisations?|organizations?|company|companies|employee|employees|staff|headcount|workforce)\b/.test(q);
     return productTerms && productAction && !nonProductEntityTerms;
   }
 
+
   private static isProductData(item: VectorMatch): boolean {
     const content = (item.content || '').toLowerCase();
     const productKeywords = [

package/dist/DocumentChunker-Dh9TvmGG.d.mts

@@ -1,45 +0,0 @@
-/**
- * DocumentChunker — splits long text into overlapping chunks
- * suitable for vector embedding and retrieval.
- *
- * Strategy: sentence-aware sliding window
- *   1. Split on sentence boundaries
- *   2. Accumulate sentences into chunks up to `chunkSize` characters
- *   3. Carry over `chunkOverlap` characters from the previous chunk
- */
-interface Chunk {
-    id: string;
-    content: string;
-    metadata?: Record<string, unknown>;
-}
-interface ChunkOptions {
-    /** Target chunk size in characters (default 1000) */
-    chunkSize?: number;
-    /** Overlap between consecutive chunks in characters (default 200) */
-    chunkOverlap?: number;
-    /** Source document identifier used as ID prefix */
-    docId?: string | number;
-    /** Extra metadata to attach to every chunk */
-    metadata?: Record<string, unknown>;
-    /** Characters used to split text, in order of priority */
-    separators?: string[];
-}
-declare class DocumentChunker {
-    private readonly chunkSize;
-    private readonly chunkOverlap;
-    private readonly separators;
-    constructor(chunkSize?: number, chunkOverlap?: number, separators?: string[]);
-    /**
-     * Split a single text string into overlapping chunks using a recursive strategy.
-     * Preserves structural boundaries (Markdown headers) where possible.
-     */
-    chunk(text: string, options?: ChunkOptions): Chunk[];
-    private recursiveSplit;
-    chunkMany(documents: Array<{
-        content: string;
-        docId?: string | number;
-        metadata?: Record<string, unknown>;
-    }>): Chunk[];
-}
-
-export { type Chunk as C, DocumentChunker as D, type ChunkOptions as a };

package/dist/DocumentChunker-Dh9TvmGG.d.ts

@@ -1,45 +0,0 @@
-/**
- * DocumentChunker — splits long text into overlapping chunks
- * suitable for vector embedding and retrieval.
- *
- * Strategy: sentence-aware sliding window
- *   1. Split on sentence boundaries
- *   2. Accumulate sentences into chunks up to `chunkSize` characters
- *   3. Carry over `chunkOverlap` characters from the previous chunk
- */
-interface Chunk {
-    id: string;
-    content: string;
-    metadata?: Record<string, unknown>;
-}
-interface ChunkOptions {
-    /** Target chunk size in characters (default 1000) */
-    chunkSize?: number;
-    /** Overlap between consecutive chunks in characters (default 200) */
-    chunkOverlap?: number;
-    /** Source document identifier used as ID prefix */
-    docId?: string | number;
-    /** Extra metadata to attach to every chunk */
-    metadata?: Record<string, unknown>;
-    /** Characters used to split text, in order of priority */
-    separators?: string[];
-}
-declare class DocumentChunker {
-    private readonly chunkSize;
-    private readonly chunkOverlap;
-    private readonly separators;
-    constructor(chunkSize?: number, chunkOverlap?: number, separators?: string[]);
-    /**
-     * Split a single text string into overlapping chunks using a recursive strategy.
-     * Preserves structural boundaries (Markdown headers) where possible.
-     */
-    chunk(text: string, options?: ChunkOptions): Chunk[];
-    private recursiveSplit;
-    chunkMany(documents: Array<{
-        content: string;
-        docId?: string | number;
-        metadata?: Record<string, unknown>;
-    }>): Chunk[];
-}
-
-export { type Chunk as C, DocumentChunker as D, type ChunkOptions as a };