@retrivora-ai/rag-engine 1.7.3 → 1.7.5

package/src/core/Pipeline.ts

@@ -101,72 +101,198 @@ export class Pipeline {
 
     // Augment system prompt with a Universal Visualization Protocol
     // We use a unique marker to ensure this is only injected once but is ALWAYS present.
-    const CHART_MARKER = '<!-- UI_PROTOCOL_V6 -->';
-    const chartInstruction = `\n\n${CHART_MARKER}
-### UNIVERSAL UI PROTOCOL
-When the user asks for a visual representation, comparison, distribution, breakdown, table, list, catalog, products, carousel, stock view, or category summary, you MUST include exactly one \`\`\`ui\`\`\` block.
-
-1. VIEW SELECTION
-   - Use "carousel" for browsing specific products, items, or a catalog. This is the DEFAULT for "show me", "what are", or "find me" requests for a specific category (e.g. "beauty").
-   - Use "chart" ONLY for numerical aggregations: distributions, percentages, counts, comparisons over time, or trends (e.g. "how many beauty products?").
-   - Use "table" for detailed technical specifications, row-heavy comparisons, or when the user explicitly asks for a list/table format.
-
-2. REQUIRED JSON SHAPE
-   {
-     "view": "chart" | "carousel" | "table",
-     "title": "Short heading",
-     "description": "One sentence describing the view",
-     "chartType": "pie" | "bar" | "line", // Required if view is "chart"
-     "xAxisKey": "label",
-     "dataKeys": ["value"],
-     "columns": ["dynamicFieldKey1", "dynamicFieldKey2"],
-     "insights": ["Short takeaway 1", "Short takeaway 2"],
-     "data": []
-   }
-
-3. DATA RULES
-   - Build the UI payload from retrieved context only. Do not invent products or values.
-   - For CAROUSEL: "data" must be a list of product objects (id, name, brand, price, image, link).
-   - For CHART: "data" must be aggregated counts or sums. Never put raw product lists in a chart view.
-   - If the user asks for "products in beauty", do NOT return a pie chart of categories; return a carousel of the beauty products themselves.
-
-4. FORMAT RULES
-   - Return valid JSON inside the \`\`\`ui\`\`\` block.
-   - Put natural language explanation before or after the block, not inside.
-
-5. EXAMPLES
+    const CHART_MARKER = '<!-- UI_PROTOCOL_V7 -->';
+    const chartInstruction = `
+
+${CHART_MARKER}
+### UNIVERSAL UI PROTOCOL V7 (STRICT MODE)
+
+You are responsible for returning a SINGLE structured UI block when visualization is required.
+
+---
+
+## 1. INTENT CLASSIFICATION (MANDATORY)
+
+Classify the query into ONE of the following intents:
+
+- "explore" → user wants to browse items (e.g., "show me products")
+- "analyze" → user wants aggregation/distribution (e.g., "distribution of products")
+- "compare" → user wants side-by-side comparison
+- "detail" → user wants structured/tabular data
+
+---
+
+## 2. VIEW MAPPING (STRICT)
+
+| Intent   | View       |
+|----------|-----------|
+| explore  | carousel  |
+| analyze  | chart     |
+| compare  | table     |
+| detail   | table     |
+
+⚠️ Override Rules:
+- If query contains "distribution", "percentage", "ratio" → ALWAYS chart
+- If query contains "show products", "list items" → ALWAYS carousel
+- NEVER mix views
+
+---
+
+## 3. RESPONSE FORMAT (STRICT JSON ONLY INSIDE BLOCK)
+
+\`\`\`ui
+{
+  "view": "carousel" | "chart" | "table",
+  "title": "string",
+  "description": "string",
+  "metadata": {
+    "intent": "explore" | "analyze" | "compare" | "detail",
+    "confidence": 0.0-1.0
+  },
+
+  // CHART ONLY
+  "chart": {
+    "type": "pie" | "bar" | "line",
+    "xKey": "label",
+    "yKeys": ["value"],
+    "data": []
+  },
+
+  // TABLE ONLY
+  "table": {
+    "columns": [],
+    "rows": []
+  },
+
+  // CAROUSEL ONLY
+  "carousel": {
+    "items": []
+  },
+
+  "insights": []
+}
+\`\`\`
+
+---
+
+## 4. DATA CONTRACT (CRITICAL)
+
+### 🔹 CAROUSEL ITEM FORMAT
+{
+  "id": "string",
+  "name": "string",
+  "brand": "string",
+  "price": number,
+  "image": "url",
+  "link": "url",
+  "inStock": boolean
+}
+
+### 🔹 CHART DATA FORMAT
+{
+  "label": "string",
+  "value": number,
+  "inStock": boolean (optional)
+}
+
+### 🔹 TABLE FORMAT
+- columns MUST match row keys
+- rows MUST be flat objects
+
+---
+
+## 5. DATA RULES
+
+- NEVER hallucinate fields
+- ONLY use retrieved vector DB data
+- NO placeholders ("...", "N/A")
+- If image/link missing → REMOVE FIELD
+- Aggregate BEFORE rendering chart
+
+---
+
+## 6. SMART RULES (IMPORTANT)
+
+- If user asks BOTH:
+  "show products AND distribution"
+  → PRIORITIZE "explore" → carousel
+
+- If dataset size > 20:
+  → aggregate → chart or table
+
+---
+
+## 7. OUTPUT RULES
+
+- ALWAYS return EXACTLY ONE \`\`\`ui\`\`\` block
+- JSON must be VALID
+- No explanation inside block
+
+---
+
+## 8. EXAMPLES
+
 User: "show me beauty products"
-Assistant:
+
 \`\`\`ui
 {
   "view": "carousel",
   "title": "Beauty Products",
-  "description": "Browse our latest skincare and makeup collection.",
-  "data": [
-    { "id": "1", "name": "Mascara", "brand": "Essence", "price": "$9.99", "image": "...", "link": "..." },
-    { "id": "2", "name": "Lipstick", "brand": "Chic", "price": "$12.99", "image": "...", "link": "..." }
-  ]
+  "description": "Browse available beauty items",
+  "metadata": {
+    "intent": "explore",
+    "confidence": 0.95
+  },
+  "carousel": {
+    "items": [
+      {
+        "id": "1",
+        "name": "Mascara",
+        "brand": "Essence",
+        "price": 9.99,
+        "image": "https://example.com/mascara.jpg",
+        "link": "https://example.com/p1",
+        "inStock": true
+      }
+    ]
+  },
+  "insights": ["Most products are affordable"]
 }
 \`\`\`
 
-User: "breakdown of products by category"
-Assistant:
+---
+
+User: "distribution of products across categories"
+
 \`\`\`ui
 {
   "view": "chart",
-  "title": "Category Breakdown",
-  "chartType": "pie",
-  "xAxisKey": "label",
-  "dataKeys": ["value"],
-  "data": [
-    { "label": "Beauty", "value": 15 },
-    { "label": "Fragrances", "value": 8 }
-  ]
+  "title": "Product Distribution",
+  "description": "Category-wise product distribution",
+  "metadata": {
+    "intent": "analyze",
+    "confidence": 0.98
+  },
+  "chart": {
+    "type": "pie",
+    "xKey": "label",
+    "yKeys": ["value"],
+    "data": [
+      { "label": "Beauty", "value": 15 },
+      { "label": "Electronics", "value": 10 }
+    ]
+  },
+  "insights": ["Beauty dominates inventory"]
 }
-\`\`\``;
-
+\`\`\`
+`;
     if (!this.config.llm.systemPrompt?.includes(CHART_MARKER)) {
-      this.config.llm.systemPrompt = (this.config.llm.systemPrompt || '') + chartInstruction;
+      // Clean up ANY existing protocol markers (V1-V7) to avoid prompt bloat
+      let cleanPrompt = this.config.llm.systemPrompt || '';
+      cleanPrompt = cleanPrompt.replace(/\n\n### UNIVERSAL UI PROTOCOL[\s\S]*?(?=\n\n|$)/g, '');
+      cleanPrompt = cleanPrompt.replace(/<!-- UI_PROTOCOL_V\d+ -->[\s\S]*?(?=\n\n|$)/g, '');
+      
+      this.config.llm.systemPrompt = cleanPrompt.trim() + chartInstruction;
     }
 
     console.log(`[Pipeline] Initializing with provider: ${this.config.vectorDb.provider}...`);

package/src/llm/providers/GeminiProvider.ts

@@ -128,7 +128,10 @@ export class GeminiProvider implements ILLMProvider {
       },
     });
 
-    return response.text ?? '';
+    const text = typeof (response as { text: unknown }).text === 'function'
+      ? (response as { text: () => string }).text()
+      : (response.text ?? '');
+    return text;
   }
 
   async *chatStream(messages: ChatMessage[], context: string, options?: ChatOptions): AsyncIterable<string> {
@@ -162,7 +165,9 @@ export class GeminiProvider implements ILLMProvider {
     }
 
     for await (const chunk of stream) {
-      const text = chunk.text;
+      const text = typeof (chunk as { text: unknown }).text === 'function'
+        ? (chunk as { text: () => string }).text()
+        : (chunk as { text?: string }).text;
       if (text) yield text;
     }
   }

package/src/providers/vectordb/MultiTablePostgresProvider.ts

@@ -151,12 +151,22 @@ export class MultiTablePostgresProvider extends BaseVectorProvider {
     console.log(`[MultiTablePostgresProvider] queryText: "${queryText}"`);
     console.log(`[MultiTablePostgresProvider] entityHints: [${entityHints.join(', ')}]`);
 
+    // Helper to clean conversational filler for better keyword matching
+    const cleanKeywordQuery = (text: string) => {
+      return text
+        .toLowerCase()
+        .replace(/\b(show|me|the|product|cards|of|category|find|list|browse|what|are|display)\b/g, '')
+        .trim()
+        .replace(/\s+/g, ' & '); // Join remaining words with AND for tsquery
+    };
+
     const queryPromises = this.tables.map(async (table) => {
       try {
         let sqlQuery = '';
         let params: unknown[] = [];
 
         if (queryText) {
+          const cleanedText = cleanKeywordQuery(queryText) || queryText;
           const hasEntityHints = entityHints.length > 0;
           const exactNameScoreExpr = hasEntityHints
             ? `+ (
@@ -173,13 +183,13 @@ export class MultiTablePostgresProvider extends BaseVectorProvider {
           sqlQuery = `
             SELECT *, 
              (1 - (embedding <=> $1::vector)) AS vector_score,
-             COALESCE(ts_rank(to_tsvector('english', (to_jsonb(t) - 'embedding')::text), NULLIF(REPLACE(plainto_tsquery('english', $2)::text, '&', '|'), '')::tsquery), 0) AS keyword_score,
-             ((1 - (embedding <=> $1::vector)) + (COALESCE(ts_rank(to_tsvector('english', (to_jsonb(t) - 'embedding')::text), NULLIF(REPLACE(plainto_tsquery('english', $2)::text, '&', '|'), '')::tsquery), 0) * 2.0) ${exactNameScoreExpr}) AS hybrid_score
+             COALESCE(ts_rank(to_tsvector('english', (to_jsonb(t) - 'embedding')::text), to_tsquery('english', $2)), 0) AS keyword_score,
+             ((1 - (embedding <=> $1::vector)) + (COALESCE(ts_rank(to_tsvector('english', (to_jsonb(t) - 'embedding')::text), to_tsquery('english', $2)), 0) * 2.0) ${exactNameScoreExpr}) AS hybrid_score
             FROM "${table}" t
             ORDER BY hybrid_score DESC
             LIMIT 50
           `;
-          params = [vectorLiteral, queryText];
+          params = [vectorLiteral, cleanedText];
         } else {
           // Fallback to Vector-only Search
           sqlQuery = `