@retrivora-ai/rag-engine 1.8.0 → 1.8.2

package/src/providers/vectordb/MultiTablePostgresProvider.ts

@@ -21,6 +21,7 @@ export class MultiTablePostgresProvider extends BaseVectorProvider {
   private readonly connectionString: string;
   private tables: string[];
   private searchFields: string[];
+  private readonly uploadTable: string;
 
   constructor(config: VectorDBConfig) {
     super(config);
@@ -31,22 +32,8 @@ export class MultiTablePostgresProvider extends BaseVectorProvider {
     this.connectionString = opts.connectionString as string;
     this.dimensions = (opts.dimensions as number) ?? 768;
 
-    // Tables can come from options.tables (custom key) or be derived from VECTOR_DB_TABLES env var
-    const rawTables: string | string[] =
-      (opts.tables as string | string[]) ??
-      process.env.VECTOR_DB_TABLES ??
-      '';
-
-    this.tables = typeof rawTables === 'string'
-      ? rawTables.split(',').map((t) => t.trim()).filter(Boolean)
-      : rawTables;
-
-    if (this.tables.length === 0) {
-      console.warn(
-        '[MultiTablePostgresProvider] No tables configured. ' +
-        'Set VECTOR_DB_TABLES as a comma-separated list of table names or pass options.tables.'
-      );
-    }
+    // The provider will dynamically discover tables during initialize().
+    this.tables = [];
 
     // Exact match fields can come from options.searchFields or be derived from VECTOR_DB_SEARCH_FIELDS env var
     const rawSearchFields: string | string[] =
@@ -57,6 +44,8 @@ export class MultiTablePostgresProvider extends BaseVectorProvider {
     this.searchFields = typeof rawSearchFields === 'string'
       ? rawSearchFields.split(',').map((f) => f.trim()).filter(Boolean)
       : rawSearchFields;
+
+    this.uploadTable = (opts.uploadTable as string) || 'document_chunks';
   }
 
   async initialize(): Promise<void> {
@@ -65,6 +54,24 @@ export class MultiTablePostgresProvider extends BaseVectorProvider {
     // Dynamically discover all tables that have an 'embedding' column
     const client: PoolClient = await this.pool.connect();
     try {
+      await client.query('CREATE EXTENSION IF NOT EXISTS vector');
+      
+      // Ensure the upload table exists
+      await client.query(`
+        CREATE TABLE IF NOT EXISTS ${this.uploadTable} (
+          id          TEXT PRIMARY KEY,
+          namespace   TEXT NOT NULL DEFAULT '',
+          content     TEXT NOT NULL,
+          metadata    JSONB,
+          embedding   VECTOR(${this.dimensions})
+        )
+      `);
+      await client.query(`
+        CREATE INDEX IF NOT EXISTS ${this.uploadTable}_embedding_idx
+        ON ${this.uploadTable}
+        USING hnsw (embedding vector_cosine_ops)
+      `);
+
       const res = await client.query(`
         SELECT table_name 
         FROM information_schema.columns 
@@ -72,21 +79,8 @@ export class MultiTablePostgresProvider extends BaseVectorProvider {
         AND table_schema = 'public'
       `);
 
-      const discoveredTables = res.rows.map(r => r.table_name);
-
-      if (this.tables.length === 0) {
-        // No static tables configured, use everything discovered
-        this.tables = discoveredTables;
-      } else {
-        // Filter configured tables to ensure they actually exist with embeddings
-        const staticTables = [...this.tables];
-        this.tables = staticTables.filter(t => discoveredTables.includes(t));
-
-        if (this.tables.length < staticTables.length) {
-          const missing = staticTables.filter(t => !discoveredTables.includes(t));
-          console.warn(`[MultiTablePostgresProvider] Some configured tables were skipped (missing 'embedding' column): ${missing.join(', ')}`);
-        }
-      }
+      // Dynamically use all tables that have an embedding column
+      this.tables = res.rows.map(r => r.table_name);
 
       if (this.tables.length === 0) {
         console.warn('[MultiTablePostgresProvider] No tables with "embedding" columns found in the database.');
@@ -101,23 +95,125 @@ export class MultiTablePostgresProvider extends BaseVectorProvider {
   }
 
   /**
-   * Upsert is not supported for MultiTablePostgresProvider as it's designed for 
-   * searching across pre-existing tables with varying schemas.
+   * Upsert a document by dynamically provisioning a table and columns.
    */
-  async upsert(_doc: UpsertDocument, _namespace?: string): Promise<void> { // eslint-disable-line @typescript-eslint/no-unused-vars
-    throw new Error(
-      '[MultiTablePostgresProvider] upsert() is not supported in multi-table mode. ' +
-      'Please use the standard PostgreSQLProvider for single-table managed indices.'
-    );
+  async upsert(doc: UpsertDocument, namespace = ''): Promise<void> {
+    await this.batchUpsert([doc], namespace);
   }
 
   /**
-   * Batch upsert is not supported for MultiTablePostgresProvider.
+   * Batch upsert documents by dynamically provisioning tables and columns based on CSV headers.
    */
-  async batchUpsert(_docs: UpsertDocument[], _namespace?: string): Promise<void> { // eslint-disable-line @typescript-eslint/no-unused-vars
-    throw new Error(
-      '[MultiTablePostgresProvider] batchUpsert() is not supported in multi-table mode.'
-    );
+  async batchUpsert(docs: UpsertDocument[], namespace = ''): Promise<void> {
+    if (docs.length === 0) return;
+
+    // Group docs by fileName to support dynamic table creation per CSV
+    const docsByFile: Record<string, UpsertDocument[]> = {};
+    for (const doc of docs) {
+      const fileName = (doc.metadata?.fileName as string) || this.uploadTable;
+      if (!docsByFile[fileName]) docsByFile[fileName] = [];
+      docsByFile[fileName].push(doc);
+    }
+
+    const client = await this.pool.connect();
+    try {
+      await client.query('BEGIN');
+      
+      for (const [fileName, fileDocs] of Object.entries(docsByFile)) {
+        // Sanitize table name (e.g., "jewelery.csv" -> "jewelery")
+        let tableName = fileName;
+        if (tableName.toLowerCase().endsWith('.csv')) {
+           tableName = tableName.slice(0, -4);
+        }
+        tableName = tableName.replace(/[^a-z0-9_]/gi, '_').toLowerCase() || this.uploadTable;
+        
+        // Extract headers from the first doc's metadata
+        const firstMeta = fileDocs[0].metadata || {};
+        const systemKeys = ['fileName', 'fileSize', 'fileType', 'uploadedAt', 'dimension', 'chunkIndex', 'id', 'namespace', 'content', 'metadata', 'embedding'];
+        const csvHeaders = Object.keys(firstMeta).filter(k => !systemKeys.includes(k) && !systemKeys.includes(k.toLowerCase()));
+        
+        // 1. Create Table Dynamically
+        const columnDefs = csvHeaders.map(h => `"${h}" TEXT`).join(',\n          ');
+        const createTableSql = `
+          CREATE TABLE IF NOT EXISTS "${tableName}" (
+            id          TEXT PRIMARY KEY,
+            ${columnDefs ? columnDefs + ',' : ''}
+            namespace   TEXT NOT NULL DEFAULT '',
+            content     TEXT NOT NULL,
+            metadata    JSONB,
+            embedding   VECTOR(${this.dimensions})
+          )
+        `;
+        await client.query(createTableSql);
+        
+        // Create Index
+        await client.query(`
+          CREATE INDEX IF NOT EXISTS "${tableName}_embedding_idx"
+          ON "${tableName}"
+          USING hnsw (embedding vector_cosine_ops)
+        `);
+        
+        // Ensure this new table is added to our search scope
+        if (!this.tables.includes(tableName)) {
+          this.tables.push(tableName);
+        }
+        
+        // 2. Insert Data
+        const BATCH_SIZE = 50;
+        for (let i = 0; i < fileDocs.length; i += BATCH_SIZE) {
+          const batch = fileDocs.slice(i, i + BATCH_SIZE);
+          const values: unknown[] = [];
+          
+          const headerColumns = csvHeaders.map(h => `"${h}"`).join(', ');
+          const allColumns = `id, ${headerColumns ? headerColumns + ', ' : ''}namespace, content, metadata, embedding`;
+          
+          const valuePlaceholders = batch.map((doc, idx) => {
+            const offset = idx * (5 + csvHeaders.length);
+            
+            values.push(doc.id);
+            // Push header values
+            for (const h of csvHeaders) {
+              values.push(String(doc.metadata?.[h] || ''));
+            }
+            
+            values.push(namespace, doc.content, JSON.stringify(doc.metadata ?? {}), `[${doc.vector.join(',')}]`);
+            
+            const docPlaceholders = [];
+            for (let j = 1; j <= 5 + csvHeaders.length; j++) {
+              if (j === 5 + csvHeaders.length) {
+                docPlaceholders.push(`$${offset + j}::vector`);
+              } else {
+                docPlaceholders.push(`$${offset + j}`);
+              }
+            }
+            
+            return `(${docPlaceholders.join(', ')})`;
+          }).join(', ');
+          
+          const conflictUpdates = csvHeaders.map(h => `"${h}" = EXCLUDED."${h}"`).join(',\n              ');
+          
+          const query = `
+            INSERT INTO "${tableName}" (${allColumns})
+            VALUES ${valuePlaceholders}
+            ON CONFLICT (id) DO UPDATE
+              SET ${conflictUpdates ? conflictUpdates + ',' : ''}
+                  namespace = EXCLUDED.namespace,
+                  content   = EXCLUDED.content,
+                  metadata  = EXCLUDED.metadata,
+                  embedding = EXCLUDED.embedding
+          `;
+          
+          await client.query(query, values);
+        }
+      }
+      
+      await client.query('COMMIT');
+    } catch (error) {
+      await client.query('ROLLBACK');
+      throw error;
+    } finally {
+      client.release();
+    }
   }
 
   /**
@@ -258,30 +354,20 @@ export class MultiTablePostgresProvider extends BaseVectorProvider {
       allResults.push(...tableResults);
     }
 
-    // 1. Group results by table to ensure we take the top ones from each
-    const resultsByTable: Record<string, VectorMatch[]> = {};
-    for (const res of allResults) {
-      const table = res.metadata?.source_table as string;
-      if (!resultsByTable[table]) resultsByTable[table] = [];
-      resultsByTable[table].push(res);
-    }
-
-    // 2. Take top 3 from EACH table guaranteed, then fill the rest with overall best
-    const balancedResults: VectorMatch[] = [];
-    const tables = Object.keys(resultsByTable);
+    // 1. Sort all results globally by semantic similarity score
+    allResults.sort((a, b) => b.score - a.score);
 
-    // First, ensure every table's best match is included
-    for (const table of tables) {
-      balancedResults.push(...resultsByTable[table].slice(0, 3));
-    }
+    if (allResults.length === 0) return [];
 
-    // 3. Sort the balanced list by score
-    const finalSorted = balancedResults.sort((a, b) => b.score - a.score);
+    // 2. Identify the "Active Table Context" from the absolute highest scoring match
+    const bestMatchTable = allResults[0].metadata?.source_table as string;
+    
+    // 3. Filter out all results that don't belong to the active table to prevent cross-contamination
+    const finalSorted = allResults.filter(res => res.metadata?.source_table === bestMatchTable);
 
-    if (finalSorted.length > 0) {
-      console.log(`[MultiTablePostgresProvider] --- Search Complete ---`);
-      console.log(`[MultiTablePostgresProvider] Final top match from "${finalSorted[0].metadata?.source_table ?? 'unknown'}" with score ${finalSorted[0].score.toFixed(4)}`);
-    }
+    console.log(`[MultiTablePostgresProvider] --- Search Complete ---`);
+    console.log(`[MultiTablePostgresProvider] Final top match from "${bestMatchTable}" with score ${finalSorted[0].score.toFixed(4)}`);
+    console.log(`[MultiTablePostgresProvider] Restricting recommendations strictly to table: "${bestMatchTable}"`);
 
     return finalSorted.slice(0, Math.max(topK, 15));
   }

package/src/types/chat.ts

@@ -11,6 +11,8 @@ export interface RagMessage extends ChatMessage {
   id: string;
   sources?: VectorMatch[];
   uiTransformation?: unknown;
+  /** Full observability trace emitted by the backend. Only present on assistant messages. */
+  trace?: import('./index').ObservabilityTrace;
   createdAt: string;
 }
 
@@ -21,6 +23,12 @@ export interface ChatOptions {
   temperature?: number;
   /** Stop sequences */
   stop?: string[];
+  /**
+   * Per-call system prompt override. When provided, providers should use this
+   * instead of (or in addition to) their configured llmConfig.systemPrompt.
+   * Useful for one-off analytical calls (e.g. UITransformer.analyzeAndDecide).
+   */
+  systemPrompt?: string;
 }
 
 export interface UseRagChatOptions {

package/src/types/index.ts

@@ -1,3 +1,53 @@
+// ─── Observability ────────────────────────────────────────────────────────────
+
+/** Per-stage latency breakdown for a single RAG request. */
+export interface LatencyBreakdown {
+  embedMs: number;
+  retrieveMs: number;
+  rerankMs?: number;
+  generateMs: number;
+  totalMs: number;
+}
+
+/** Token usage reported by the LLM (or estimated when the provider doesn't expose it). */
+export interface TokenUsage {
+  promptTokens: number;
+  completionTokens: number;
+  totalTokens: number;
+  /** Rough cost in USD — estimated from token counts and model pricing. */
+  estimatedCostUsd?: number;
+}
+
+/** A single retrieved chunk annotated with its retrieval context. */
+export interface RetrievedChunk {
+  id: string | number;
+  score: number;
+  content: string;
+  metadata: Record<string, unknown>;
+  namespace: string;
+}
+
+/**
+ * Full observability trace for one RAG request.
+ * Emitted by the backend as an SSE frame and stored on each RagMessage.
+ */
+export interface ObservabilityTrace {
+  requestId: string;
+  query: string;
+  rewrittenQuery?: string;
+  systemPrompt: string;
+  userPrompt: string;
+  chunks: RetrievedChunk[];
+  latency: LatencyBreakdown;
+  tokens?: TokenUsage;
+  /** 0 = fully grounded, 1 = likely hallucinated. */
+  hallucinationScore?: number;
+  hallucinationReason?: string;
+  timestamp: string;
+}
+
+// ─── Vector / RAG ─────────────────────────────────────────────────────────────
+
 export interface VectorMatch {
   id: string | number;
   score: number;
@@ -23,6 +73,8 @@ export interface ChatResponse {
   sources: VectorMatch[];
   graphData?: GraphSearchResult;
   ui_transformation?: unknown;
+  /** Observability trace — populated by the instrumented Pipeline. */
+  trace?: ObservabilityTrace;
 }
 
 export interface GraphNode {
@@ -66,5 +118,85 @@ export interface Product {
   description?: string;
 }
 
+/**
+ * Supported UI visualization types
+ */
+export type VisualizationType =
+  | 'pie_chart'
+  | 'bar_chart'
+  | 'line_chart'
+  | 'radar_chart'
+  | 'table'
+  | 'product_carousel'
+  | 'carousel'
+  | 'text';
+
+/**
+ * Base structure for all UI transformation responses
+ */
+export interface UITransformationResponse {
+  type: VisualizationType;
+  title: string;
+  description?: string;
+  data: unknown;
+}
+
+/**
+ * Pie chart data structure
+ */
+export interface PieChartData {
+  label: string;
+  value: number;
+  inStockCount?: number;
+  outOfStockCount?: number;
+  [key: string]: unknown;
+}
+
+/**
+ * Bar chart data structure
+ */
+export interface BarChartData {
+  category: string;
+  value: number;
+  inStockCount?: number;
+  outOfStockCount?: number;
+  [key: string]: unknown;
+}
+
+/**
+ * Line chart data point
+ */
+export interface LineChartDataPoint {
+  timestamp: string | number;
+  value: number;
+  label?: string;
+  [key: string]: unknown;
+}
+
+/**
+ * Table representation
+ */
+export interface TableData {
+  columns: string[];
+  rows: (string | number | boolean)[][];
+}
+
+
+/**
+ * Product carousel item
+ */
+export interface CarouselProduct {
+  id: string | number;
+  name: string;
+  price?: number | string;
+  image?: string;
+  inStock?: boolean;
+  brand?: string;
+  description?: string;
+  [key: string]: unknown;
+}
+
+
+
 export * from './props';
 export * from './chat';

package/src/types/props.ts

@@ -74,9 +74,17 @@ export interface SourceCardProps {
 export interface ClientConfig {
   projectId: string;
   ui: Required<UIConfig>;
+  embedding?: {
+    model?: string;
+    dimensions?: number;
+  };
 }
 
 export interface ConfigProviderProps {
-  config?: { projectId?: string; ui?: Partial<UIConfig> };
+  config?: { 
+    projectId?: string; 
+    ui?: Partial<UIConfig>;
+    embedding?: { model?: string; dimensions?: number }
+  };
   children: ReactNode;
 }