@retrivora-ai/rag-engine 1.8.0 → 1.8.2

package/dist/server.mjs

@@ -39,14 +39,15 @@ import {
   sseFrame,
   sseMetaFrame,
   sseTextFrame
-} from "./chunk-PV3MFHWU.mjs";
+} from "./chunk-LZVVLSDN.mjs";
+import "./chunk-ICKRMZQK.mjs";
 import "./chunk-YLTMFW4M.mjs";
 import {
   PineconeProvider
 } from "./chunk-VUQJVIJT.mjs";
 import {
-  PostgreSQLProvider
-} from "./chunk-FLOSGE6A.mjs";
+  MultiTablePostgresProvider
+} from "./chunk-OZFBG4BA.mjs";
 import {
   MongoDBProvider
 } from "./chunk-5AJ4XHLW.mjs";
@@ -60,8 +61,6 @@ import {
   BaseVectorProvider
 } from "./chunk-IMP6FUCY.mjs";
 import {
-  __objRest,
-  __spreadProps,
   __spreadValues
 } from "./chunk-X4TOT24V.mjs";
 
@@ -237,196 +236,187 @@ function createFromPreset(presetName) {
   return new ConfigBuilder().vectorDb(preset.vectorDb, { apiKey: process.env[`${preset.vectorDb.toUpperCase()}_API_KEY`] }).llm(preset.llm, void 0, process.env[`${preset.llm.toUpperCase()}_API_KEY`]).embedding(preset.embedding, void 0, process.env[`${preset.embedding.toUpperCase()}_API_KEY`]);
 }
 
-// src/providers/vectordb/MultiTablePostgresProvider.ts
+// src/providers/vectordb/PostgreSQLProvider.ts
 import { Pool } from "pg";
-var MultiTablePostgresProvider = class extends BaseVectorProvider {
+var PostgreSQLProvider = class extends BaseVectorProvider {
   constructor(config) {
-    var _a, _b, _c, _d, _e;
+    var _a;
     super(config);
-    const opts = config.options || {};
-    if (!opts.connectionString) {
-      throw new Error("[MultiTablePostgresProvider] options.connectionString is required");
-    }
+    this.tableName = this.indexName.replace(/[^a-z0-9_]/gi, "_");
+    const opts = config.options;
+    if (!opts.connectionString) throw new Error("[PostgreSQLProvider] options.connectionString is required");
     this.connectionString = opts.connectionString;
-    this.dimensions = (_a = opts.dimensions) != null ? _a : 768;
-    const rawTables = (_c = (_b = opts.tables) != null ? _b : process.env.VECTOR_DB_TABLES) != null ? _c : "";
-    this.tables = typeof rawTables === "string" ? rawTables.split(",").map((t) => t.trim()).filter(Boolean) : rawTables;
-    if (this.tables.length === 0) {
-      console.warn(
-        "[MultiTablePostgresProvider] No tables configured. Set VECTOR_DB_TABLES as a comma-separated list of table names or pass options.tables."
-      );
-    }
-    const rawSearchFields = (_e = (_d = opts.searchFields) != null ? _d : process.env.VECTOR_DB_SEARCH_FIELDS) != null ? _e : ["name", "product_name", "productname", "title"];
-    this.searchFields = typeof rawSearchFields === "string" ? rawSearchFields.split(",").map((f) => f.trim()).filter(Boolean) : rawSearchFields;
+    this.dimensions = (_a = opts.dimensions) != null ? _a : 1536;
+  }
+  static getValidator() {
+    return {
+      validate(config) {
+        const errors = [];
+        const opts = config.options || {};
+        if (!opts.connectionString) {
+          errors.push({
+            field: "vectorDb.options.connectionString",
+            message: "PostgreSQL connection string is required",
+            suggestion: "Set PGVECTOR_CONNECTION_STRING environment variable",
+            severity: "error"
+          });
+        }
+        if (opts.tables && typeof opts.tables !== "string" && !Array.isArray(opts.tables)) {
+          errors.push({
+            field: "vectorDb.options.tables",
+            message: "PostgreSQL tables must be a string or a string array",
+            severity: "error"
+          });
+        }
+        return errors;
+      }
+    };
+  }
+  static getHealthChecker() {
+    return {
+      async check(config) {
+        const opts = config.options || {};
+        const timestamp = Date.now();
+        try {
+          const { Client } = await import("pg");
+          const client = new Client({ connectionString: opts.connectionString });
+          await client.connect();
+          const result = await client.query(`
+            SELECT EXISTS(SELECT 1 FROM pg_extension WHERE extname = 'vector');
+          `);
+          const hasVector = result.rows[0].exists;
+          await client.end();
+          return {
+            healthy: true,
+            provider: "postgresql",
+            capabilities: { pgvectorInstalled: hasVector },
+            timestamp
+          };
+        } catch (error) {
+          return {
+            healthy: false,
+            provider: "postgresql",
+            error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
+            timestamp
+          };
+        }
+      }
+    };
   }
   async initialize() {
     this.pool = new Pool({ connectionString: this.connectionString });
     const client = await this.pool.connect();
     try {
-      const res = await client.query(`
-        SELECT table_name 
-        FROM information_schema.columns 
-        WHERE column_name = 'embedding' 
-        AND table_schema = 'public'
+      await client.query("CREATE EXTENSION IF NOT EXISTS vector");
+      await client.query(`
+        CREATE TABLE IF NOT EXISTS ${this.tableName} (
+          id          TEXT PRIMARY KEY,
+          namespace   TEXT NOT NULL DEFAULT '',
+          content     TEXT NOT NULL,
+          metadata    JSONB,
+          embedding   VECTOR(${this.dimensions})
+        )
+      `);
+      await client.query(`
+        CREATE INDEX IF NOT EXISTS ${this.tableName}_embedding_idx
+        ON ${this.tableName}
+        USING hnsw (embedding vector_cosine_ops)
       `);
-      const discoveredTables = res.rows.map((r) => r.table_name);
-      if (this.tables.length === 0) {
-        this.tables = discoveredTables;
-      } else {
-        const staticTables = [...this.tables];
-        this.tables = staticTables.filter((t) => discoveredTables.includes(t));
-        if (this.tables.length < staticTables.length) {
-          const missing = staticTables.filter((t) => !discoveredTables.includes(t));
-          console.warn(`[MultiTablePostgresProvider] Some configured tables were skipped (missing 'embedding' column): ${missing.join(", ")}`);
-        }
-      }
-      if (this.tables.length === 0) {
-        console.warn('[MultiTablePostgresProvider] No tables with "embedding" columns found in the database.');
-      } else {
-        console.log(
-          `[MultiTablePostgresProvider] Connected. Searching across ${this.tables.length} table(s): ${this.tables.join(", ")}`
-        );
-      }
     } finally {
       client.release();
     }
   }
-  /**
-   * Upsert is not supported for MultiTablePostgresProvider as it's designed for 
-   * searching across pre-existing tables with varying schemas.
-   */
-  async upsert(_doc, _namespace) {
-    throw new Error(
-      "[MultiTablePostgresProvider] upsert() is not supported in multi-table mode. Please use the standard PostgreSQLProvider for single-table managed indices."
-    );
-  }
-  /**
-   * Batch upsert is not supported for MultiTablePostgresProvider.
-   */
-  async batchUpsert(_docs, _namespace) {
-    throw new Error(
-      "[MultiTablePostgresProvider] batchUpsert() is not supported in multi-table mode."
+  async upsert(doc, namespace = "") {
+    var _a;
+    const vectorLiteral = `[${doc.vector.join(",")}]`;
+    await this.pool.query(
+      `INSERT INTO ${this.tableName} (id, namespace, content, metadata, embedding)
+       VALUES ($1, $2, $3, $4, $5::vector)
+       ON CONFLICT (id) DO UPDATE
+         SET namespace = EXCLUDED.namespace,
+             content   = EXCLUDED.content,
+             metadata  = EXCLUDED.metadata,
+             embedding = EXCLUDED.embedding`,
+      [doc.id, namespace, doc.content, JSON.stringify((_a = doc.metadata) != null ? _a : {}), vectorLiteral]
     );
   }
-  /**
-   * Query all configured tables and merge results, sorted by cosine similarity score.
-   */
-  async query(vector, topK, _namespace, _filter) {
-    var _a, _b, _c;
-    if (!this.pool) {
-      throw new Error("[MultiTablePostgresProvider] Provider not initialized. Call initialize() first.");
-    }
-    const vectorLiteral = `[${vector.join(",")}]`;
-    const allResults = [];
-    console.log(`[MultiTablePostgresProvider] --- Starting Multi-Table Search ---`);
-    const queryText = _filter == null ? void 0 : _filter.queryText;
-    const entityHints = Array.isArray(_filter == null ? void 0 : _filter.__entityHints) ? _filter.__entityHints.filter(
-      (hint) => typeof hint === "object" && hint !== null && typeof hint.value === "string"
-    ).map((hint) => hint.value.trim().toLowerCase()).filter(Boolean) : [];
-    console.log(`[MultiTablePostgresProvider] queryText: "${queryText}"`);
-    console.log(`[MultiTablePostgresProvider] entityHints: [${entityHints.join(", ")}]`);
-    const getDynamicKeywordQuery = () => {
-      if (entityHints.length > 0) {
-        return entityHints.map((h) => h.includes(" ") ? `"${h}"` : h).join(" & ");
-      }
-      if (queryText) {
-        return queryText.toLowerCase().replace(/\b(show|me|the|product[s]?|item[s]?|card[s]?|of|category|find|list|browse|what|are|display|all|under|for|in|with|about)\b/g, "").trim().replace(/\s+/g, " & ");
-      }
-      return "";
-    };
-    const dynamicKeywordQuery = getDynamicKeywordQuery();
-    const queryPromises = this.tables.map(async (table) => {
-      try {
-        let sqlQuery = "";
-        let params = [];
-        if (queryText) {
-          const hasEntityHints = entityHints.length > 0;
-          const exactNameScoreExpr = hasEntityHints ? `+ (
-                 SELECT COALESCE(MAX(
-                   CASE WHEN LOWER(val) IN (${entityHints.map((h) => `'${h.replace(/'/g, "''")}'`).join(", ")})
-                   THEN 1.0 ELSE 0.0 END
-                 ), 0)
-                 FROM jsonb_each_text(to_jsonb(t)) AS kv(key, val)
-                 WHERE key IN (${this.searchFields.map((f) => `'${f}'`).join(", ")})
-               ) * 3.0` : "";
-          sqlQuery = `
-            SELECT *, 
-             (1 - (embedding <=> $1::vector)) AS vector_score,
-             COALESCE(ts_rank(to_tsvector('english', (to_jsonb(t) - 'embedding')::text), to_tsquery('english', $2)), 0) AS keyword_score,
-             ((1 - (embedding <=> $1::vector)) + (COALESCE(ts_rank(to_tsvector('english', (to_jsonb(t) - 'embedding')::text), to_tsquery('english', $2)), 0) * 2.0) ${exactNameScoreExpr}) AS hybrid_score
-            FROM "${table}" t
-            ORDER BY hybrid_score DESC
-            LIMIT 50
-          `;
-          params = [vectorLiteral, dynamicKeywordQuery];
-        } else {
-          sqlQuery = `
-            SELECT *, 
-             (1 - (embedding <=> $1::vector)) AS hybrid_score
-            FROM "${table}" t
-            ORDER BY hybrid_score DESC
-            LIMIT 50
-          `;
-          params = [vectorLiteral];
-        }
-        const result = await this.pool.query(sqlQuery, params);
-        if (result.rowCount && result.rowCount > 0) {
-          console.log(`   \u2705 Table "${table}": Found ${result.rowCount} potential matches. Top score: ${result.rows[0].hybrid_score.toFixed(4)}`);
-        } else {
-          console.log(`   \u26AA Table "${table}": No relevant matches found.`);
-        }
-        const tableResults = [];
-        for (const row of result.rows) {
-          const _a2 = row, { hybrid_score, id } = _a2, rest = __objRest(_a2, ["hybrid_score", "id"]);
-          delete rest.embedding;
-          delete rest.vector_score;
-          delete rest.keyword_score;
-          delete rest.exact_name_score;
-          const content = `[TYPE: ${table.replace(/s$/, "").toUpperCase()}]
-` + Object.entries(rest).filter(([k, v]) => v !== null && typeof v !== "object" && k !== "id").map(([k, v]) => `${k}: ${v}`).join("\n");
-          tableResults.push({
-            id: `${table}-${id}`,
-            score: parseFloat(String(hybrid_score)),
-            content,
-            metadata: __spreadProps(__spreadValues({}, rest), { source_table: table })
-          });
-        }
-        return tableResults;
-      } catch (err) {
-        console.error(
-          `[MultiTablePostgresProvider] Error querying table "${table}":`,
-          err.message
-        );
-        return [];
+  async batchUpsert(docs, namespace = "") {
+    if (docs.length === 0) return;
+    const client = await this.pool.connect();
+    try {
+      await client.query("BEGIN");
+      const BATCH_SIZE = 50;
+      for (let i = 0; i < docs.length; i += BATCH_SIZE) {
+        const batch = docs.slice(i, i + BATCH_SIZE);
+        const values = [];
+        const valuePlaceholders = batch.map((doc, idx) => {
+          var _a;
+          const offset = idx * 5;
+          values.push(doc.id, namespace, doc.content, JSON.stringify((_a = doc.metadata) != null ? _a : {}), `[${doc.vector.join(",")}]`);
+          return `($${offset + 1}, $${offset + 2}, $${offset + 3}, $${offset + 4}, $${offset + 5}::vector)`;
+        }).join(", ");
+        const query = `
+          INSERT INTO ${this.tableName} (id, namespace, content, metadata, embedding)
+          VALUES ${valuePlaceholders}
+          ON CONFLICT (id) DO UPDATE
+            SET namespace = EXCLUDED.namespace,
+                content   = EXCLUDED.content,
+                metadata  = EXCLUDED.metadata,
+                embedding = EXCLUDED.embedding
+        `;
+        await client.query(query, values);
       }
-    });
-    const resultsArray = await Promise.all(queryPromises);
-    for (const tableResults of resultsArray) {
-      allResults.push(...tableResults);
-    }
-    const resultsByTable = {};
-    for (const res of allResults) {
-      const table = (_a = res.metadata) == null ? void 0 : _a.source_table;
-      if (!resultsByTable[table]) resultsByTable[table] = [];
-      resultsByTable[table].push(res);
+      await client.query("COMMIT");
+    } catch (error) {
+      await client.query("ROLLBACK");
+      throw error;
+    } finally {
+      client.release();
     }
-    const balancedResults = [];
-    const tables = Object.keys(resultsByTable);
-    for (const table of tables) {
-      balancedResults.push(...resultsByTable[table].slice(0, 3));
+  }
+  async query(vector, topK, namespace, filter) {
+    const vectorLiteral = `[${vector.join(",")}]`;
+    let whereClause = namespace ? `WHERE namespace = $3` : "";
+    const params = [vectorLiteral, topK];
+    if (namespace) params.push(namespace);
+    const publicFilter = this.sanitizeFilter(filter);
+    if (Object.keys(publicFilter).length > 0) {
+      const filterConditions = Object.entries(publicFilter).map(([key, val]) => {
+        const paramIdx = params.length + 1;
+        params.push(JSON.stringify(val));
+        return `metadata->>'${key}' = $${paramIdx}`;
+      }).join(" AND ");
+      whereClause = whereClause ? `${whereClause} AND ${filterConditions}` : `WHERE ${filterConditions}`;
     }
-    const finalSorted = balancedResults.sort((a, b) => b.score - a.score);
-    if (finalSorted.length > 0) {
-      console.log(`[MultiTablePostgresProvider] --- Search Complete ---`);
-      console.log(`[MultiTablePostgresProvider] Final top match from "${(_c = (_b = finalSorted[0].metadata) == null ? void 0 : _b.source_table) != null ? _c : "unknown"}" with score ${finalSorted[0].score.toFixed(4)}`);
+    const client = await this.pool.connect();
+    try {
+      const efSearch = this.config.options.efSearch || Math.max(topK * 10, 40);
+      await client.query(`SET LOCAL hnsw.ef_search = ${efSearch}`);
+      const result = await client.query(
+        `SELECT id, content, metadata, 1 - (embedding <=> $1::vector) AS score
+         FROM ${this.tableName}
+         ${whereClause}
+         ORDER BY embedding <=> $1::vector
+         LIMIT $2`,
+        params
+      );
+      return result.rows.map((row) => ({
+        id: String(row["id"]),
+        score: parseFloat(String(row["score"])),
+        content: String(row["content"]),
+        metadata: row["metadata"]
+      }));
+    } finally {
+      client.release();
     }
-    return finalSorted.slice(0, Math.max(topK, 15));
   }
-  async delete(_id, _namespace) {
-    console.warn("[MultiTablePostgresProvider] delete() is a no-op for multi-table mode.");
+  async delete(id, namespace) {
+    const where = namespace ? "WHERE id = $1 AND namespace = $2" : "WHERE id = $1";
+    const params = namespace ? [id, namespace] : [id];
+    await this.pool.query(`DELETE FROM ${this.tableName} ${where}`, params);
   }
-  async deleteNamespace(_namespace) {
-    console.warn("[MultiTablePostgresProvider] deleteNamespace() is a no-op for multi-table mode.");
+  async deleteNamespace(namespace) {
+    await this.pool.query(`DELETE FROM ${this.tableName} WHERE namespace = $1`, [namespace]);
   }
   async ping() {
     try {
@@ -437,9 +427,7 @@ var MultiTablePostgresProvider = class extends BaseVectorProvider {
     }
   }
   async disconnect() {
-    if (this.pool) {
-      await this.pool.end();
-    }
+    await this.pool.end();
   }
 };
 export {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@retrivora-ai/rag-engine",
-  "version": "1.8.0",
+  "version": "1.8.2",
   "description": "Retrivora AI is a plug-and-play AI engine for RAG chat experiences — generic vector DB + LLM provider, embeddable or standalone.",
   "author": "Abhinav Alkuchi",
   "license": "MIT",
@@ -85,15 +85,19 @@
   },
   "dependencies": {
     "@anthropic-ai/sdk": "^0.90.0",
-    "@google/genai": "^1.50.1",
+    "@google/genai": "^0.8.0",
+    "@google/generative-ai": "^0.24.1",
     "@pinecone-database/pinecone": "^7.2.0",
+    "@types/papaparse": "^5.5.2",
     "axios": "^1.15.0",
     "lucide-react": "^1.8.0",
     "next": ">=14.0.0",
     "next-themes": "^0.4.6",
     "openai": "^6.34.0",
+    "papaparse": "^5.5.3",
     "react": ">=18.0.0",
     "react-dom": ">=18.0.0",
+    "react-is": "^18.3.1",
     "react-markdown": "^10.1.0",
     "recharts": "^3.8.1",
     "remark-gfm": "^4.0.1"

package/src/app/api/upload/route.ts

@@ -0,0 +1,4 @@
+import { createUploadHandler } from '@/handlers';
+import { getRagConfig } from '@/config/serverConfig';
+
+export const POST = createUploadHandler(getRagConfig());

package/src/app/constants.tsx

@@ -112,12 +112,12 @@ const { reply, sources } = await pipeline.ask('How does Retrivora work?');`,
     title: 'Route Handlers',
     language: 'typescript',
     description: 'Deploy production-ready API endpoints in seconds using pre-built factories.',
-    code: `import { createChatHandler, createUploadHandler, getRagConfig } from '@retrivora-ai/rag-engine/server';
+    code: `import { createChatStreamHandler, createUploadHandler, getRagConfig } from '@retrivora-ai/rag-engine/server';
 
 const config = getRagConfig();
 
 // src/app/api/chat/route.ts
-export const POST = createChatHandler(config);
+export const POST = createChatStreamHandler(config);
 
 // src/app/api/upload/route.ts
 export const POST = createUploadHandler(config);`,