@retrivora-ai/rag-engine 0.1.5 → 0.1.7

package/src/providers/vectordb/PineconeProvider.ts

@@ -31,7 +31,7 @@ export class PineconeProvider extends BaseVectorProvider {
   async upsert(doc: UpsertDocument, namespace?: string): Promise<void> {
     await this.index(namespace).upsert({
       records: [{
-        id: doc.id,
+        id: String(doc.id),
         values: doc.vector,
         metadata: { content: doc.content, ...(doc.metadata ?? {}) } as RecordMetadata,
       }],
@@ -42,7 +42,7 @@ export class PineconeProvider extends BaseVectorProvider {
     const BATCH = 100;
     for (let i = 0; i < docs.length; i += BATCH) {
       const records = docs.slice(i, i + BATCH).map((d) => ({
-        id: d.id,
+        id: String(d.id),
         values: d.vector,
         metadata: { content: d.content, ...(d.metadata ?? {}) } as RecordMetadata,
       }));
@@ -65,8 +65,8 @@ export class PineconeProvider extends BaseVectorProvider {
     }));
   }
 
-  async delete(id: string, namespace?: string): Promise<void> {
-    await this.index(namespace).deleteOne({ id });
+  async delete(id: string | number, namespace?: string): Promise<void> {
+    await this.index(namespace).deleteOne({ id: String(id) });
   }
 
   async deleteNamespace(namespace: string): Promise<void> {

package/src/providers/vectordb/PostgreSQLProvider.ts

@@ -60,8 +60,40 @@ export class PostgreSQLProvider extends BaseVectorProvider {
   }
 
   async batchUpsert(docs: UpsertDocument[], namespace = ''): Promise<void> {
-    for (const doc of docs) {
-      await this.upsert(doc, namespace);
+    if (docs.length === 0) return;
+
+    const client = await this.pool.connect();
+    try {
+      await client.query('BEGIN');
+      
+      const BATCH_SIZE = 50;
+      for (let i = 0; i < docs.length; i += BATCH_SIZE) {
+        const batch = docs.slice(i, i + BATCH_SIZE);
+        const values: unknown[] = [];
+        const valuePlaceholders = batch.map((doc, idx) => {
+          const offset = idx * 5;
+          values.push(doc.id, namespace, doc.content, JSON.stringify(doc.metadata ?? {}), `[${doc.vector.join(',')}]`);
+          return `($${offset + 1}, $${offset + 2}, $${offset + 3}, $${offset + 4}, $${offset + 5}::vector)`;
+        }).join(', ');
+
+        const query = `
+          INSERT INTO ${this.tableName} (id, namespace, content, metadata, embedding)
+          VALUES ${valuePlaceholders}
+          ON CONFLICT (id) DO UPDATE
+            SET namespace = EXCLUDED.namespace,
+                content   = EXCLUDED.content,
+                metadata  = EXCLUDED.metadata,
+                embedding = EXCLUDED.embedding
+        `;
+        await client.query(query, values);
+      }
+      
+      await client.query('COMMIT');
+    } catch (error) {
+      await client.query('ROLLBACK');
+      throw error;
+    } finally {
+      client.release();
     }
   }
 
@@ -99,7 +131,7 @@ export class PostgreSQLProvider extends BaseVectorProvider {
     }));
   }
 
-  async delete(id: string, namespace?: string): Promise<void> {
+  async delete(id: string | number, namespace?: string): Promise<void> {
     const where = namespace ? 'WHERE id = $1 AND namespace = $2' : 'WHERE id = $1';
     const params = namespace ? [id, namespace] : [id];
     await this.pool.query(`DELETE FROM ${this.tableName} ${where}`, params);

package/src/providers/vectordb/QdrantProvider.ts

@@ -1,4 +1,5 @@
 import axios, { AxiosInstance } from 'axios';
+import crypto from 'crypto';
 import { VectorDBConfig } from '../../config/RagConfig';
 import { BaseVectorProvider } from './BaseVectorProvider';
 import { VectorMatch, UpsertDocument } from '../../types';
@@ -26,6 +27,67 @@ export class QdrantProvider extends BaseVectorProvider {
 
   async initialize(): Promise<void> {
     await this.ping();
+    await this.ensureCollection();
+    await this.ensureIndex();
+  }
+
+  /**
+   * Ensures the collection exists. Creates it if missing.
+   */
+  private async ensureCollection(): Promise<void> {
+    try {
+      const opts = this.config.options as Record<string, unknown>;
+      const dimensions = (opts.dimensions as number) || 1536;
+
+      await this.http.get(`/collections/${this.indexName}`);
+      console.log(`[QdrantProvider] ✅ Collection "${this.indexName}" already exists.`);
+    } catch (err) {
+      if (axios.isAxiosError(err) && err.response?.status === 404) {
+        const opts = this.config.options as Record<string, unknown>;
+        const dimensions = (opts.dimensions as number) || 1536;
+
+        console.log(`[QdrantProvider] ⏳ Creating collection "${this.indexName}" (dimensions: ${dimensions})...`);
+        await this.http.put(`/collections/${this.indexName}`, {
+          vectors: {
+            size: dimensions,
+            distance: 'Cosine',
+          },
+        });
+        console.log(`[QdrantProvider] ✅ Created collection "${this.indexName}"`);
+      } else {
+        throw err;
+      }
+    }
+  }
+
+  /**
+   * Ensures that the 'namespace' field has a keyword index for efficient filtering.
+   * Qdrant requires this for search filters to work in many configurations.
+   */
+  private async ensureIndex(): Promise<void> {
+    try {
+      await this.http.put(`/collections/${this.indexName}/index`, {
+        field_name: 'namespace',
+        field_schema: 'keyword',
+      });
+      console.log(`[QdrantProvider] ✅ Ensured keyword index for "namespace" on collection "${this.indexName}"`);
+    } catch (err) {
+      let status: number | undefined;
+      let data: unknown;
+
+      if (axios.isAxiosError(err)) {
+        status = err.response?.status;
+        data = err.response?.data;
+      }
+      
+      // 409 means it already exists, which is fine.
+      if (status === 409) {
+        return;
+      }
+
+      const errorMessage = err instanceof Error ? err.message : String(err);
+      console.warn(`[QdrantProvider] ⚠️ Could not ensure namespace index (Status: ${status}):`, JSON.stringify(data || errorMessage));
+    }
   }
 
   async upsert(doc: UpsertDocument, namespace?: string): Promise<void> {
@@ -35,7 +97,7 @@ export class QdrantProvider extends BaseVectorProvider {
   async batchUpsert(docs: UpsertDocument[], namespace?: string): Promise<void> {
     const payload = {
       points: docs.map(doc => ({
-        id: doc.id,
+        id: this.normalizeId(doc.id),
         vector: doc.vector,
         payload: {
           content: doc.content,
@@ -67,13 +129,12 @@ export class QdrantProvider extends BaseVectorProvider {
   }
 
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  async delete(id: string, _namespace?: string): Promise<void> {
+  async delete(id: string | number, _namespace?: string): Promise<void> {
     await this.http.post(`/collections/${this.indexName}/points/delete`, {
-      points: [id],
+      points: [this.normalizeId(id)],
     });
   }
 
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
   async deleteNamespace(_namespace: string): Promise<void> {
     await this.http.post(`/collections/${this.indexName}/points/delete`, {
       filter: {
@@ -91,5 +152,21 @@ export class QdrantProvider extends BaseVectorProvider {
     }
   }
 
+  /**
+   * Normalizes an ID into a format Qdrant accepts (UUID or integer).
+   * For string IDs that aren't UUIDs, it generates a deterministic UUID v5-like hash.
+   */
+  private normalizeId(id: string | number): string | number {
+    if (typeof id === 'number') return id;
+    
+    // Check if already a valid UUID
+    const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+    if (uuidRegex.test(id)) return id.toLowerCase();
+
+    // Hash string to deterministic UUID
+    const hash = crypto.createHash('md5').update(id).digest('hex');
+    return `${hash.slice(0, 8)}-${hash.slice(8, 12)}-${hash.slice(12, 16)}-${hash.slice(16, 20)}-${hash.slice(20, 32)}`;
+  }
+
   async disconnect(): Promise<void> {}
 }

package/src/rag/DocumentChunker.ts

@@ -20,7 +20,7 @@ export interface ChunkOptions {
   /** Overlap between consecutive chunks in characters (default 200) */
   chunkOverlap?: number;
   /** Source document identifier used as ID prefix */
-  docId?: string;
+  docId?: string | number;
   /** Extra metadata to attach to every chunk */
   metadata?: Record<string, unknown>;
 }
@@ -96,7 +96,7 @@ export class DocumentChunker {
    * Chunk multiple documents at once.
    */
   chunkMany(
-    documents: Array<{ content: string; docId?: string; metadata?: Record<string, unknown> }>
+    documents: Array<{ content: string; docId?: string | number; metadata?: Record<string, unknown> }>
   ): Chunk[] {
     return documents.flatMap((doc) =>
       this.chunk(doc.content, { docId: doc.docId, metadata: doc.metadata })

package/src/types/index.ts

@@ -1,19 +1,19 @@
 export interface VectorMatch {
-  id: string;
+  id: string | number;
   score: number;
   content: string;
   metadata?: Record<string, unknown>;
 }
 
 export interface UpsertDocument {
-  id: string;
+  id: string | number;
   vector: number[];
   content: string;
   metadata?: Record<string, unknown>;
 }
 
 export interface IngestDocument {
-  docId: string;
+  docId: string | number;
   content: string;
   metadata?: Record<string, unknown>;
 }

package/dist/chunk-6FODXNUF.mjs

@@ -1,91 +0,0 @@
-import {
-  BaseVectorProvider
-} from "./chunk-VOIWNO5O.mjs";
-import {
-  __spreadValues
-} from "./chunk-I4E63NIC.mjs";
-
-// src/providers/vectordb/QdrantProvider.ts
-import axios from "axios";
-var QdrantProvider = class extends BaseVectorProvider {
-  constructor(config) {
-    super(config);
-    const opts = config.options;
-    const baseUrl = opts.baseUrl;
-    if (!baseUrl) throw new Error("[QdrantProvider] baseUrl is required");
-    this.http = axios.create({
-      baseURL: baseUrl,
-      headers: __spreadValues({
-        "Content-Type": "application/json"
-      }, opts.apiKey ? { "api-key": opts.apiKey } : {})
-    });
-  }
-  async initialize() {
-    await this.ping();
-  }
-  async upsert(doc, namespace) {
-    await this.batchUpsert([doc], namespace);
-  }
-  async batchUpsert(docs, namespace) {
-    const payload = {
-      points: docs.map((doc) => ({
-        id: doc.id,
-        vector: doc.vector,
-        payload: __spreadValues({
-          content: doc.content,
-          metadata: doc.metadata || {}
-        }, namespace ? { namespace } : {})
-      }))
-    };
-    await this.http.put(`/collections/${this.indexName}/points`, payload);
-  }
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  async query(vector, topK, namespace, _filter) {
-    const payload = {
-      vector,
-      limit: topK,
-      with_payload: true,
-      filter: namespace ? {
-        must: [{ key: "namespace", match: { value: namespace } }]
-      } : void 0
-    };
-    const { data } = await this.http.post(`/collections/${this.indexName}/points/search`, payload);
-    return (data.result || []).map((res) => {
-      var _a, _b;
-      return {
-        id: res["id"],
-        score: res["score"],
-        content: ((_a = res["payload"]) == null ? void 0 : _a.content) || "",
-        metadata: ((_b = res["payload"]) == null ? void 0 : _b.metadata) || {}
-      };
-    });
-  }
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  async delete(id, _namespace) {
-    await this.http.post(`/collections/${this.indexName}/points/delete`, {
-      points: [id]
-    });
-  }
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  async deleteNamespace(_namespace) {
-    await this.http.post(`/collections/${this.indexName}/points/delete`, {
-      filter: {
-        must: [{ key: "namespace", match: { value: _namespace } }]
-      }
-    });
-  }
-  async ping() {
-    try {
-      await this.http.get("/healthz");
-      return true;
-    } catch (e) {
-      return false;
-    }
-  }
-  async disconnect() {
-  }
-};
-
-export {
-  QdrantProvider
-};