@retrivora-ai/rag-engine 0.1.5 → 0.1.6

package/src/providers/vectordb/PineconeProvider.ts

@@ -31,7 +31,7 @@ export class PineconeProvider extends BaseVectorProvider {
   async upsert(doc: UpsertDocument, namespace?: string): Promise<void> {
     await this.index(namespace).upsert({
       records: [{
-        id: doc.id,
+        id: String(doc.id),
         values: doc.vector,
         metadata: { content: doc.content, ...(doc.metadata ?? {}) } as RecordMetadata,
       }],
@@ -42,7 +42,7 @@ export class PineconeProvider extends BaseVectorProvider {
     const BATCH = 100;
     for (let i = 0; i < docs.length; i += BATCH) {
       const records = docs.slice(i, i + BATCH).map((d) => ({
-        id: d.id,
+        id: String(d.id),
         values: d.vector,
         metadata: { content: d.content, ...(d.metadata ?? {}) } as RecordMetadata,
       }));
@@ -65,8 +65,8 @@ export class PineconeProvider extends BaseVectorProvider {
     }));
   }
 
-  async delete(id: string, namespace?: string): Promise<void> {
-    await this.index(namespace).deleteOne({ id });
+  async delete(id: string | number, namespace?: string): Promise<void> {
+    await this.index(namespace).deleteOne({ id: String(id) });
   }
 
   async deleteNamespace(namespace: string): Promise<void> {

package/src/providers/vectordb/PostgreSQLProvider.ts

@@ -60,8 +60,40 @@ export class PostgreSQLProvider extends BaseVectorProvider {
   }
 
   async batchUpsert(docs: UpsertDocument[], namespace = ''): Promise<void> {
-    for (const doc of docs) {
-      await this.upsert(doc, namespace);
+    if (docs.length === 0) return;
+
+    const client = await this.pool.connect();
+    try {
+      await client.query('BEGIN');
+      
+      const BATCH_SIZE = 50;
+      for (let i = 0; i < docs.length; i += BATCH_SIZE) {
+        const batch = docs.slice(i, i + BATCH_SIZE);
+        const values: unknown[] = [];
+        const valuePlaceholders = batch.map((doc, idx) => {
+          const offset = idx * 5;
+          values.push(doc.id, namespace, doc.content, JSON.stringify(doc.metadata ?? {}), `[${doc.vector.join(',')}]`);
+          return `($${offset + 1}, $${offset + 2}, $${offset + 3}, $${offset + 4}, $${offset + 5}::vector)`;
+        }).join(', ');
+
+        const query = `
+          INSERT INTO ${this.tableName} (id, namespace, content, metadata, embedding)
+          VALUES ${valuePlaceholders}
+          ON CONFLICT (id) DO UPDATE
+            SET namespace = EXCLUDED.namespace,
+                content   = EXCLUDED.content,
+                metadata  = EXCLUDED.metadata,
+                embedding = EXCLUDED.embedding
+        `;
+        await client.query(query, values);
+      }
+      
+      await client.query('COMMIT');
+    } catch (error) {
+      await client.query('ROLLBACK');
+      throw error;
+    } finally {
+      client.release();
     }
   }
 
@@ -99,7 +131,7 @@ export class PostgreSQLProvider extends BaseVectorProvider {
     }));
   }
 
-  async delete(id: string, namespace?: string): Promise<void> {
+  async delete(id: string | number, namespace?: string): Promise<void> {
     const where = namespace ? 'WHERE id = $1 AND namespace = $2' : 'WHERE id = $1';
     const params = namespace ? [id, namespace] : [id];
     await this.pool.query(`DELETE FROM ${this.tableName} ${where}`, params);

package/src/providers/vectordb/QdrantProvider.ts

@@ -26,6 +26,37 @@ export class QdrantProvider extends BaseVectorProvider {
 
   async initialize(): Promise<void> {
     await this.ping();
+    await this.ensureIndex();
+  }
+
+  /**
+   * Ensures that the 'namespace' field has a keyword index for efficient filtering.
+   * Qdrant requires this for search filters to work in many configurations.
+   */
+  private async ensureIndex(): Promise<void> {
+    try {
+      await this.http.put(`/collections/${this.indexName}/index`, {
+        field_name: 'namespace',
+        field_schema: 'keyword',
+      });
+      console.log(`[QdrantProvider] ✅ Ensured keyword index for "namespace" on collection "${this.indexName}"`);
+    } catch (err) {
+      let status: number | undefined;
+      let data: unknown;
+
+      if (axios.isAxiosError(err)) {
+        status = err.response?.status;
+        data = err.response?.data;
+      }
+      
+      // 409 means it already exists, which is fine.
+      if (status === 409) {
+        return;
+      }
+
+      const errorMessage = err instanceof Error ? err.message : String(err);
+      console.warn(`[QdrantProvider] ⚠️ Could not ensure namespace index (Status: ${status}):`, JSON.stringify(data || errorMessage));
+    }
   }
 
   async upsert(doc: UpsertDocument, namespace?: string): Promise<void> {
@@ -67,13 +98,12 @@ export class QdrantProvider extends BaseVectorProvider {
   }
 
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  async delete(id: string, _namespace?: string): Promise<void> {
+  async delete(id: string | number, _namespace?: string): Promise<void> {
     await this.http.post(`/collections/${this.indexName}/points/delete`, {
       points: [id],
     });
   }
 
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
   async deleteNamespace(_namespace: string): Promise<void> {
     await this.http.post(`/collections/${this.indexName}/points/delete`, {
       filter: {

package/src/rag/DocumentChunker.ts

@@ -20,7 +20,7 @@ export interface ChunkOptions {
   /** Overlap between consecutive chunks in characters (default 200) */
   chunkOverlap?: number;
   /** Source document identifier used as ID prefix */
-  docId?: string;
+  docId?: string | number;
   /** Extra metadata to attach to every chunk */
   metadata?: Record<string, unknown>;
 }
@@ -96,7 +96,7 @@ export class DocumentChunker {
    * Chunk multiple documents at once.
    */
   chunkMany(
-    documents: Array<{ content: string; docId?: string; metadata?: Record<string, unknown> }>
+    documents: Array<{ content: string; docId?: string | number; metadata?: Record<string, unknown> }>
   ): Chunk[] {
     return documents.flatMap((doc) =>
       this.chunk(doc.content, { docId: doc.docId, metadata: doc.metadata })

package/src/types/index.ts

@@ -1,19 +1,19 @@
 export interface VectorMatch {
-  id: string;
+  id: string | number;
   score: number;
   content: string;
   metadata?: Record<string, unknown>;
 }
 
 export interface UpsertDocument {
-  id: string;
+  id: string | number;
   vector: number[];
   content: string;
   metadata?: Record<string, unknown>;
 }
 
 export interface IngestDocument {
-  docId: string;
+  docId: string | number;
   content: string;
   metadata?: Record<string, unknown>;
 }