@retrivora-ai/rag-engine 0.1.4 → 0.1.6

package/src/providers/vectordb/MilvusProvider.ts

@@ -74,7 +74,7 @@ export class MilvusProvider extends BaseVectorProvider {
   }
 
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  async delete(id: string, _namespace?: string): Promise<void> {
+  async delete(id: string | number, _namespace?: string): Promise<void> {
     await this.http.post('/v1/vector/delete', {
       collectionName: this.indexName,
       id: id,

package/src/providers/vectordb/MongoDBProvider.ts

@@ -95,7 +95,7 @@ export class MongoDBProvider extends BaseVectorProvider {
     }));
   }
 
-  async delete(id: string, namespace?: string): Promise<void> {
+  async delete(id: string | number, namespace?: string): Promise<void> {
     await this.collection!.deleteOne({ _id: id, ...(namespace ? { namespace } : {}) });
   }
 

package/src/providers/vectordb/PineconeProvider.ts

@@ -31,7 +31,7 @@ export class PineconeProvider extends BaseVectorProvider {
   async upsert(doc: UpsertDocument, namespace?: string): Promise<void> {
     await this.index(namespace).upsert({
       records: [{
-        id: doc.id,
+        id: String(doc.id),
         values: doc.vector,
         metadata: { content: doc.content, ...(doc.metadata ?? {}) } as RecordMetadata,
       }],
@@ -42,7 +42,7 @@ export class PineconeProvider extends BaseVectorProvider {
     const BATCH = 100;
     for (let i = 0; i < docs.length; i += BATCH) {
       const records = docs.slice(i, i + BATCH).map((d) => ({
-        id: d.id,
+        id: String(d.id),
         values: d.vector,
         metadata: { content: d.content, ...(d.metadata ?? {}) } as RecordMetadata,
       }));
@@ -65,8 +65,8 @@ export class PineconeProvider extends BaseVectorProvider {
     }));
   }
 
-  async delete(id: string, namespace?: string): Promise<void> {
-    await this.index(namespace).deleteOne({ id });
+  async delete(id: string | number, namespace?: string): Promise<void> {
+    await this.index(namespace).deleteOne({ id: String(id) });
   }
 
   async deleteNamespace(namespace: string): Promise<void> {

package/src/providers/vectordb/PostgreSQLProvider.ts

@@ -60,8 +60,40 @@ export class PostgreSQLProvider extends BaseVectorProvider {
   }
 
   async batchUpsert(docs: UpsertDocument[], namespace = ''): Promise<void> {
-    for (const doc of docs) {
-      await this.upsert(doc, namespace);
+    if (docs.length === 0) return;
+
+    const client = await this.pool.connect();
+    try {
+      await client.query('BEGIN');
+      
+      const BATCH_SIZE = 50;
+      for (let i = 0; i < docs.length; i += BATCH_SIZE) {
+        const batch = docs.slice(i, i + BATCH_SIZE);
+        const values: unknown[] = [];
+        const valuePlaceholders = batch.map((doc, idx) => {
+          const offset = idx * 5;
+          values.push(doc.id, namespace, doc.content, JSON.stringify(doc.metadata ?? {}), `[${doc.vector.join(',')}]`);
+          return `($${offset + 1}, $${offset + 2}, $${offset + 3}, $${offset + 4}, $${offset + 5}::vector)`;
+        }).join(', ');
+
+        const query = `
+          INSERT INTO ${this.tableName} (id, namespace, content, metadata, embedding)
+          VALUES ${valuePlaceholders}
+          ON CONFLICT (id) DO UPDATE
+            SET namespace = EXCLUDED.namespace,
+                content   = EXCLUDED.content,
+                metadata  = EXCLUDED.metadata,
+                embedding = EXCLUDED.embedding
+        `;
+        await client.query(query, values);
+      }
+      
+      await client.query('COMMIT');
+    } catch (error) {
+      await client.query('ROLLBACK');
+      throw error;
+    } finally {
+      client.release();
     }
   }
 
@@ -99,7 +131,7 @@ export class PostgreSQLProvider extends BaseVectorProvider {
     }));
   }
 
-  async delete(id: string, namespace?: string): Promise<void> {
+  async delete(id: string | number, namespace?: string): Promise<void> {
     const where = namespace ? 'WHERE id = $1 AND namespace = $2' : 'WHERE id = $1';
     const params = namespace ? [id, namespace] : [id];
     await this.pool.query(`DELETE FROM ${this.tableName} ${where}`, params);

package/src/providers/vectordb/QdrantProvider.ts

@@ -26,6 +26,37 @@ export class QdrantProvider extends BaseVectorProvider {
 
   async initialize(): Promise<void> {
     await this.ping();
+    await this.ensureIndex();
+  }
+
+  /**
+   * Ensures that the 'namespace' field has a keyword index for efficient filtering.
+   * Qdrant requires this for search filters to work in many configurations.
+   */
+  private async ensureIndex(): Promise<void> {
+    try {
+      await this.http.put(`/collections/${this.indexName}/index`, {
+        field_name: 'namespace',
+        field_schema: 'keyword',
+      });
+      console.log(`[QdrantProvider] ✅ Ensured keyword index for "namespace" on collection "${this.indexName}"`);
+    } catch (err) {
+      let status: number | undefined;
+      let data: unknown;
+
+      if (axios.isAxiosError(err)) {
+        status = err.response?.status;
+        data = err.response?.data;
+      }
+      
+      // 409 means it already exists, which is fine.
+      if (status === 409) {
+        return;
+      }
+
+      const errorMessage = err instanceof Error ? err.message : String(err);
+      console.warn(`[QdrantProvider] ⚠️ Could not ensure namespace index (Status: ${status}):`, JSON.stringify(data || errorMessage));
+    }
   }
 
   async upsert(doc: UpsertDocument, namespace?: string): Promise<void> {
@@ -67,13 +98,12 @@ export class QdrantProvider extends BaseVectorProvider {
   }
 
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  async delete(id: string, _namespace?: string): Promise<void> {
+  async delete(id: string | number, _namespace?: string): Promise<void> {
     await this.http.post(`/collections/${this.indexName}/points/delete`, {
       points: [id],
     });
   }
 
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
   async deleteNamespace(_namespace: string): Promise<void> {
     await this.http.post(`/collections/${this.indexName}/points/delete`, {
       filter: {

package/src/rag/DocumentChunker.ts

@@ -20,7 +20,7 @@ export interface ChunkOptions {
   /** Overlap between consecutive chunks in characters (default 200) */
   chunkOverlap?: number;
   /** Source document identifier used as ID prefix */
-  docId?: string;
+  docId?: string | number;
   /** Extra metadata to attach to every chunk */
   metadata?: Record<string, unknown>;
 }
@@ -96,7 +96,7 @@ export class DocumentChunker {
    * Chunk multiple documents at once.
    */
   chunkMany(
-    documents: Array<{ content: string; docId?: string; metadata?: Record<string, unknown> }>
+    documents: Array<{ content: string; docId?: string | number; metadata?: Record<string, unknown> }>
   ): Chunk[] {
     return documents.flatMap((doc) =>
       this.chunk(doc.content, { docId: doc.docId, metadata: doc.metadata })

package/src/server.ts

@@ -7,7 +7,7 @@
 export type { RagConfig, VectorDBConfig, LLMConfig, EmbeddingConfig, UIConfig, RAGConfig } from './config/RagConfig';
 export type { VectorMatch, UpsertDocument } from './types';
 export type { ILLMProvider, ChatMessage, ChatOptions, EmbedOptions } from './llm/ILLMProvider';
-export type { IngestDocument, ChatResponse } from './core/Pipeline';
+export type { IngestDocument, ChatResponse } from './types';
 export type { Chunk, ChunkOptions } from './rag/DocumentChunker';
 
 // ── Server-Only Logic (Node.js dependent) ─────────────────────

package/src/types/index.ts

@@ -1,13 +1,24 @@
 export interface VectorMatch {
-  id: string;
+  id: string | number;
   score: number;
   content: string;
   metadata?: Record<string, unknown>;
 }
 
 export interface UpsertDocument {
-  id: string;
+  id: string | number;
   vector: number[];
   content: string;
   metadata?: Record<string, unknown>;
 }
+
+export interface IngestDocument {
+  docId: string | number;
+  content: string;
+  metadata?: Record<string, unknown>;
+}
+
+export interface ChatResponse {
+  reply: string;
+  sources: VectorMatch[];
+}

package/dist/DocumentChunker-BQ5kQD7B.d.ts

@@ -1,155 +0,0 @@
-import { e as RagConfig, C as ChatMessage, b as ChatResponse, d as IngestDocument, g as VectorDBConfig, f as UpsertDocument, V as VectorMatch, L as LLMConfig, c as EmbeddingConfig, I as ILLMProvider } from './Pipeline-Bo6CUCox.js';
-
-/**
- * VectorPlugin — main orchestrator class.
- * This is the primary interface for host applications.
- */
-declare class VectorPlugin {
-    private config;
-    private pipeline;
-    /**
-     * Initializes the plugin with the host configuration.
-     * @param hostConfig - Configuration object passed from the host application.
-     */
-    constructor(hostConfig?: Partial<RagConfig>);
-    /**
-     * Get the current resolved configuration.
-     */
-    getConfig(): RagConfig;
-    /**
-     * Run a chat query.
-     */
-    chat(message: string, history?: ChatMessage[], namespace?: string): Promise<ChatResponse>;
-    /**
-     * Ingest documents into the vector database.
-     */
-    ingest(documents: IngestDocument[], namespace?: string): Promise<Array<{
-        docId: string;
-        chunksIngested: number;
-    }>>;
-    /**
-     * Check the health of the connected providers.
-     */
-    health(): Promise<Record<string, boolean>>;
-}
-
-/**
- * ConfigResolver — validates and normalizes host configuration.
- * It merges host-provided config with environment defaults.
- */
-declare class ConfigResolver {
-    /**
-     * Resolves the final configuration by merging host-provided config with defaults.
-     * @param hostConfig - Partial configuration passed from the host application.
-     */
-    static resolve(hostConfig?: Partial<RagConfig>): RagConfig;
-    /**
-     * Validates the configuration for required fields.
-     */
-    static validate(config: RagConfig): void;
-}
-
-/**
- * BaseVectorProvider — Abstract base class for all vector database providers.
- * Each provider (Pinecone, Milvus, Redis, etc.) must extend this class.
- */
-declare abstract class BaseVectorProvider {
-    protected readonly config: VectorDBConfig;
-    protected readonly indexName: string;
-    constructor(config: VectorDBConfig);
-    /**
-     * Initialise the connection (create tables, verify index, etc.)
-     */
-    abstract initialize(): Promise<void>;
-    /**
-     * Insert or update a single vector + content pair.
-     */
-    abstract upsert(doc: UpsertDocument, namespace?: string): Promise<void>;
-    /**
-     * Batch upsert for efficient ingestion of many documents.
-     */
-    abstract batchUpsert(docs: UpsertDocument[], namespace?: string): Promise<void>;
-    /**
-     * Find the top-K most similar vectors to the query vector.
-     */
-    abstract query(vector: number[], topK: number, namespace?: string, filter?: Record<string, unknown>): Promise<VectorMatch[]>;
-    /**
-     * Delete a stored vector by ID.
-     */
-    abstract delete(id: string, namespace?: string): Promise<void>;
-    /**
-     * Delete all vectors in a namespace (full data reset for a project).
-     */
-    abstract deleteNamespace(namespace: string): Promise<void>;
-    /**
-     * Check if the underlying DB is reachable.
-     */
-    abstract ping(): Promise<boolean>;
-    /**
-     * Gracefully close the connection.
-     */
-    abstract disconnect(): Promise<void>;
-}
-
-/**
- * ProviderRegistry — dynamic provider loader for Vector DBs and LLMs.
- */
-declare class ProviderRegistry {
-    private static vectorProviders;
-    /**
-     * Register a custom vector provider.
-     */
-    static registerVectorProvider(name: string, providerClass: new (config: VectorDBConfig) => BaseVectorProvider): void;
-    /**
-     * Creates a vector database provider based on the configuration.
-     */
-    static createVectorProvider(config: VectorDBConfig): Promise<BaseVectorProvider>;
-    /**
-     * Creates an LLM provider based on the configuration.
-     */
-    static createLLMProvider(llmConfig: LLMConfig, embeddingConfig?: EmbeddingConfig): ILLMProvider;
-}
-
-/**
- * DocumentChunker — splits long text into overlapping chunks
- * suitable for vector embedding and retrieval.
- *
- * Strategy: sentence-aware sliding window
- *   1. Split on sentence boundaries
- *   2. Accumulate sentences into chunks up to `chunkSize` characters
- *   3. Carry over `chunkOverlap` characters from the previous chunk
- */
-interface Chunk {
-    id: string;
-    content: string;
-    metadata?: Record<string, unknown>;
-}
-interface ChunkOptions {
-    /** Target chunk size in characters (default 1000) */
-    chunkSize?: number;
-    /** Overlap between consecutive chunks in characters (default 200) */
-    chunkOverlap?: number;
-    /** Source document identifier used as ID prefix */
-    docId?: string;
-    /** Extra metadata to attach to every chunk */
-    metadata?: Record<string, unknown>;
-}
-declare class DocumentChunker {
-    private readonly chunkSize;
-    private readonly chunkOverlap;
-    constructor(chunkSize?: number, chunkOverlap?: number);
-    /**
-     * Split a single text string into overlapping chunks.
-     */
-    chunk(text: string, options?: ChunkOptions): Chunk[];
-    /**
-     * Chunk multiple documents at once.
-     */
-    chunkMany(documents: Array<{
-        content: string;
-        docId?: string;
-        metadata?: Record<string, unknown>;
-    }>): Chunk[];
-}
-
-export { BaseVectorProvider as B, type Chunk as C, DocumentChunker as D, ProviderRegistry as P, VectorPlugin as V, type ChunkOptions as a, ConfigResolver as b };

package/dist/DocumentChunker-D9-fObJp.d.mts

@@ -1,155 +0,0 @@
-import { e as RagConfig, C as ChatMessage, b as ChatResponse, d as IngestDocument, g as VectorDBConfig, f as UpsertDocument, V as VectorMatch, L as LLMConfig, c as EmbeddingConfig, I as ILLMProvider } from './Pipeline-Bo6CUCox.mjs';
-
-/**
- * VectorPlugin — main orchestrator class.
- * This is the primary interface for host applications.
- */
-declare class VectorPlugin {
-    private config;
-    private pipeline;
-    /**
-     * Initializes the plugin with the host configuration.
-     * @param hostConfig - Configuration object passed from the host application.
-     */
-    constructor(hostConfig?: Partial<RagConfig>);
-    /**
-     * Get the current resolved configuration.
-     */
-    getConfig(): RagConfig;
-    /**
-     * Run a chat query.
-     */
-    chat(message: string, history?: ChatMessage[], namespace?: string): Promise<ChatResponse>;
-    /**
-     * Ingest documents into the vector database.
-     */
-    ingest(documents: IngestDocument[], namespace?: string): Promise<Array<{
-        docId: string;
-        chunksIngested: number;
-    }>>;
-    /**
-     * Check the health of the connected providers.
-     */
-    health(): Promise<Record<string, boolean>>;
-}
-
-/**
- * ConfigResolver — validates and normalizes host configuration.
- * It merges host-provided config with environment defaults.
- */
-declare class ConfigResolver {
-    /**
-     * Resolves the final configuration by merging host-provided config with defaults.
-     * @param hostConfig - Partial configuration passed from the host application.
-     */
-    static resolve(hostConfig?: Partial<RagConfig>): RagConfig;
-    /**
-     * Validates the configuration for required fields.
-     */
-    static validate(config: RagConfig): void;
-}
-
-/**
- * BaseVectorProvider — Abstract base class for all vector database providers.
- * Each provider (Pinecone, Milvus, Redis, etc.) must extend this class.
- */
-declare abstract class BaseVectorProvider {
-    protected readonly config: VectorDBConfig;
-    protected readonly indexName: string;
-    constructor(config: VectorDBConfig);
-    /**
-     * Initialise the connection (create tables, verify index, etc.)
-     */
-    abstract initialize(): Promise<void>;
-    /**
-     * Insert or update a single vector + content pair.
-     */
-    abstract upsert(doc: UpsertDocument, namespace?: string): Promise<void>;
-    /**
-     * Batch upsert for efficient ingestion of many documents.
-     */
-    abstract batchUpsert(docs: UpsertDocument[], namespace?: string): Promise<void>;
-    /**
-     * Find the top-K most similar vectors to the query vector.
-     */
-    abstract query(vector: number[], topK: number, namespace?: string, filter?: Record<string, unknown>): Promise<VectorMatch[]>;
-    /**
-     * Delete a stored vector by ID.
-     */
-    abstract delete(id: string, namespace?: string): Promise<void>;
-    /**
-     * Delete all vectors in a namespace (full data reset for a project).
-     */
-    abstract deleteNamespace(namespace: string): Promise<void>;
-    /**
-     * Check if the underlying DB is reachable.
-     */
-    abstract ping(): Promise<boolean>;
-    /**
-     * Gracefully close the connection.
-     */
-    abstract disconnect(): Promise<void>;
-}
-
-/**
- * ProviderRegistry — dynamic provider loader for Vector DBs and LLMs.
- */
-declare class ProviderRegistry {
-    private static vectorProviders;
-    /**
-     * Register a custom vector provider.
-     */
-    static registerVectorProvider(name: string, providerClass: new (config: VectorDBConfig) => BaseVectorProvider): void;
-    /**
-     * Creates a vector database provider based on the configuration.
-     */
-    static createVectorProvider(config: VectorDBConfig): Promise<BaseVectorProvider>;
-    /**
-     * Creates an LLM provider based on the configuration.
-     */
-    static createLLMProvider(llmConfig: LLMConfig, embeddingConfig?: EmbeddingConfig): ILLMProvider;
-}
-
-/**
- * DocumentChunker — splits long text into overlapping chunks
- * suitable for vector embedding and retrieval.
- *
- * Strategy: sentence-aware sliding window
- *   1. Split on sentence boundaries
- *   2. Accumulate sentences into chunks up to `chunkSize` characters
- *   3. Carry over `chunkOverlap` characters from the previous chunk
- */
-interface Chunk {
-    id: string;
-    content: string;
-    metadata?: Record<string, unknown>;
-}
-interface ChunkOptions {
-    /** Target chunk size in characters (default 1000) */
-    chunkSize?: number;
-    /** Overlap between consecutive chunks in characters (default 200) */
-    chunkOverlap?: number;
-    /** Source document identifier used as ID prefix */
-    docId?: string;
-    /** Extra metadata to attach to every chunk */
-    metadata?: Record<string, unknown>;
-}
-declare class DocumentChunker {
-    private readonly chunkSize;
-    private readonly chunkOverlap;
-    constructor(chunkSize?: number, chunkOverlap?: number);
-    /**
-     * Split a single text string into overlapping chunks.
-     */
-    chunk(text: string, options?: ChunkOptions): Chunk[];
-    /**
-     * Chunk multiple documents at once.
-     */
-    chunkMany(documents: Array<{
-        content: string;
-        docId?: string;
-        metadata?: Record<string, unknown>;
-    }>): Chunk[];
-}
-
-export { BaseVectorProvider as B, type Chunk as C, DocumentChunker as D, ProviderRegistry as P, VectorPlugin as V, type ChunkOptions as a, ConfigResolver as b };