@retrivora-ai/rag-engine 1.0.9 → 1.1.1

package/src/providers/vectordb/QdrantProvider.ts

@@ -6,9 +6,14 @@ import { VectorMatch, UpsertDocument } from '../../types';
 
 /**
  * QdrantProvider — implementation for Qdrant using its REST API.
+ * Fully dynamic: automatically discovers and indexes schema from available data.
  */
 export class QdrantProvider extends BaseVectorProvider {
   private http: AxiosInstance;
+  private contentField: string;
+  private metadataField: string;
+  private isFlatPayload: boolean;
+  private schemaDiscovered = false;
 
   constructor(config: VectorDBConfig) {
     super(config);
@@ -16,8 +21,13 @@ export class QdrantProvider extends BaseVectorProvider {
     const baseUrl = opts.baseUrl as string;
     if (!baseUrl) throw new Error('[QdrantProvider] baseUrl is required');
 
+    this.contentField = (opts.contentField as string) || 'content';
+    this.metadataField = (opts.metadataField as string) || 'metadata';
+    this.isFlatPayload = !!opts.flatPayload;
+
     this.http = axios.create({
       baseURL: baseUrl,
+      timeout: 15000, // 15s timeout for vector DB operations
       headers: {
         'Content-Type': 'application/json',
         ...(opts.apiKey ? { 'api-key': opts.apiKey as string } : {}),
@@ -26,9 +36,76 @@ export class QdrantProvider extends BaseVectorProvider {
   }
 
   async initialize(): Promise<void> {
+    if (this.schemaDiscovered) return;
+    
     await this.ping();
     await this.ensureCollection();
-    await this.ensureIndex();
+
+    // 1. Discover schema from existing data
+    console.log(`[QdrantProvider] 🔍 Discovering schema for collection "${this.indexName}"...`);
+    const discoveredFields = await this.discoverSchema();
+    
+    // 2. Add client-provided fields (if any) as fallback/override
+    const opts = this.config.options as Record<string, unknown>;
+    const configFields = (opts.filterableFields as string[]) || [];
+    
+    const allFields = [...new Set([...discoveredFields, ...configFields])];
+
+    if (allFields.length > 0) {
+      console.log(`[QdrantProvider] 🛠 Ensuring indexes for ${allFields.length} discovered fields...`);
+      // Parallelize index creation for much faster startup
+      await Promise.all(
+        allFields.map(async (fieldInfo) => {
+          const [fieldName, schemaType] = fieldInfo.split(':');
+          return this.ensureIndex(fieldName, (schemaType as any) || 'keyword');
+        })
+      );
+    } else {
+      console.log(`[QdrantProvider] ℹ️ No fields discovered for indexing.`);
+    }
+
+    this.schemaDiscovered = true;
+  }
+
+  /**
+   * Samples points from the collection to discover available payload fields.
+   */
+  private async discoverSchema(): Promise<string[]> {
+    try {
+      // Fetch 20 points to get a representative sample of the schema
+      const { data } = await this.http.post(`/collections/${this.indexName}/points/scroll`, {
+        limit: 20,
+        with_payload: true,
+        with_vector: false
+      });
+
+      const points = data.result?.points || [];
+      const keys = new Set<string>();
+
+      for (const point of points) {
+        const payload = point.payload || {};
+        
+        // Extract root keys
+        Object.keys(payload).forEach(k => {
+          if (k !== 'namespace' && k !== this.contentField && k !== this.metadataField) {
+            keys.add(k);
+          }
+        });
+
+        // Extract nested metadata keys if applicable
+        if (!this.isFlatPayload && payload[this.metadataField]) {
+          const meta = payload[this.metadataField];
+          if (typeof meta === 'object' && meta !== null) {
+            Object.keys(meta).forEach(k => keys.add(k));
+          }
+        }
+      }
+
+      return Array.from(keys);
+    } catch (err) {
+      console.warn(`[QdrantProvider] ⚠️ Schema discovery failed:`, err instanceof Error ? err.message : String(err));
+      return [];
+    }
   }
 
   /**
@@ -58,32 +135,25 @@ export class QdrantProvider extends BaseVectorProvider {
   }
 
   /**
-   * Ensures that the 'namespace' field has a keyword index for efficient filtering.
-   * Qdrant requires this for search filters to work in many configurations.
+   * Ensures that a payload field has an index.
    */
-  private async ensureIndex(): Promise<void> {
+  private async ensureIndex(fieldName: string, schema: string = 'keyword'): Promise<void> {
+    let fullPath = fieldName;
+    if (!this.isFlatPayload && !fieldName.includes('.') && fieldName !== 'namespace' && fieldName !== this.contentField) {
+      fullPath = `${this.metadataField}.${fieldName}`;
+    }
+
     try {
       await this.http.put(`/collections/${this.indexName}/index`, {
-        field_name: 'namespace',
-        field_schema: 'keyword',
+        field_name: fullPath,
+        field_schema: schema,
       });
-      console.log(`[QdrantProvider] ✅ Ensured keyword index for "namespace" on collection "${this.indexName}"`);
+      console.log(`[QdrantProvider] ✅ Ensured ${schema} index for "${fullPath}"`);
     } catch (err) {
       let status: number | undefined;
-      let data: unknown;
-
-      if (axios.isAxiosError(err)) {
-        status = err.response?.status;
-        data = err.response?.data;
-      }
-      
-      // 409 means it already exists, which is fine.
-      if (status === 409) {
-        return;
-      }
-
-      const errorMessage = err instanceof Error ? err.message : String(err);
-      console.warn(`[QdrantProvider] ⚠️ Could not ensure namespace index (Status: ${status}):`, JSON.stringify(data || errorMessage));
+      if (axios.isAxiosError(err)) status = err.response?.status;
+      if (status === 409) return;
+      console.warn(`[QdrantProvider] ⚠️ Could not ensure index for "${fullPath}":`, err instanceof Error ? err.message : String(err));
     }
   }
 
@@ -93,15 +163,24 @@ export class QdrantProvider extends BaseVectorProvider {
 
   async batchUpsert(docs: UpsertDocument[], namespace?: string): Promise<void> {
     const payload = {
-      points: docs.map(doc => ({
-        id: this.normalizeId(doc.id),
-        vector: doc.vector,
-        payload: {
-          content: doc.content,
-          metadata: doc.metadata || {},
+      points: docs.map(doc => {
+        const pointPayload: Record<string, any> = {
+          [this.contentField]: doc.content,
           ...(namespace ? { namespace } : {}),
-        },
-      })),
+        };
+
+        if (this.isFlatPayload) {
+          Object.assign(pointPayload, doc.metadata || {});
+        } else {
+          pointPayload[this.metadataField] = doc.metadata || {};
+        }
+
+        return {
+          id: this.normalizeId(doc.id),
+          vector: doc.vector,
+          payload: pointPayload,
+        };
+      }),
     };
     await this.http.put(`/collections/${this.indexName}/points`, payload);
   }
@@ -113,12 +192,18 @@ export class QdrantProvider extends BaseVectorProvider {
       must.push({ key: 'namespace', match: { value: namespace } });
     }
 
-    // 1. Apply metadata filters
     const sanitizedFilter = this.sanitizeFilter(_filter);
     for (const [key, value] of Object.entries(sanitizedFilter)) {
-      // Qdrant uses dot notation for nested payload fields. 
-      // Our payload structure is { content, metadata: { ... }, namespace }
-      must.push({ key: `metadata.${key}`, match: { value: value } });
+      let filterKey = key;
+      if (!this.isFlatPayload && !key.includes('.') && key !== 'namespace' && key !== this.contentField) {
+        filterKey = `${this.metadataField}.${key}`;
+      }
+      
+      if (Array.isArray(value)) {
+        must.push({ key: filterKey, match: { any: value } });
+      } else {
+        must.push({ key: filterKey, match: { value: value } });
+      }
     }
 
     const payload = {
@@ -132,31 +217,45 @@ export class QdrantProvider extends BaseVectorProvider {
       filter: must.length > 0 ? { must } : undefined,
     };
 
-    console.log(`[QdrantProvider] 🔍 Searching "${this.indexName}" | Namespace: ${namespace || 'default'} | TopK: ${topK}`);
-    if (must.length > (namespace ? 1 : 0)) {
-      console.log(`[QdrantProvider] 🛠 Filters:`, JSON.stringify(must.filter(m => m.key !== 'namespace')));
-    }
-
     const { data } = await this.http.post(`/collections/${this.indexName}/points/search`, payload);
     
-    const results = (data.result || []).map((res: Record<string, unknown>) => ({
-      id: res['id'] as string,
-      score: res['score'] as number,
-      content: (res['payload'] as Record<string, unknown>)?.content as string || '',
-      metadata: (res['payload'] as Record<string, unknown>)?.metadata as Record<string, unknown> || {},
-    }));
-
-    if (results.length === 0) {
-      console.warn(`[QdrantProvider] ⚠️ Retrieval returned 0 results. Check if data exists in namespace "${namespace}"`);
-    } else {
-      console.log(`[QdrantProvider] ✅ Found ${results.length} results (best score: ${results[0].score.toFixed(4)})`);
-    }
+    const results = (data.result || []).map((res: Record<string, any>) => {
+      const p = res.payload || {};
+      let content = p[this.contentField] || '';
+      
+      if (!content) {
+        const stringFields = Object.entries(p)
+          .filter(([_, v]) => typeof v === 'string')
+          .map(([k, v]) => ({ key: k, val: v as string }));
+        
+        if (stringFields.length > 0) {
+          const bestMatch = stringFields.sort((a, b) => b.val.length - a.val.length)[0];
+          content = bestMatch.val;
+        } else {
+          content = JSON.stringify(p);
+        }
+      }
+
+      let metadata: Record<string, any> = {};
+      if (this.isFlatPayload) {
+        metadata = { ...p };
+        delete metadata[this.contentField];
+        delete metadata['namespace'];
+      } else {
+        metadata = p[this.metadataField] || p;
+      }
+
+      return {
+        id: res.id as string,
+        score: res.score as number,
+        content,
+        metadata,
+      };
+    });
 
     return results;
   }
 
-
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
   async delete(id: string | number, _namespace?: string): Promise<void> {
     await this.http.post(`/collections/${this.indexName}/points/delete`, {
       points: [this.normalizeId(id)],
@@ -180,21 +279,16 @@ export class QdrantProvider extends BaseVectorProvider {
     }
   }
 
-  /**
-   * Normalizes an ID into a format Qdrant accepts (UUID or integer).
-   * For string IDs that aren't UUIDs, it generates a deterministic UUID v5-like hash.
-   */
   private normalizeId(id: string | number): string | number {
     if (typeof id === 'number') return id;
-    
-    // Check if already a valid UUID
     const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
     if (uuidRegex.test(id)) return id.toLowerCase();
-
-    // Hash string to deterministic UUID
     const hash = crypto.createHash('md5').update(id).digest('hex');
     return `${hash.slice(0, 8)}-${hash.slice(8, 12)}-${hash.slice(12, 16)}-${hash.slice(16, 20)}-${hash.slice(20, 32)}`;
   }
 
   async disconnect(): Promise<void> {}
 }
+
+
+

package/dist/chunk-NT5VP7MT.mjs

@@ -1,174 +0,0 @@
-import {
-  BaseVectorProvider
-} from "./chunk-IMP6FUCY.mjs";
-import {
-  __spreadValues
-} from "./chunk-X4TOT24V.mjs";
-
-// src/providers/vectordb/QdrantProvider.ts
-import axios from "axios";
-import crypto from "crypto";
-var QdrantProvider = class extends BaseVectorProvider {
-  constructor(config) {
-    super(config);
-    const opts = config.options;
-    const baseUrl = opts.baseUrl;
-    if (!baseUrl) throw new Error("[QdrantProvider] baseUrl is required");
-    this.http = axios.create({
-      baseURL: baseUrl,
-      headers: __spreadValues({
-        "Content-Type": "application/json"
-      }, opts.apiKey ? { "api-key": opts.apiKey } : {})
-    });
-  }
-  async initialize() {
-    await this.ping();
-    await this.ensureCollection();
-    await this.ensureIndex();
-  }
-  /**
-   * Ensures the collection exists. Creates it if missing.
-   */
-  async ensureCollection() {
-    var _a;
-    try {
-      await this.http.get(`/collections/${this.indexName}`);
-      console.log(`[QdrantProvider] \u2705 Collection "${this.indexName}" already exists.`);
-    } catch (err) {
-      if (axios.isAxiosError(err) && ((_a = err.response) == null ? void 0 : _a.status) === 404) {
-        const opts = this.config.options;
-        const dimensionsForCreate = opts.dimensions || 1536;
-        console.log(`[QdrantProvider] \u23F3 Creating collection "${this.indexName}" (dimensions: ${dimensionsForCreate})...`);
-        await this.http.put(`/collections/${this.indexName}`, {
-          vectors: {
-            size: dimensionsForCreate,
-            distance: "Cosine"
-          }
-        });
-        console.log(`[QdrantProvider] \u2705 Created collection "${this.indexName}"`);
-      } else {
-        throw err;
-      }
-    }
-  }
-  /**
-   * Ensures that the 'namespace' field has a keyword index for efficient filtering.
-   * Qdrant requires this for search filters to work in many configurations.
-   */
-  async ensureIndex() {
-    var _a, _b;
-    try {
-      await this.http.put(`/collections/${this.indexName}/index`, {
-        field_name: "namespace",
-        field_schema: "keyword"
-      });
-      console.log(`[QdrantProvider] \u2705 Ensured keyword index for "namespace" on collection "${this.indexName}"`);
-    } catch (err) {
-      let status;
-      let data;
-      if (axios.isAxiosError(err)) {
-        status = (_a = err.response) == null ? void 0 : _a.status;
-        data = (_b = err.response) == null ? void 0 : _b.data;
-      }
-      if (status === 409) {
-        return;
-      }
-      const errorMessage = err instanceof Error ? err.message : String(err);
-      console.warn(`[QdrantProvider] \u26A0\uFE0F Could not ensure namespace index (Status: ${status}):`, JSON.stringify(data || errorMessage));
-    }
-  }
-  async upsert(doc, namespace) {
-    await this.batchUpsert([doc], namespace);
-  }
-  async batchUpsert(docs, namespace) {
-    const payload = {
-      points: docs.map((doc) => ({
-        id: this.normalizeId(doc.id),
-        vector: doc.vector,
-        payload: __spreadValues({
-          content: doc.content,
-          metadata: doc.metadata || {}
-        }, namespace ? { namespace } : {})
-      }))
-    };
-    await this.http.put(`/collections/${this.indexName}/points`, payload);
-  }
-  async query(vector, topK, namespace, _filter) {
-    const must = [];
-    if (namespace) {
-      must.push({ key: "namespace", match: { value: namespace } });
-    }
-    const sanitizedFilter = this.sanitizeFilter(_filter);
-    for (const [key, value] of Object.entries(sanitizedFilter)) {
-      must.push({ key: `metadata.${key}`, match: { value } });
-    }
-    const payload = {
-      vector,
-      limit: topK,
-      with_payload: true,
-      params: {
-        hnsw_ef: this.config.options.efSearch || Math.max(topK * 20, 128),
-        exact: false
-      },
-      filter: must.length > 0 ? { must } : void 0
-    };
-    console.log(`[QdrantProvider] \u{1F50D} Searching "${this.indexName}" | Namespace: ${namespace || "default"} | TopK: ${topK}`);
-    if (must.length > (namespace ? 1 : 0)) {
-      console.log(`[QdrantProvider] \u{1F6E0} Filters:`, JSON.stringify(must.filter((m) => m.key !== "namespace")));
-    }
-    const { data } = await this.http.post(`/collections/${this.indexName}/points/search`, payload);
-    const results = (data.result || []).map((res) => {
-      var _a, _b;
-      return {
-        id: res["id"],
-        score: res["score"],
-        content: ((_a = res["payload"]) == null ? void 0 : _a.content) || "",
-        metadata: ((_b = res["payload"]) == null ? void 0 : _b.metadata) || {}
-      };
-    });
-    if (results.length === 0) {
-      console.warn(`[QdrantProvider] \u26A0\uFE0F Retrieval returned 0 results. Check if data exists in namespace "${namespace}"`);
-    } else {
-      console.log(`[QdrantProvider] \u2705 Found ${results.length} results (best score: ${results[0].score.toFixed(4)})`);
-    }
-    return results;
-  }
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  async delete(id, _namespace) {
-    await this.http.post(`/collections/${this.indexName}/points/delete`, {
-      points: [this.normalizeId(id)]
-    });
-  }
-  async deleteNamespace(_namespace) {
-    await this.http.post(`/collections/${this.indexName}/points/delete`, {
-      filter: {
-        must: [{ key: "namespace", match: { value: _namespace } }]
-      }
-    });
-  }
-  async ping() {
-    try {
-      await this.http.get("/healthz");
-      return true;
-    } catch (e) {
-      return false;
-    }
-  }
-  /**
-   * Normalizes an ID into a format Qdrant accepts (UUID or integer).
-   * For string IDs that aren't UUIDs, it generates a deterministic UUID v5-like hash.
-   */
-  normalizeId(id) {
-    if (typeof id === "number") return id;
-    const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
-    if (uuidRegex.test(id)) return id.toLowerCase();
-    const hash = crypto.createHash("md5").update(id).digest("hex");
-    return `${hash.slice(0, 8)}-${hash.slice(8, 12)}-${hash.slice(12, 16)}-${hash.slice(16, 20)}-${hash.slice(20, 32)}`;
-  }
-  async disconnect() {
-  }
-};
-
-export {
-  QdrantProvider
-};