@retrivora-ai/rag-engine 1.0.9 → 1.1.1

package/dist/handlers/index.js

@@ -842,20 +842,76 @@ var init_QdrantProvider = __esm({
     QdrantProvider = class extends BaseVectorProvider {
       constructor(config) {
         super(config);
+        this.schemaDiscovered = false;
         const opts = config.options;
         const baseUrl = opts.baseUrl;
         if (!baseUrl) throw new Error("[QdrantProvider] baseUrl is required");
+        this.contentField = opts.contentField || "content";
+        this.metadataField = opts.metadataField || "metadata";
+        this.isFlatPayload = !!opts.flatPayload;
         this.http = import_axios4.default.create({
           baseURL: baseUrl,
+          timeout: 15e3,
+          // 15s timeout for vector DB operations
           headers: __spreadValues({
             "Content-Type": "application/json"
           }, opts.apiKey ? { "api-key": opts.apiKey } : {})
         });
       }
       async initialize() {
+        if (this.schemaDiscovered) return;
         await this.ping();
         await this.ensureCollection();
-        await this.ensureIndex();
+        console.log(`[QdrantProvider] \u{1F50D} Discovering schema for collection "${this.indexName}"...`);
+        const discoveredFields = await this.discoverSchema();
+        const opts = this.config.options;
+        const configFields = opts.filterableFields || [];
+        const allFields = [.../* @__PURE__ */ new Set([...discoveredFields, ...configFields])];
+        if (allFields.length > 0) {
+          console.log(`[QdrantProvider] \u{1F6E0} Ensuring indexes for ${allFields.length} discovered fields...`);
+          await Promise.all(
+            allFields.map(async (fieldInfo) => {
+              const [fieldName, schemaType] = fieldInfo.split(":");
+              return this.ensureIndex(fieldName, schemaType || "keyword");
+            })
+          );
+        } else {
+          console.log(`[QdrantProvider] \u2139\uFE0F No fields discovered for indexing.`);
+        }
+        this.schemaDiscovered = true;
+      }
+      /**
+       * Samples points from the collection to discover available payload fields.
+       */
+      async discoverSchema() {
+        var _a;
+        try {
+          const { data } = await this.http.post(`/collections/${this.indexName}/points/scroll`, {
+            limit: 20,
+            with_payload: true,
+            with_vector: false
+          });
+          const points = ((_a = data.result) == null ? void 0 : _a.points) || [];
+          const keys = /* @__PURE__ */ new Set();
+          for (const point of points) {
+            const payload = point.payload || {};
+            Object.keys(payload).forEach((k) => {
+              if (k !== "namespace" && k !== this.contentField && k !== this.metadataField) {
+                keys.add(k);
+              }
+            });
+            if (!this.isFlatPayload && payload[this.metadataField]) {
+              const meta = payload[this.metadataField];
+              if (typeof meta === "object" && meta !== null) {
+                Object.keys(meta).forEach((k) => keys.add(k));
+              }
+            }
+          }
+          return Array.from(keys);
+        } catch (err) {
+          console.warn(`[QdrantProvider] \u26A0\uFE0F Schema discovery failed:`, err instanceof Error ? err.message : String(err));
+          return [];
+        }
       }
       /**
        * Ensures the collection exists. Creates it if missing.
@@ -883,29 +939,25 @@ var init_QdrantProvider = __esm({
         }
       }
       /**
-       * Ensures that the 'namespace' field has a keyword index for efficient filtering.
-       * Qdrant requires this for search filters to work in many configurations.
+       * Ensures that a payload field has an index.
        */
-      async ensureIndex() {
-        var _a, _b;
+      async ensureIndex(fieldName, schema = "keyword") {
+        var _a;
+        let fullPath = fieldName;
+        if (!this.isFlatPayload && !fieldName.includes(".") && fieldName !== "namespace" && fieldName !== this.contentField) {
+          fullPath = `${this.metadataField}.${fieldName}`;
+        }
         try {
           await this.http.put(`/collections/${this.indexName}/index`, {
-            field_name: "namespace",
-            field_schema: "keyword"
+            field_name: fullPath,
+            field_schema: schema
           });
-          console.log(`[QdrantProvider] \u2705 Ensured keyword index for "namespace" on collection "${this.indexName}"`);
+          console.log(`[QdrantProvider] \u2705 Ensured ${schema} index for "${fullPath}"`);
         } catch (err) {
           let status;
-          let data;
-          if (import_axios4.default.isAxiosError(err)) {
-            status = (_a = err.response) == null ? void 0 : _a.status;
-            data = (_b = err.response) == null ? void 0 : _b.data;
-          }
-          if (status === 409) {
-            return;
-          }
-          const errorMessage = err instanceof Error ? err.message : String(err);
-          console.warn(`[QdrantProvider] \u26A0\uFE0F Could not ensure namespace index (Status: ${status}):`, JSON.stringify(data || errorMessage));
+          if (import_axios4.default.isAxiosError(err)) status = (_a = err.response) == null ? void 0 : _a.status;
+          if (status === 409) return;
+          console.warn(`[QdrantProvider] \u26A0\uFE0F Could not ensure index for "${fullPath}":`, err instanceof Error ? err.message : String(err));
         }
       }
       async upsert(doc, namespace) {
@@ -913,14 +965,21 @@ var init_QdrantProvider = __esm({
       }
       async batchUpsert(docs, namespace) {
         const payload = {
-          points: docs.map((doc) => ({
-            id: this.normalizeId(doc.id),
-            vector: doc.vector,
-            payload: __spreadValues({
-              content: doc.content,
-              metadata: doc.metadata || {}
-            }, namespace ? { namespace } : {})
-          }))
+          points: docs.map((doc) => {
+            const pointPayload = __spreadValues({
+              [this.contentField]: doc.content
+            }, namespace ? { namespace } : {});
+            if (this.isFlatPayload) {
+              Object.assign(pointPayload, doc.metadata || {});
+            } else {
+              pointPayload[this.metadataField] = doc.metadata || {};
+            }
+            return {
+              id: this.normalizeId(doc.id),
+              vector: doc.vector,
+              payload: pointPayload
+            };
+          })
         };
         await this.http.put(`/collections/${this.indexName}/points`, payload);
       }
@@ -931,7 +990,15 @@ var init_QdrantProvider = __esm({
         }
         const sanitizedFilter = this.sanitizeFilter(_filter);
         for (const [key, value] of Object.entries(sanitizedFilter)) {
-          must.push({ key: `metadata.${key}`, match: { value } });
+          let filterKey = key;
+          if (!this.isFlatPayload && !key.includes(".") && key !== "namespace" && key !== this.contentField) {
+            filterKey = `${this.metadataField}.${key}`;
+          }
+          if (Array.isArray(value)) {
+            must.push({ key: filterKey, match: { any: value } });
+          } else {
+            must.push({ key: filterKey, match: { value } });
+          }
         }
         const payload = {
           vector,
@@ -943,28 +1010,36 @@ var init_QdrantProvider = __esm({
           },
           filter: must.length > 0 ? { must } : void 0
         };
-        console.log(`[QdrantProvider] \u{1F50D} Searching "${this.indexName}" | Namespace: ${namespace || "default"} | TopK: ${topK}`);
-        if (must.length > (namespace ? 1 : 0)) {
-          console.log(`[QdrantProvider] \u{1F6E0} Filters:`, JSON.stringify(must.filter((m) => m.key !== "namespace")));
-        }
         const { data } = await this.http.post(`/collections/${this.indexName}/points/search`, payload);
         const results = (data.result || []).map((res) => {
-          var _a, _b;
+          const p = res.payload || {};
+          let content = p[this.contentField] || "";
+          if (!content) {
+            const stringFields = Object.entries(p).filter(([_, v]) => typeof v === "string").map(([k, v]) => ({ key: k, val: v }));
+            if (stringFields.length > 0) {
+              const bestMatch = stringFields.sort((a, b) => b.val.length - a.val.length)[0];
+              content = bestMatch.val;
+            } else {
+              content = JSON.stringify(p);
+            }
+          }
+          let metadata = {};
+          if (this.isFlatPayload) {
+            metadata = __spreadValues({}, p);
+            delete metadata[this.contentField];
+            delete metadata["namespace"];
+          } else {
+            metadata = p[this.metadataField] || p;
+          }
           return {
-            id: res["id"],
-            score: res["score"],
-            content: ((_a = res["payload"]) == null ? void 0 : _a.content) || "",
-            metadata: ((_b = res["payload"]) == null ? void 0 : _b.metadata) || {}
+            id: res.id,
+            score: res.score,
+            content,
+            metadata
           };
         });
-        if (results.length === 0) {
-          console.warn(`[QdrantProvider] \u26A0\uFE0F Retrieval returned 0 results. Check if data exists in namespace "${namespace}"`);
-        } else {
-          console.log(`[QdrantProvider] \u2705 Found ${results.length} results (best score: ${results[0].score.toFixed(4)})`);
-        }
         return results;
       }
-      // eslint-disable-next-line @typescript-eslint/no-unused-vars
       async delete(id, _namespace) {
         await this.http.post(`/collections/${this.indexName}/points/delete`, {
           points: [this.normalizeId(id)]
@@ -985,10 +1060,6 @@ var init_QdrantProvider = __esm({
           return false;
         }
       }
-      /**
-       * Normalizes an ID into a format Qdrant accepts (UUID or integer).
-       * For string IDs that aren't UUIDs, it generates a deterministic UUID v5-like hash.
-       */
       normalizeId(id) {
         if (typeof id === "number") return id;
         const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
@@ -1868,6 +1939,53 @@ ${context}`
     });
     return (_h = (_g = (_f = completion.choices[0]) == null ? void 0 : _f.message) == null ? void 0 : _g.content) != null ? _h : "";
   }
+  chatStream(messages, context, options) {
+    return __asyncGenerator(this, null, function* () {
+      var _a, _b, _c, _d, _e, _f, _g;
+      const systemContent = (_a = this.llmConfig.systemPrompt) != null ? _a : `You are a helpful assistant. Answer questions based on the provided context.
+
+Context:
+${context}`;
+      const systemMessage = {
+        role: "system",
+        content: systemContent.includes("{{context}}") ? systemContent.replace("{{context}}", context) : `${systemContent}
+
+Context:
+${context}`
+      };
+      const formattedMessages = [
+        systemMessage,
+        ...messages.map((m) => ({
+          role: m.role,
+          content: m.content
+        }))
+      ];
+      const stream = yield new __await(this.client.chat.completions.create({
+        model: this.llmConfig.model,
+        messages: formattedMessages,
+        max_tokens: (_c = (_b = options == null ? void 0 : options.maxTokens) != null ? _b : this.llmConfig.maxTokens) != null ? _c : 1024,
+        temperature: (_e = (_d = options == null ? void 0 : options.temperature) != null ? _d : this.llmConfig.temperature) != null ? _e : 0.7,
+        stop: options == null ? void 0 : options.stop,
+        stream: true
+      }));
+      try {
+        for (var iter = __forAwait(stream), more, temp, error; more = !(temp = yield new __await(iter.next())).done; more = false) {
+          const chunk = temp.value;
+          const content = ((_g = (_f = chunk.choices[0]) == null ? void 0 : _f.delta) == null ? void 0 : _g.content) || "";
+          if (content) yield content;
+        }
+      } catch (temp) {
+        error = [temp];
+      } finally {
+        try {
+          more && (temp = iter.return) && (yield new __await(temp.call(iter)));
+        } finally {
+          if (error)
+            throw error[0];
+        }
+      }
+    });
+  }
   async embed(text, options) {
     const results = await this.batchEmbed([text], options);
     return results[0];
@@ -1973,6 +2091,47 @@ ${context}`;
     const block = response.content[0];
     return block.type === "text" ? block.text : "";
   }
+  chatStream(messages, context, options) {
+    return __asyncGenerator(this, null, function* () {
+      var _a, _b, _c;
+      const systemPrompt = (_a = this.llmConfig.systemPrompt) != null ? _a : `You are a helpful assistant. Use the context below to answer the user's question accurately.
+
+Context:
+${context}`;
+      const system = systemPrompt.includes("{{context}}") ? systemPrompt.replace("{{context}}", context) : `${systemPrompt}
+
+Context:
+${context}`;
+      const anthropicMessages = messages.map((m) => ({
+        role: m.role === "assistant" ? "assistant" : "user",
+        content: m.content
+      }));
+      const stream = yield new __await(this.client.messages.create({
+        model: this.llmConfig.model,
+        max_tokens: (_c = (_b = options == null ? void 0 : options.maxTokens) != null ? _b : this.llmConfig.maxTokens) != null ? _c : 1024,
+        system,
+        messages: anthropicMessages,
+        stream: true
+      }));
+      try {
+        for (var iter = __forAwait(stream), more, temp, error; more = !(temp = yield new __await(iter.next())).done; more = false) {
+          const chunk = temp.value;
+          if (chunk.type === "content_block_delta" && chunk.delta.type === "text_delta") {
+            yield chunk.delta.text;
+          }
+        }
+      } catch (temp) {
+        error = [temp];
+      } finally {
+        try {
+          more && (temp = iter.return) && (yield new __await(temp.call(iter)));
+        } finally {
+          if (error)
+            throw error[0];
+        }
+      }
+    });
+  }
   async embed(text, options) {
     void text;
     void options;
@@ -2078,7 +2237,7 @@ var OllamaProvider = class {
   }
   chatStream(messages, context, options) {
     return __asyncGenerator(this, null, function* () {
-      var _a, _b, _c, _d, _e;
+      var _a, _b, _c, _d, _e, _f;
       const system = this.buildSystemPrompt(context);
       const response = yield new __await(this.http.post("/api/chat", {
         model: this.llmConfig.model,
@@ -2092,11 +2251,15 @@ var OllamaProvider = class {
           ...messages.map((m) => ({ role: m.role, content: m.content }))
         ]
       }, { responseType: "stream" }));
+      let lineBuffer = "";
       try {
         for (var iter = __forAwait(response.data), more, temp, error; more = !(temp = yield new __await(iter.next())).done; more = false) {
           const chunk = temp.value;
-          const lines = chunk.toString().split("\n").filter(Boolean);
+          lineBuffer += chunk.toString();
+          const lines = lineBuffer.split("\n");
+          lineBuffer = lines.pop() || "";
           for (const line of lines) {
+            if (!line.trim()) continue;
             try {
               const json = JSON.parse(line);
               if ((_e = json.message) == null ? void 0 : _e.content) {
@@ -2104,6 +2267,7 @@ var OllamaProvider = class {
               }
               if (json.done) return;
             } catch (e) {
+              console.warn("[OllamaProvider] Failed to parse streaming line:", line);
             }
           }
         }
@@ -2117,6 +2281,13 @@ var OllamaProvider = class {
             throw error[0];
         }
       }
+      if (lineBuffer.trim()) {
+        try {
+          const json = JSON.parse(lineBuffer);
+          if ((_f = json.message) == null ? void 0 : _f.content) yield json.message.content;
+        } catch (e) {
+        }
+      }
     });
   }
   buildSystemPrompt(context) {
@@ -2262,6 +2433,49 @@ ${context}`;
     });
     return (_f = response.text) != null ? _f : "";
   }
+  chatStream(messages, context, options) {
+    return __asyncGenerator(this, null, function* () {
+      var _a, _b, _c, _d, _e;
+      const systemPrompt = (_a = this.llmConfig.systemPrompt) != null ? _a : `You are a helpful assistant. Answer questions based on the provided context.
+
+Context:
+${context}`;
+      const systemInstruction = systemPrompt.includes("{{context}}") ? systemPrompt.replace("{{context}}", context) : `${systemPrompt}
+
+Context:
+${context}`;
+      const geminiMessages = messages.map((m) => ({
+        role: m.role === "assistant" ? "model" : "user",
+        parts: [{ text: m.content }]
+      }));
+      const result = yield new __await(this.client.models.generateContentStream({
+        model: this.llmConfig.model,
+        contents: geminiMessages,
+        config: {
+          systemInstruction,
+          temperature: (_c = (_b = options == null ? void 0 : options.temperature) != null ? _b : this.llmConfig.temperature) != null ? _c : 0.7,
+          maxOutputTokens: (_e = (_d = options == null ? void 0 : options.maxTokens) != null ? _d : this.llmConfig.maxTokens) != null ? _e : 1024,
+          stopSequences: options == null ? void 0 : options.stop
+        }
+      }));
+      try {
+        for (var iter = __forAwait(result.stream), more, temp, error; more = !(temp = yield new __await(iter.next())).done; more = false) {
+          const chunk = temp.value;
+          const text = chunk.text;
+          if (text) yield text;
+        }
+      } catch (temp) {
+        error = [temp];
+      } finally {
+        try {
+          more && (temp = iter.return) && (yield new __await(temp.call(iter)));
+        } finally {
+          if (error)
+            throw error[0];
+        }
+      }
+    });
+  }
   async embed(text, options) {
     var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k;
     const model = this.sanitizeModel(
@@ -3764,6 +3978,7 @@ var Pipeline = class {
         }
         const hints = QueryProcessor.extractQueryFieldHints(question, (_f = this.config.rag) == null ? void 0 : _f.filterableFields);
         const filter = QueryProcessor.buildQueryFilter(question, hints);
+        console.log(`[Pipeline] \u{1F9E9} Extracted filters:`, JSON.stringify(filter));
         const { sources: rawSources, graphData } = yield new __await(this.retrieve(searchQuery, {
           namespace: ns,
           topK: topK * 2,

package/dist/handlers/index.mjs

@@ -8,7 +8,7 @@ import {
   sseFrame,
   sseMetaFrame,
   sseTextFrame
-} from "../chunk-7S2SRQGL.mjs";
+} from "../chunk-JZ4H7EP6.mjs";
 import "../chunk-YLTMFW4M.mjs";
 import "../chunk-X4TOT24V.mjs";
 export {

package/dist/server.d.mts

@@ -731,18 +731,26 @@ declare class MilvusProvider extends BaseVectorProvider {
 
 /**
  * QdrantProvider — implementation for Qdrant using its REST API.
+ * Fully dynamic: automatically discovers and indexes schema from available data.
  */
 declare class QdrantProvider extends BaseVectorProvider {
     private http;
+    private contentField;
+    private metadataField;
+    private isFlatPayload;
+    private schemaDiscovered;
     constructor(config: VectorDBConfig);
     initialize(): Promise<void>;
+    /**
+     * Samples points from the collection to discover available payload fields.
+     */
+    private discoverSchema;
     /**
      * Ensures the collection exists. Creates it if missing.
      */
     private ensureCollection;
     /**
-     * Ensures that the 'namespace' field has a keyword index for efficient filtering.
-     * Qdrant requires this for search filters to work in many configurations.
+     * Ensures that a payload field has an index.
      */
     private ensureIndex;
     upsert(doc: UpsertDocument, namespace?: string): Promise<void>;
@@ -751,10 +759,6 @@ declare class QdrantProvider extends BaseVectorProvider {
     delete(id: string | number, _namespace?: string): Promise<void>;
     deleteNamespace(_namespace: string): Promise<void>;
     ping(): Promise<boolean>;
-    /**
-     * Normalizes an ID into a format Qdrant accepts (UUID or integer).
-     * For string IDs that aren't UUIDs, it generates a deterministic UUID v5-like hash.
-     */
     private normalizeId;
     disconnect(): Promise<void>;
 }
@@ -889,6 +893,7 @@ declare class OpenAIProvider implements ILLMProvider {
     static getValidator(): IProviderValidator;
     static getHealthChecker(): IProviderHealthChecker;
     chat(messages: ChatMessage[], context: string, options?: ChatOptions): Promise<string>;
+    chatStream(messages: ChatMessage[], context: string, options?: ChatOptions): AsyncIterable<string>;
     embed(text: string, options?: EmbedOptions): Promise<number[]>;
     batchEmbed(texts: string[], options?: EmbedOptions): Promise<number[][]>;
     ping(): Promise<boolean>;
@@ -906,6 +911,7 @@ declare class AnthropicProvider implements ILLMProvider {
     static getValidator(): IProviderValidator;
     static getHealthChecker(): IProviderHealthChecker;
     chat(messages: ChatMessage[], context: string, options?: ChatOptions): Promise<string>;
+    chatStream(messages: ChatMessage[], context: string, options?: ChatOptions): AsyncIterable<string>;
     embed(text: string, options?: EmbedOptions): Promise<number[]>;
     batchEmbed(texts: string[], options?: EmbedOptions): Promise<number[][]>;
     ping(): Promise<boolean>;

package/dist/server.d.ts

@@ -731,18 +731,26 @@ declare class MilvusProvider extends BaseVectorProvider {
 
 /**
  * QdrantProvider — implementation for Qdrant using its REST API.
+ * Fully dynamic: automatically discovers and indexes schema from available data.
  */
 declare class QdrantProvider extends BaseVectorProvider {
     private http;
+    private contentField;
+    private metadataField;
+    private isFlatPayload;
+    private schemaDiscovered;
     constructor(config: VectorDBConfig);
     initialize(): Promise<void>;
+    /**
+     * Samples points from the collection to discover available payload fields.
+     */
+    private discoverSchema;
     /**
      * Ensures the collection exists. Creates it if missing.
      */
     private ensureCollection;
     /**
-     * Ensures that the 'namespace' field has a keyword index for efficient filtering.
-     * Qdrant requires this for search filters to work in many configurations.
+     * Ensures that a payload field has an index.
      */
     private ensureIndex;
     upsert(doc: UpsertDocument, namespace?: string): Promise<void>;
@@ -751,10 +759,6 @@ declare class QdrantProvider extends BaseVectorProvider {
     delete(id: string | number, _namespace?: string): Promise<void>;
     deleteNamespace(_namespace: string): Promise<void>;
     ping(): Promise<boolean>;
-    /**
-     * Normalizes an ID into a format Qdrant accepts (UUID or integer).
-     * For string IDs that aren't UUIDs, it generates a deterministic UUID v5-like hash.
-     */
     private normalizeId;
     disconnect(): Promise<void>;
 }
@@ -889,6 +893,7 @@ declare class OpenAIProvider implements ILLMProvider {
     static getValidator(): IProviderValidator;
     static getHealthChecker(): IProviderHealthChecker;
     chat(messages: ChatMessage[], context: string, options?: ChatOptions): Promise<string>;
+    chatStream(messages: ChatMessage[], context: string, options?: ChatOptions): AsyncIterable<string>;
     embed(text: string, options?: EmbedOptions): Promise<number[]>;
     batchEmbed(texts: string[], options?: EmbedOptions): Promise<number[][]>;
     ping(): Promise<boolean>;
@@ -906,6 +911,7 @@ declare class AnthropicProvider implements ILLMProvider {
     static getValidator(): IProviderValidator;
     static getHealthChecker(): IProviderHealthChecker;
     chat(messages: ChatMessage[], context: string, options?: ChatOptions): Promise<string>;
+    chatStream(messages: ChatMessage[], context: string, options?: ChatOptions): AsyncIterable<string>;
     embed(text: string, options?: EmbedOptions): Promise<number[]>;
     batchEmbed(texts: string[], options?: EmbedOptions): Promise<number[][]>;
     ping(): Promise<boolean>;