@retrivora-ai/rag-engine 1.0.9 → 1.1.1

package/dist/{QdrantProvider-NGFXVDCF.mjs → QdrantProvider-NMXOULCU.mjs}

@@ -1,6 +1,6 @@
 import {
   QdrantProvider
-} from "./chunk-NT5VP7MT.mjs";
+} from "./chunk-63HITIWC.mjs";
 import "./chunk-IMP6FUCY.mjs";
 import "./chunk-X4TOT24V.mjs";
 export {

package/dist/chunk-63HITIWC.mjs

@@ -0,0 +1,245 @@
+import {
+  BaseVectorProvider
+} from "./chunk-IMP6FUCY.mjs";
+import {
+  __spreadValues
+} from "./chunk-X4TOT24V.mjs";
+
+// src/providers/vectordb/QdrantProvider.ts
+import axios from "axios";
+import crypto from "crypto";
+var QdrantProvider = class extends BaseVectorProvider {
+  constructor(config) {
+    super(config);
+    this.schemaDiscovered = false;
+    const opts = config.options;
+    const baseUrl = opts.baseUrl;
+    if (!baseUrl) throw new Error("[QdrantProvider] baseUrl is required");
+    this.contentField = opts.contentField || "content";
+    this.metadataField = opts.metadataField || "metadata";
+    this.isFlatPayload = !!opts.flatPayload;
+    this.http = axios.create({
+      baseURL: baseUrl,
+      timeout: 15e3,
+      // 15s timeout for vector DB operations
+      headers: __spreadValues({
+        "Content-Type": "application/json"
+      }, opts.apiKey ? { "api-key": opts.apiKey } : {})
+    });
+  }
+  async initialize() {
+    if (this.schemaDiscovered) return;
+    await this.ping();
+    await this.ensureCollection();
+    console.log(`[QdrantProvider] \u{1F50D} Discovering schema for collection "${this.indexName}"...`);
+    const discoveredFields = await this.discoverSchema();
+    const opts = this.config.options;
+    const configFields = opts.filterableFields || [];
+    const allFields = [.../* @__PURE__ */ new Set([...discoveredFields, ...configFields])];
+    if (allFields.length > 0) {
+      console.log(`[QdrantProvider] \u{1F6E0} Ensuring indexes for ${allFields.length} discovered fields...`);
+      await Promise.all(
+        allFields.map(async (fieldInfo) => {
+          const [fieldName, schemaType] = fieldInfo.split(":");
+          return this.ensureIndex(fieldName, schemaType || "keyword");
+        })
+      );
+    } else {
+      console.log(`[QdrantProvider] \u2139\uFE0F No fields discovered for indexing.`);
+    }
+    this.schemaDiscovered = true;
+  }
+  /**
+   * Samples points from the collection to discover available payload fields.
+   */
+  async discoverSchema() {
+    var _a;
+    try {
+      const { data } = await this.http.post(`/collections/${this.indexName}/points/scroll`, {
+        limit: 20,
+        with_payload: true,
+        with_vector: false
+      });
+      const points = ((_a = data.result) == null ? void 0 : _a.points) || [];
+      const keys = /* @__PURE__ */ new Set();
+      for (const point of points) {
+        const payload = point.payload || {};
+        Object.keys(payload).forEach((k) => {
+          if (k !== "namespace" && k !== this.contentField && k !== this.metadataField) {
+            keys.add(k);
+          }
+        });
+        if (!this.isFlatPayload && payload[this.metadataField]) {
+          const meta = payload[this.metadataField];
+          if (typeof meta === "object" && meta !== null) {
+            Object.keys(meta).forEach((k) => keys.add(k));
+          }
+        }
+      }
+      return Array.from(keys);
+    } catch (err) {
+      console.warn(`[QdrantProvider] \u26A0\uFE0F Schema discovery failed:`, err instanceof Error ? err.message : String(err));
+      return [];
+    }
+  }
+  /**
+   * Ensures the collection exists. Creates it if missing.
+   */
+  async ensureCollection() {
+    var _a;
+    try {
+      await this.http.get(`/collections/${this.indexName}`);
+      console.log(`[QdrantProvider] \u2705 Collection "${this.indexName}" already exists.`);
+    } catch (err) {
+      if (axios.isAxiosError(err) && ((_a = err.response) == null ? void 0 : _a.status) === 404) {
+        const opts = this.config.options;
+        const dimensionsForCreate = opts.dimensions || 1536;
+        console.log(`[QdrantProvider] \u23F3 Creating collection "${this.indexName}" (dimensions: ${dimensionsForCreate})...`);
+        await this.http.put(`/collections/${this.indexName}`, {
+          vectors: {
+            size: dimensionsForCreate,
+            distance: "Cosine"
+          }
+        });
+        console.log(`[QdrantProvider] \u2705 Created collection "${this.indexName}"`);
+      } else {
+        throw err;
+      }
+    }
+  }
+  /**
+   * Ensures that a payload field has an index.
+   */
+  async ensureIndex(fieldName, schema = "keyword") {
+    var _a;
+    let fullPath = fieldName;
+    if (!this.isFlatPayload && !fieldName.includes(".") && fieldName !== "namespace" && fieldName !== this.contentField) {
+      fullPath = `${this.metadataField}.${fieldName}`;
+    }
+    try {
+      await this.http.put(`/collections/${this.indexName}/index`, {
+        field_name: fullPath,
+        field_schema: schema
+      });
+      console.log(`[QdrantProvider] \u2705 Ensured ${schema} index for "${fullPath}"`);
+    } catch (err) {
+      let status;
+      if (axios.isAxiosError(err)) status = (_a = err.response) == null ? void 0 : _a.status;
+      if (status === 409) return;
+      console.warn(`[QdrantProvider] \u26A0\uFE0F Could not ensure index for "${fullPath}":`, err instanceof Error ? err.message : String(err));
+    }
+  }
+  async upsert(doc, namespace) {
+    await this.batchUpsert([doc], namespace);
+  }
+  async batchUpsert(docs, namespace) {
+    const payload = {
+      points: docs.map((doc) => {
+        const pointPayload = __spreadValues({
+          [this.contentField]: doc.content
+        }, namespace ? { namespace } : {});
+        if (this.isFlatPayload) {
+          Object.assign(pointPayload, doc.metadata || {});
+        } else {
+          pointPayload[this.metadataField] = doc.metadata || {};
+        }
+        return {
+          id: this.normalizeId(doc.id),
+          vector: doc.vector,
+          payload: pointPayload
+        };
+      })
+    };
+    await this.http.put(`/collections/${this.indexName}/points`, payload);
+  }
+  async query(vector, topK, namespace, _filter) {
+    const must = [];
+    if (namespace) {
+      must.push({ key: "namespace", match: { value: namespace } });
+    }
+    const sanitizedFilter = this.sanitizeFilter(_filter);
+    for (const [key, value] of Object.entries(sanitizedFilter)) {
+      let filterKey = key;
+      if (!this.isFlatPayload && !key.includes(".") && key !== "namespace" && key !== this.contentField) {
+        filterKey = `${this.metadataField}.${key}`;
+      }
+      if (Array.isArray(value)) {
+        must.push({ key: filterKey, match: { any: value } });
+      } else {
+        must.push({ key: filterKey, match: { value } });
+      }
+    }
+    const payload = {
+      vector,
+      limit: topK,
+      with_payload: true,
+      params: {
+        hnsw_ef: this.config.options.efSearch || Math.max(topK * 20, 128),
+        exact: false
+      },
+      filter: must.length > 0 ? { must } : void 0
+    };
+    const { data } = await this.http.post(`/collections/${this.indexName}/points/search`, payload);
+    const results = (data.result || []).map((res) => {
+      const p = res.payload || {};
+      let content = p[this.contentField] || "";
+      if (!content) {
+        const stringFields = Object.entries(p).filter(([_, v]) => typeof v === "string").map(([k, v]) => ({ key: k, val: v }));
+        if (stringFields.length > 0) {
+          const bestMatch = stringFields.sort((a, b) => b.val.length - a.val.length)[0];
+          content = bestMatch.val;
+        } else {
+          content = JSON.stringify(p);
+        }
+      }
+      let metadata = {};
+      if (this.isFlatPayload) {
+        metadata = __spreadValues({}, p);
+        delete metadata[this.contentField];
+        delete metadata["namespace"];
+      } else {
+        metadata = p[this.metadataField] || p;
+      }
+      return {
+        id: res.id,
+        score: res.score,
+        content,
+        metadata
+      };
+    });
+    return results;
+  }
+  async delete(id, _namespace) {
+    await this.http.post(`/collections/${this.indexName}/points/delete`, {
+      points: [this.normalizeId(id)]
+    });
+  }
+  async deleteNamespace(_namespace) {
+    await this.http.post(`/collections/${this.indexName}/points/delete`, {
+      filter: {
+        must: [{ key: "namespace", match: { value: _namespace } }]
+      }
+    });
+  }
+  async ping() {
+    try {
+      await this.http.get("/healthz");
+      return true;
+    } catch (e) {
+      return false;
+    }
+  }
+  normalizeId(id) {
+    if (typeof id === "number") return id;
+    const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+    if (uuidRegex.test(id)) return id.toLowerCase();
+    const hash = crypto.createHash("md5").update(id).digest("hex");
+    return `${hash.slice(0, 8)}-${hash.slice(8, 12)}-${hash.slice(12, 16)}-${hash.slice(16, 20)}-${hash.slice(20, 32)}`;
+  }
+  async disconnect() {
+  }
+};
+
+export {
+  QdrantProvider
+};

package/dist/{chunk-7S2SRQGL.mjs → chunk-JZ4H7EP6.mjs}

@@ -333,6 +333,53 @@ ${context}`
     });
     return (_h = (_g = (_f = completion.choices[0]) == null ? void 0 : _f.message) == null ? void 0 : _g.content) != null ? _h : "";
   }
+  chatStream(messages, context, options) {
+    return __asyncGenerator(this, null, function* () {
+      var _a, _b, _c, _d, _e, _f, _g;
+      const systemContent = (_a = this.llmConfig.systemPrompt) != null ? _a : `You are a helpful assistant. Answer questions based on the provided context.
+
+Context:
+${context}`;
+      const systemMessage = {
+        role: "system",
+        content: systemContent.includes("{{context}}") ? systemContent.replace("{{context}}", context) : `${systemContent}
+
+Context:
+${context}`
+      };
+      const formattedMessages = [
+        systemMessage,
+        ...messages.map((m) => ({
+          role: m.role,
+          content: m.content
+        }))
+      ];
+      const stream = yield new __await(this.client.chat.completions.create({
+        model: this.llmConfig.model,
+        messages: formattedMessages,
+        max_tokens: (_c = (_b = options == null ? void 0 : options.maxTokens) != null ? _b : this.llmConfig.maxTokens) != null ? _c : 1024,
+        temperature: (_e = (_d = options == null ? void 0 : options.temperature) != null ? _d : this.llmConfig.temperature) != null ? _e : 0.7,
+        stop: options == null ? void 0 : options.stop,
+        stream: true
+      }));
+      try {
+        for (var iter = __forAwait(stream), more, temp, error; more = !(temp = yield new __await(iter.next())).done; more = false) {
+          const chunk = temp.value;
+          const content = ((_g = (_f = chunk.choices[0]) == null ? void 0 : _f.delta) == null ? void 0 : _g.content) || "";
+          if (content) yield content;
+        }
+      } catch (temp) {
+        error = [temp];
+      } finally {
+        try {
+          more && (temp = iter.return) && (yield new __await(temp.call(iter)));
+        } finally {
+          if (error)
+            throw error[0];
+        }
+      }
+    });
+  }
   async embed(text, options) {
     const results = await this.batchEmbed([text], options);
     return results[0];
@@ -438,6 +485,47 @@ ${context}`;
     const block = response.content[0];
     return block.type === "text" ? block.text : "";
   }
+  chatStream(messages, context, options) {
+    return __asyncGenerator(this, null, function* () {
+      var _a, _b, _c;
+      const systemPrompt = (_a = this.llmConfig.systemPrompt) != null ? _a : `You are a helpful assistant. Use the context below to answer the user's question accurately.
+
+Context:
+${context}`;
+      const system = systemPrompt.includes("{{context}}") ? systemPrompt.replace("{{context}}", context) : `${systemPrompt}
+
+Context:
+${context}`;
+      const anthropicMessages = messages.map((m) => ({
+        role: m.role === "assistant" ? "assistant" : "user",
+        content: m.content
+      }));
+      const stream = yield new __await(this.client.messages.create({
+        model: this.llmConfig.model,
+        max_tokens: (_c = (_b = options == null ? void 0 : options.maxTokens) != null ? _b : this.llmConfig.maxTokens) != null ? _c : 1024,
+        system,
+        messages: anthropicMessages,
+        stream: true
+      }));
+      try {
+        for (var iter = __forAwait(stream), more, temp, error; more = !(temp = yield new __await(iter.next())).done; more = false) {
+          const chunk = temp.value;
+          if (chunk.type === "content_block_delta" && chunk.delta.type === "text_delta") {
+            yield chunk.delta.text;
+          }
+        }
+      } catch (temp) {
+        error = [temp];
+      } finally {
+        try {
+          more && (temp = iter.return) && (yield new __await(temp.call(iter)));
+        } finally {
+          if (error)
+            throw error[0];
+        }
+      }
+    });
+  }
   async embed(text, options) {
     void text;
     void options;
@@ -543,7 +631,7 @@ var OllamaProvider = class {
   }
   chatStream(messages, context, options) {
     return __asyncGenerator(this, null, function* () {
-      var _a, _b, _c, _d, _e;
+      var _a, _b, _c, _d, _e, _f;
       const system = this.buildSystemPrompt(context);
       const response = yield new __await(this.http.post("/api/chat", {
         model: this.llmConfig.model,
@@ -557,11 +645,15 @@ var OllamaProvider = class {
           ...messages.map((m) => ({ role: m.role, content: m.content }))
         ]
       }, { responseType: "stream" }));
+      let lineBuffer = "";
       try {
         for (var iter = __forAwait(response.data), more, temp, error; more = !(temp = yield new __await(iter.next())).done; more = false) {
           const chunk = temp.value;
-          const lines = chunk.toString().split("\n").filter(Boolean);
+          lineBuffer += chunk.toString();
+          const lines = lineBuffer.split("\n");
+          lineBuffer = lines.pop() || "";
           for (const line of lines) {
+            if (!line.trim()) continue;
             try {
               const json = JSON.parse(line);
               if ((_e = json.message) == null ? void 0 : _e.content) {
@@ -569,6 +661,7 @@ var OllamaProvider = class {
               }
               if (json.done) return;
             } catch (e) {
+              console.warn("[OllamaProvider] Failed to parse streaming line:", line);
             }
           }
         }
@@ -582,6 +675,13 @@ var OllamaProvider = class {
             throw error[0];
         }
       }
+      if (lineBuffer.trim()) {
+        try {
+          const json = JSON.parse(lineBuffer);
+          if ((_f = json.message) == null ? void 0 : _f.content) yield json.message.content;
+        } catch (e) {
+        }
+      }
     });
   }
   buildSystemPrompt(context) {
@@ -727,6 +827,49 @@ ${context}`;
     });
     return (_f = response.text) != null ? _f : "";
   }
+  chatStream(messages, context, options) {
+    return __asyncGenerator(this, null, function* () {
+      var _a, _b, _c, _d, _e;
+      const systemPrompt = (_a = this.llmConfig.systemPrompt) != null ? _a : `You are a helpful assistant. Answer questions based on the provided context.
+
+Context:
+${context}`;
+      const systemInstruction = systemPrompt.includes("{{context}}") ? systemPrompt.replace("{{context}}", context) : `${systemPrompt}
+
+Context:
+${context}`;
+      const geminiMessages = messages.map((m) => ({
+        role: m.role === "assistant" ? "model" : "user",
+        parts: [{ text: m.content }]
+      }));
+      const result = yield new __await(this.client.models.generateContentStream({
+        model: this.llmConfig.model,
+        contents: geminiMessages,
+        config: {
+          systemInstruction,
+          temperature: (_c = (_b = options == null ? void 0 : options.temperature) != null ? _b : this.llmConfig.temperature) != null ? _c : 0.7,
+          maxOutputTokens: (_e = (_d = options == null ? void 0 : options.maxTokens) != null ? _d : this.llmConfig.maxTokens) != null ? _e : 1024,
+          stopSequences: options == null ? void 0 : options.stop
+        }
+      }));
+      try {
+        for (var iter = __forAwait(result.stream), more, temp, error; more = !(temp = yield new __await(iter.next())).done; more = false) {
+          const chunk = temp.value;
+          const text = chunk.text;
+          if (text) yield text;
+        }
+      } catch (temp) {
+        error = [temp];
+      } finally {
+        try {
+          more && (temp = iter.return) && (yield new __await(temp.call(iter)));
+        } finally {
+          if (error)
+            throw error[0];
+        }
+      }
+    });
+  }
   async embed(text, options) {
     var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k;
     const model = this.sanitizeModel(
@@ -1106,7 +1249,7 @@ var ProviderRegistry = class {
         return MilvusProvider;
       }
       case "qdrant": {
-        const { QdrantProvider } = await import("./QdrantProvider-NGFXVDCF.mjs");
+        const { QdrantProvider } = await import("./QdrantProvider-NMXOULCU.mjs");
         return QdrantProvider;
       }
       case "chromadb": {
@@ -2273,6 +2416,7 @@ var Pipeline = class {
         }
         const hints = QueryProcessor.extractQueryFieldHints(question, (_f = this.config.rag) == null ? void 0 : _f.filterableFields);
         const filter = QueryProcessor.buildQueryFilter(question, hints);
+        console.log(`[Pipeline] \u{1F9E9} Extracted filters:`, JSON.stringify(filter));
         const { sources: rawSources, graphData } = yield new __await(this.retrieve(searchQuery, {
           namespace: ns,
           topK: topK * 2,