@snap-agent/rag-ecommerce 0.1.0

package/dist/index.mjs

@@ -0,0 +1,999 @@
+// src/EcommerceRAGPlugin.ts
+import { MongoClient } from "mongodb";
+import OpenAI from "openai";
+var EcommerceRAGPlugin = class {
+  constructor(config) {
+    this.name = "ecommerce-rag";
+    this.type = "rag";
+    this.db = null;
+    // Caching layers
+    this.embeddingCache = /* @__PURE__ */ new Map();
+    this.attributeCache = /* @__PURE__ */ new Map();
+    // Cache statistics
+    this.cacheStats = {
+      embeddings: { hits: 0, misses: 0 },
+      attributes: { hits: 0, misses: 0 }
+    };
+    this.config = {
+      dbName: "agentStudio",
+      collection: "products",
+      embeddingModel: "voyage-multilingual-2",
+      attributeList: [
+        "category",
+        "color",
+        "gender",
+        "brand",
+        "material",
+        "size",
+        "season",
+        "priceMin",
+        "priceMax"
+      ],
+      enableAttributeExtraction: true,
+      numCandidates: 200,
+      limit: 50,
+      vectorIndexName: "product_vector_index",
+      rescoringWeights: {
+        color: 0.15,
+        size: 0.1,
+        material: 0.1,
+        category: 0.12,
+        brand: 0.08,
+        popularity: 0.05,
+        ctr: 0.1,
+        sales: 0.1
+      },
+      enableReranking: false,
+      rerankTopK: 10,
+      contextProductCount: 8,
+      language: "es",
+      includeOutOfStock: false,
+      priority: 10,
+      ...config,
+      cache: {
+        embeddings: {
+          enabled: config.cache?.embeddings?.enabled ?? true,
+          ttl: config.cache?.embeddings?.ttl ?? 36e5,
+          // 1 hour
+          maxSize: config.cache?.embeddings?.maxSize ?? 1e3
+        },
+        attributes: {
+          enabled: config.cache?.attributes?.enabled ?? true,
+          ttl: config.cache?.attributes?.ttl ?? 18e5,
+          // 30 minutes
+          maxSize: config.cache?.attributes?.maxSize ?? 500
+        }
+      }
+    };
+    this.priority = this.config.priority;
+    this.client = new MongoClient(this.config.mongoUri);
+    this.openai = new OpenAI({ apiKey: this.config.openaiApiKey });
+    this.startCacheCleanup();
+  }
+  async ensureConnection() {
+    if (!this.db) {
+      await this.client.connect();
+      this.db = this.client.db(this.config.dbName);
+    }
+    return this.db;
+  }
+  /**
+   * Main retrieval method - called by the SDK
+   */
+  async retrieveContext(message, options) {
+    const queryVector = await this.embedText(message);
+    let attributes = {};
+    if (this.config.enableAttributeExtraction) {
+      attributes = await this.extractAttributes(message);
+    }
+    const searchResults = await this.vectorSearch({
+      queryVector,
+      agentId: options.agentId,
+      hardFilters: options.filters || {}
+    });
+    const rescored = this.softRescore(searchResults, attributes);
+    let final = rescored;
+    if (this.config.enableReranking) {
+      final = await this.rerank(message, rescored);
+    }
+    if (!this.config.includeOutOfStock) {
+      final = final.filter((p) => p.inStock !== false);
+    }
+    return {
+      content: this.buildContextString(final),
+      sources: final.slice(0, this.config.contextProductCount).map((p) => ({
+        id: p.sku,
+        title: p.title,
+        score: p.vectorSearchScore,
+        type: "product",
+        attributes: p.attributes,
+        inStock: p.inStock
+      })),
+      metadata: {
+        productCount: final.length,
+        extractedAttributes: attributes,
+        topProducts: final.slice(0, 3).map((p) => ({
+          sku: p.sku,
+          title: p.title,
+          score: p.vectorSearchScore
+        }))
+      }
+    };
+  }
+  /**
+   * Format context for LLM
+   */
+  formatContext(context) {
+    return context.content;
+  }
+  // ============================================================================
+  // Private Methods
+  // ============================================================================
+  /**
+   * Embed text using Voyage with caching
+   */
+  async embedText(text) {
+    if (this.config.cache?.embeddings?.enabled) {
+      const cacheKey = `${this.config.embeddingModel}:${text}`;
+      const cached = this.embeddingCache.get(cacheKey);
+      if (cached) {
+        const age = Date.now() - cached.timestamp;
+        if (age < (this.config.cache?.embeddings?.ttl ?? 36e5)) {
+          this.cacheStats.embeddings.hits++;
+          return cached.value;
+        } else {
+          this.embeddingCache.delete(cacheKey);
+        }
+      }
+      this.cacheStats.embeddings.misses++;
+    }
+    const response = await fetch("https://api.voyageai.com/v1/embeddings", {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        "Authorization": `Bearer ${this.config.voyageApiKey}`
+      },
+      body: JSON.stringify({
+        input: text,
+        model: this.config.embeddingModel
+      })
+    });
+    if (!response.ok) {
+      throw new Error(`Voyage API error: ${response.statusText}`);
+    }
+    const data = await response.json();
+    const embedding = data.data[0].embedding;
+    if (this.config.cache?.embeddings?.enabled) {
+      const cacheKey = `${this.config.embeddingModel}:${text}`;
+      const maxSize = this.config.cache?.embeddings?.maxSize ?? 1e3;
+      if (this.embeddingCache.size >= maxSize) {
+        const firstKey = this.embeddingCache.keys().next().value;
+        if (firstKey) {
+          this.embeddingCache.delete(firstKey);
+        }
+      }
+      this.embeddingCache.set(cacheKey, {
+        value: embedding,
+        timestamp: Date.now()
+      });
+    }
+    return embedding;
+  }
+  /**
+   * Extract attributes from user message using OpenAI with caching
+   */
+  async extractAttributes(message) {
+    if (this.config.cache?.attributes?.enabled) {
+      const cacheKey = message.toLowerCase().trim();
+      const cached = this.attributeCache.get(cacheKey);
+      if (cached) {
+        const age = Date.now() - cached.timestamp;
+        if (age < (this.config.cache?.attributes?.ttl ?? 18e5)) {
+          this.cacheStats.attributes.hits++;
+          return cached.value;
+        } else {
+          this.attributeCache.delete(cacheKey);
+        }
+      }
+      this.cacheStats.attributes.misses++;
+    }
+    let attrs = {};
+    try {
+      const completion = await this.openai.chat.completions.create({
+        model: "gpt-4o-mini",
+        messages: [
+          {
+            role: "system",
+            content: `Extract product attributes from the user message. Return a JSON object with only the attributes you can identify from this list: ${this.config.attributeList.join(", ")}. If an attribute is not mentioned, omit it from the response.`
+          },
+          {
+            role: "user",
+            content: message
+          }
+        ],
+        response_format: { type: "json_object" },
+        temperature: 0.1
+      });
+      const extracted = JSON.parse(completion.choices[0]?.message?.content || "{}");
+      if (extracted.category) attrs.category = String(extracted.category);
+      if (extracted.color) attrs.color = String(extracted.color);
+      if (extracted.gender) {
+        const g = String(extracted.gender).toLowerCase();
+        if (g.includes("hombre") || g.includes("male") || g.includes("man") || g === "m") {
+          attrs.gender = "M";
+        } else if (g.includes("mujer") || g.includes("female") || g.includes("woman") || g === "f") {
+          attrs.gender = "F";
+        } else if (g.includes("unisex")) {
+          attrs.gender = "Unisex";
+        }
+      }
+      if (extracted.brand) attrs.brand = String(extracted.brand);
+      if (extracted.material) attrs.material = String(extracted.material);
+      if (extracted.size) attrs.size = String(extracted.size);
+      if (extracted.season) attrs.season = String(extracted.season);
+      if (extracted.priceMin) attrs.priceMin = Number(extracted.priceMin);
+      if (extracted.priceMax) attrs.priceMax = Number(extracted.priceMax);
+    } catch (error) {
+      console.error("Attribute extraction failed:", error);
+      return {};
+    }
+    if (this.config.cache?.attributes?.enabled) {
+      const cacheKey = message.toLowerCase().trim();
+      const maxSize = this.config.cache?.attributes?.maxSize ?? 500;
+      if (this.attributeCache.size >= maxSize) {
+        const firstKey = this.attributeCache.keys().next().value;
+        if (firstKey) {
+          this.attributeCache.delete(firstKey);
+        }
+      }
+      this.attributeCache.set(cacheKey, {
+        value: attrs,
+        timestamp: Date.now()
+      });
+    }
+    return attrs;
+  }
+  /**
+   * MongoDB Atlas Vector Search
+   */
+  async vectorSearch(options) {
+    const db = await this.ensureConnection();
+    const collection = db.collection(this.config.collection);
+    const filter = { tenantId: this.config.tenantId };
+    if (options.agentId) {
+      filter.agentId = options.agentId;
+    }
+    Object.entries(options.hardFilters).forEach(([key, value]) => {
+      if (value !== void 0 && value !== null) {
+        filter[key] = value;
+      }
+    });
+    const pipeline = [
+      {
+        $vectorSearch: {
+          index: this.config.vectorIndexName,
+          path: "embedding",
+          queryVector: options.queryVector,
+          numCandidates: this.config.numCandidates,
+          limit: this.config.limit,
+          filter
+        }
+      },
+      {
+        $addFields: {
+          vectorSearchScore: { $meta: "vectorSearchScore" }
+        }
+      }
+    ];
+    const results = await collection.aggregate(pipeline).toArray();
+    return results;
+  }
+  /**
+   * Soft rescore based on attributes and metrics
+   */
+  softRescore(results, attrs) {
+    const weights = this.config.rescoringWeights;
+    return results.map((product) => {
+      let boost = product.vectorSearchScore || 0;
+      if (attrs.color && product.attributes.color) {
+        const match = product.attributes.color.toLowerCase() === attrs.color.toLowerCase();
+        if (match) boost += weights.color;
+      }
+      if (attrs.size && product.attributes.size) {
+        const match = product.attributes.size.some(
+          (s) => s.toLowerCase() === attrs.size?.toLowerCase()
+        );
+        if (match) boost += weights.size;
+      }
+      if (attrs.material && product.attributes.material) {
+        const match = product.attributes.material.toLowerCase() === attrs.material.toLowerCase();
+        if (match) boost += weights.material;
+      }
+      if (attrs.category && product.attributes.category) {
+        const match = product.attributes.category.toLowerCase() === attrs.category.toLowerCase();
+        if (match) boost += weights.category;
+      }
+      if (attrs.brand && product.attributes.brand) {
+        const match = product.attributes.brand.toLowerCase() === attrs.brand.toLowerCase();
+        if (match) boost += weights.brand;
+      }
+      if (attrs.priceMax && product.attributes.price) {
+        const withinBudget = product.attributes.price <= attrs.priceMax;
+        if (withinBudget) {
+          const proximity = 1 - product.attributes.price / attrs.priceMax;
+          boost += Math.max(0, proximity * 0.1);
+        }
+      }
+      if (product.metrics?.popularity) {
+        boost += Math.min(product.metrics.popularity * weights.popularity, 0.2);
+      }
+      if (product.metrics?.ctr) {
+        boost += Math.min(product.metrics.ctr * weights.ctr, 0.15);
+      }
+      if (product.metrics?.sales) {
+        const normalizedSales = Math.log10(product.metrics.sales + 1) / 10;
+        boost += Math.min(normalizedSales * weights.sales, 0.1);
+      }
+      return { ...product, vectorSearchScore: boost };
+    }).sort((a, b) => (b.vectorSearchScore || 0) - (a.vectorSearchScore || 0));
+  }
+  /**
+   * Optional Voyage reranking
+   */
+  async rerank(query, products) {
+    if (products.length === 0) return products;
+    try {
+      const docTexts = products.map(
+        (p) => `${p.title}. ${p.description || ""}. ${Object.entries(p.attributes).filter(([_, v]) => v).map(([k, v]) => `${k}: ${v}`).join(", ")}`
+      );
+      const response = await fetch("https://api.voyageai.com/v1/rerank", {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          "Authorization": `Bearer ${this.config.voyageApiKey}`
+        },
+        body: JSON.stringify({
+          query,
+          documents: docTexts,
+          model: "rerank-2",
+          top_k: this.config.rerankTopK
+        })
+      });
+      if (!response.ok) {
+        throw new Error(`Voyage rerank error: ${response.statusText}`);
+      }
+      const data = await response.json();
+      const scores = data.data.map((item) => item.relevance_score);
+      const reranked = products.map((doc, idx) => ({
+        ...doc,
+        vectorSearchScore: (doc.vectorSearchScore || 0) * 0.5 + (scores[idx] || 0) * 0.5
+      }));
+      return reranked.sort((a, b) => (b.vectorSearchScore || 0) - (a.vectorSearchScore || 0)).slice(0, this.config.rerankTopK);
+    } catch (error) {
+      console.error("Reranking failed:", error);
+      return products;
+    }
+  }
+  /**
+   * Build context string for LLM
+   */
+  buildContextString(products) {
+    const limited = products.slice(0, this.config.contextProductCount);
+    if (limited.length === 0) {
+      return this.config.language === "es" ? "No se encontraron productos en el cat\xE1logo." : "No products found in the catalog.";
+    }
+    const productBlocks = limited.map((product, idx) => {
+      const attrs = [];
+      if (product.attributes.category) attrs.push(`Category: ${product.attributes.category}`);
+      if (product.attributes.brand) attrs.push(`Brand: ${product.attributes.brand}`);
+      if (product.attributes.color) attrs.push(`Color: ${product.attributes.color}`);
+      if (product.attributes.material) attrs.push(`Material: ${product.attributes.material}`);
+      if (product.attributes.size?.length) {
+        attrs.push(`Sizes: ${product.attributes.size.join(", ")}`);
+      }
+      if (product.attributes.price !== void 0) {
+        attrs.push(`Price: $${product.attributes.price.toFixed(2)}`);
+      }
+      if (product.inStock !== void 0) {
+        attrs.push(product.inStock ? "In Stock" : "Out of Stock");
+      }
+      return `${idx + 1}. ${product.title}
+   SKU: ${product.sku}
+   ${product.description || ""}
+   ${attrs.join(" | ")}`;
+    });
+    const header = this.config.language === "es" ? "PRODUCTOS DISPONIBLES EN EL CAT\xC1LOGO:" : "AVAILABLE PRODUCTS IN CATALOG:";
+    return `${header}
+
+${productBlocks.join("\n\n")}`;
+  }
+  // ============================================================================
+  // Cache Management
+  // ============================================================================
+  /**
+   * Start periodic cache cleanup (remove expired entries)
+   */
+  startCacheCleanup() {
+    this.cleanupInterval = setInterval(() => {
+      this.cleanupExpiredCache();
+    }, 3e5);
+  }
+  /**
+   * Clean up expired cache entries
+   */
+  cleanupExpiredCache() {
+    const now = Date.now();
+    if (this.config.cache?.embeddings?.enabled) {
+      const ttl = this.config.cache?.embeddings?.ttl ?? 36e5;
+      for (const [key, entry] of this.embeddingCache.entries()) {
+        if (now - entry.timestamp >= ttl) {
+          this.embeddingCache.delete(key);
+        }
+      }
+    }
+    if (this.config.cache?.attributes?.enabled) {
+      const ttl = this.config.cache?.attributes?.ttl ?? 18e5;
+      for (const [key, entry] of this.attributeCache.entries()) {
+        if (now - entry.timestamp >= ttl) {
+          this.attributeCache.delete(key);
+        }
+      }
+    }
+  }
+  /**
+   * Get cache statistics
+   */
+  getCacheStats() {
+    return {
+      embeddings: {
+        size: this.embeddingCache.size,
+        maxSize: this.config.cache?.embeddings?.maxSize ?? 1e3,
+        hits: this.cacheStats.embeddings.hits,
+        misses: this.cacheStats.embeddings.misses,
+        hitRate: this.cacheStats.embeddings.hits + this.cacheStats.embeddings.misses > 0 ? (this.cacheStats.embeddings.hits / (this.cacheStats.embeddings.hits + this.cacheStats.embeddings.misses)).toFixed(2) : "0.00"
+      },
+      attributes: {
+        size: this.attributeCache.size,
+        maxSize: this.config.cache?.attributes?.maxSize ?? 500,
+        hits: this.cacheStats.attributes.hits,
+        misses: this.cacheStats.attributes.misses,
+        hitRate: this.cacheStats.attributes.hits + this.cacheStats.attributes.misses > 0 ? (this.cacheStats.attributes.hits / (this.cacheStats.attributes.hits + this.cacheStats.attributes.misses)).toFixed(2) : "0.00"
+      }
+    };
+  }
+  /**
+   * Clear all caches
+   */
+  clearCache() {
+    this.embeddingCache.clear();
+    this.attributeCache.clear();
+    this.cacheStats = {
+      embeddings: { hits: 0, misses: 0 },
+      attributes: { hits: 0, misses: 0 }
+    };
+  }
+  /**
+   * Get MongoDB collection
+   */
+  async getCollection() {
+    const db = await this.ensureConnection();
+    return db.collection(this.config.collection);
+  }
+  /**
+   * Generate embedding for a single text
+   */
+  async generateEmbedding(text) {
+    return this.embedText(text);
+  }
+  /**
+   * Ingest products into the RAG system
+   * Converts RAGDocuments to ProductDocs and indexes them with embeddings
+   */
+  async ingest(documents, options) {
+    const collection = await this.getCollection();
+    let indexed = 0;
+    let failed = 0;
+    const errors = [];
+    try {
+      const batchSize = options?.batchSize || 10;
+      for (let i = 0; i < documents.length; i += batchSize) {
+        const batch = documents.slice(i, i + batchSize);
+        const textsForEmbedding = batch.map((doc) => {
+          const metadata = doc.metadata || {};
+          const attributeParts = [];
+          const attributeFields = ["category", "brand", "color", "material", "size", "gender", "season"];
+          for (const field of attributeFields) {
+            if (metadata[field]) {
+              const value = Array.isArray(metadata[field]) ? metadata[field].join(", ") : metadata[field];
+              attributeParts.push(`${field}: ${value}`);
+            }
+          }
+          return attributeParts.length > 0 ? `${doc.content}. ${attributeParts.join(", ")}` : doc.content;
+        });
+        const embeddings = await this.generateEmbeddingsBatch(textsForEmbedding);
+        const productDocs = batch.map((doc, idx) => {
+          const metadata = doc.metadata || {};
+          return {
+            tenantId: this.config.tenantId,
+            agentId: options?.agentId,
+            sku: doc.id,
+            title: metadata.title || doc.content.substring(0, 100),
+            description: metadata.description || doc.content,
+            embedding: embeddings[idx],
+            attributes: {
+              category: metadata.category,
+              brand: metadata.brand,
+              color: metadata.color,
+              material: metadata.material,
+              size: metadata.size,
+              gender: metadata.gender,
+              season: metadata.season,
+              price: metadata.price,
+              ...metadata.attributes
+            },
+            inStock: metadata.inStock !== false,
+            metrics: metadata.metrics || {}
+          };
+        });
+        try {
+          if (options?.overwrite) {
+            const bulkOps = productDocs.map((doc) => ({
+              replaceOne: {
+                filter: {
+                  tenantId: this.config.tenantId,
+                  sku: doc.sku,
+                  ...options.agentId ? { agentId: options.agentId } : {}
+                },
+                replacement: doc,
+                upsert: true
+              }
+            }));
+            const result = await collection.bulkWrite(bulkOps);
+            indexed += result.upsertedCount + result.modifiedCount;
+          } else if (options?.skipExisting) {
+            const existingSkus = await collection.find({
+              tenantId: this.config.tenantId,
+              sku: { $in: productDocs.map((d) => d.sku) },
+              ...options.agentId ? { agentId: options.agentId } : {}
+            }).project({ sku: 1 }).toArray();
+            const existingSet = new Set(existingSkus.map((d) => d.sku));
+            const newDocs = productDocs.filter((d) => !existingSet.has(d.sku));
+            if (newDocs.length > 0) {
+              const result = await collection.insertMany(newDocs);
+              indexed += result.insertedCount;
+            }
+            failed += productDocs.length - newDocs.length;
+          } else {
+            const bulkOps = productDocs.map((doc) => ({
+              updateOne: {
+                filter: {
+                  tenantId: this.config.tenantId,
+                  sku: doc.sku,
+                  ...options?.agentId ? { agentId: options.agentId } : {}
+                },
+                update: { $set: doc },
+                upsert: true
+              }
+            }));
+            const result = await collection.bulkWrite(bulkOps);
+            indexed += result.upsertedCount + result.modifiedCount;
+          }
+        } catch (error) {
+          batch.forEach((doc) => {
+            failed++;
+            errors.push({
+              id: doc.id,
+              error: error.message || "Unknown error during insertion"
+            });
+          });
+        }
+      }
+      return {
+        success: failed === 0,
+        indexed,
+        failed,
+        errors: errors.length > 0 ? errors : void 0,
+        metadata: {
+          batchSize: options?.batchSize || 10,
+          totalDocuments: documents.length
+        }
+      };
+    } catch (error) {
+      return {
+        success: false,
+        indexed,
+        failed: documents.length - indexed,
+        errors: [{
+          id: "batch",
+          error: error.message || "Unknown error during ingestion"
+        }]
+      };
+    }
+  }
+  /**
+   * Update a single product
+   */
+  async update(id, document, options) {
+    const collection = await this.getCollection();
+    const update = {};
+    const metadata = document.metadata || {};
+    if (document.content) {
+      const attributeParts = [];
+      const attributeFields = ["category", "brand", "color", "material", "size", "gender", "season"];
+      for (const field of attributeFields) {
+        if (metadata[field]) {
+          const value = Array.isArray(metadata[field]) ? metadata[field].join(", ") : metadata[field];
+          attributeParts.push(`${field}: ${value}`);
+        }
+      }
+      const textForEmbedding = attributeParts.length > 0 ? `${document.content}. ${attributeParts.join(", ")}` : document.content;
+      const embedding = await this.generateEmbedding(textForEmbedding);
+      update.embedding = embedding;
+      update.description = document.content;
+    }
+    if (document.metadata) {
+      if (metadata.title) update.title = metadata.title;
+      if (metadata.inStock !== void 0) update.inStock = metadata.inStock;
+      const attributeUpdates = {};
+      const metricUpdates = {};
+      const attributeFields = ["category", "brand", "color", "material", "size", "gender", "season", "price"];
+      attributeFields.forEach((field) => {
+        if (metadata[field] !== void 0) {
+          attributeUpdates[`attributes.${field}`] = metadata[field];
+        }
+      });
+      if (metadata.metrics) {
+        Object.entries(metadata.metrics).forEach(([key, value]) => {
+          metricUpdates[`metrics.${key}`] = value;
+        });
+      }
+      Object.assign(update, attributeUpdates, metricUpdates);
+      if (metadata.attributes) {
+        Object.entries(metadata.attributes).forEach(([key, value]) => {
+          update[`attributes.${key}`] = value;
+        });
+      }
+    }
+    await collection.updateOne(
+      {
+        tenantId: this.config.tenantId,
+        sku: id,
+        ...options?.agentId ? { agentId: options.agentId } : {}
+      },
+      { $set: update }
+    );
+  }
+  /**
+   * Delete product(s) by SKU
+   */
+  async delete(ids, options) {
+    const collection = await this.getCollection();
+    const skuArray = Array.isArray(ids) ? ids : [ids];
+    const result = await collection.deleteMany({
+      tenantId: this.config.tenantId,
+      sku: { $in: skuArray },
+      ...options?.agentId ? { agentId: options.agentId } : {}
+    });
+    return result.deletedCount;
+  }
+  /**
+   * Bulk operations for efficient batch processing
+   */
+  async bulk(operations, options) {
+    let inserted = 0;
+    let updated = 0;
+    let deleted = 0;
+    let failed = 0;
+    const errors = [];
+    for (const op of operations) {
+      try {
+        switch (op.type) {
+          case "insert":
+            if (op.document) {
+              await this.ingest([op.document], options);
+              inserted++;
+            }
+            break;
+          case "update":
+            if (op.document) {
+              await this.update(op.id, op.document, options);
+              updated++;
+            }
+            break;
+          case "delete":
+            const count = await this.delete(op.id, options);
+            deleted += count;
+            break;
+        }
+      } catch (error) {
+        failed++;
+        errors.push({
+          id: op.id,
+          operation: op.type,
+          error: error.message || "Unknown error"
+        });
+      }
+    }
+    return {
+      success: failed === 0,
+      inserted,
+      updated,
+      deleted,
+      failed,
+      errors: errors.length > 0 ? errors : void 0
+    };
+  }
+  /**
+   * Generate embeddings for a batch of texts
+   */
+  async generateEmbeddingsBatch(texts) {
+    const embeddings = [];
+    const cacheConfig = this.config.cache?.embeddings;
+    for (const text of texts) {
+      if (cacheConfig?.enabled) {
+        const cached = this.embeddingCache.get(text);
+        const ttl = cacheConfig.ttl ?? 36e5;
+        if (cached && Date.now() - cached.timestamp < ttl) {
+          embeddings.push(cached.value);
+          this.cacheStats.embeddings.hits++;
+          continue;
+        }
+      }
+      this.cacheStats.embeddings.misses++;
+      const embedding = await this.generateEmbedding(text);
+      embeddings.push(embedding);
+      if (cacheConfig?.enabled) {
+        this.embeddingCache.set(text, {
+          value: embedding,
+          timestamp: Date.now()
+        });
+      }
+    }
+    return embeddings;
+  }
+  /**
+   * Ingest documents from URL source (CSV, JSON, XML, API)
+   */
+  async ingestFromUrl(source, options) {
+    try {
+      const controller = new AbortController();
+      const timeoutId = setTimeout(() => controller.abort(), source.timeout || 3e4);
+      const response = await fetch(source.url, {
+        headers: {
+          ...source.headers,
+          ...source.auth && this.buildAuthHeaders(source.auth)
+        },
+        signal: controller.signal
+      });
+      clearTimeout(timeoutId);
+      if (!response.ok) {
+        throw new Error(`HTTP error: ${response.status} ${response.statusText}`);
+      }
+      let documents;
+      if (source.type === "json" || source.type === "api") {
+        const data = await response.json();
+        documents = this.transformJsonToDocuments(data, source.transform);
+      } else if (source.type === "csv") {
+        const data = await response.text();
+        documents = await this.transformCsvToDocuments(data, source.transform);
+      } else if (source.type === "xml") {
+        const data = await response.text();
+        documents = await this.transformXmlToDocuments(data, source.transform);
+      } else {
+        throw new Error(`Unsupported source type: ${source.type}`);
+      }
+      documents = documents.map((doc) => ({
+        ...doc,
+        metadata: {
+          ...doc.metadata,
+          ...source.metadata,
+          sourceUrl: source.url,
+          fetchedAt: (/* @__PURE__ */ new Date()).toISOString()
+        }
+      }));
+      const ingestResult = await this.ingest(documents, options);
+      return {
+        ...ingestResult,
+        sourceUrl: source.url,
+        fetchedAt: /* @__PURE__ */ new Date(),
+        documentsFetched: documents.length
+      };
+    } catch (error) {
+      console.error("URL ingestion failed:", error);
+      return {
+        success: false,
+        indexed: 0,
+        failed: 0,
+        sourceUrl: source.url,
+        fetchedAt: /* @__PURE__ */ new Date(),
+        documentsFetched: 0,
+        errors: [{
+          id: "fetch",
+          error: error instanceof Error ? error.message : "Unknown error"
+        }]
+      };
+    }
+  }
+  /**
+   * Handle webhook payload for real-time updates
+   */
+  async handleWebhook(payload, source, options) {
+    try {
+      let documents = [];
+      if (source === "shopify") {
+        documents = this.parseShopifyWebhook(payload);
+      } else if (source === "woocommerce") {
+        documents = this.parseWooCommerceWebhook(payload);
+      } else if (source === "custom") {
+        documents = Array.isArray(payload) ? payload : [payload];
+      } else {
+        throw new Error(`Unsupported webhook source: ${source}`);
+      }
+      documents = documents.map((doc) => ({
+        ...doc,
+        metadata: {
+          ...doc.metadata,
+          webhookSource: source,
+          receivedAt: (/* @__PURE__ */ new Date()).toISOString()
+        }
+      }));
+      return await this.ingest(documents, { ...options, overwrite: true });
+    } catch (error) {
+      console.error("Webhook handling failed:", error);
+      return {
+        success: false,
+        indexed: 0,
+        failed: 1,
+        errors: [{
+          id: "webhook",
+          error: error instanceof Error ? error.message : "Unknown error"
+        }]
+      };
+    }
+  }
+  // ============================================================================
+  // Private Helper Methods for URL Ingestion
+  // ============================================================================
+  buildAuthHeaders(auth) {
+    if (!auth) return {};
+    if (auth.type === "bearer" && auth.token) {
+      return { Authorization: `Bearer ${auth.token}` };
+    } else if (auth.type === "basic" && auth.username && auth.password) {
+      const encoded = Buffer.from(`${auth.username}:${auth.password}`).toString("base64");
+      return { Authorization: `Basic ${encoded}` };
+    } else if (auth.type === "api-key" && auth.header && auth.key) {
+      return { [auth.header]: auth.key };
+    } else if (auth.type === "custom" && auth.headers) {
+      return auth.headers;
+    }
+    return {};
+  }
+  transformJsonToDocuments(data, transform) {
+    let items = data;
+    if (transform?.documentPath) {
+      items = this.extractByPath(data, transform.documentPath);
+    }
+    if (!Array.isArray(items)) {
+      items = [items];
+    }
+    return items.map((item, index) => {
+      const fieldMapping = transform?.fieldMapping || {};
+      const metadata = {};
+      for (const [key, path] of Object.entries(fieldMapping)) {
+        if (key !== "id" && key !== "content" && path && typeof path === "string") {
+          metadata[key] = this.extractField(item, path);
+        }
+      }
+      return {
+        id: this.extractField(item, fieldMapping.id || "id") || `doc-${index}`,
+        content: this.extractField(item, fieldMapping.content || "content") || JSON.stringify(item),
+        metadata
+      };
+    });
+  }
+  async transformCsvToDocuments(csvData, transform) {
+    const lines = csvData.trim().split("\n");
+    const headers = lines[0].split(",").map((h) => h.trim());
+    return lines.slice(1).map((line, index) => {
+      const values = line.split(",").map((v) => v.trim());
+      const item = headers.reduce((acc, header, i) => {
+        acc[header] = values[i];
+        return acc;
+      }, {});
+      const fieldMapping = transform?.fieldMapping || {};
+      const metadata = {};
+      for (const [key, path] of Object.entries(fieldMapping)) {
+        if (key !== "id" && key !== "content" && path && typeof path === "string") {
+          metadata[key] = this.extractField(item, path);
+        }
+      }
+      return {
+        id: this.extractField(item, fieldMapping.id || "id") || `doc-${index}`,
+        content: this.extractField(item, fieldMapping.content || "content") || JSON.stringify(item),
+        metadata
+      };
+    });
+  }
+  async transformXmlToDocuments(_xmlData, _transform) {
+    throw new Error("XML parsing not yet implemented. Please use JSON or CSV format.");
+  }
+  extractByPath(data, path) {
+    if (path.startsWith("$.")) {
+      const parts = path.slice(2).split(".");
+      let current = data;
+      for (const part of parts) {
+        if (part.endsWith("[*]")) {
+          const key = part.slice(0, -3);
+          current = current[key];
+          if (!Array.isArray(current)) {
+            throw new Error(`Path ${path} does not resolve to an array`);
+          }
+          return current;
+        } else {
+          current = current[part];
+        }
+      }
+      return current;
+    }
+    return data;
+  }
+  extractField(item, path) {
+    const parts = path.split(".");
+    let current = item;
+    for (const part of parts) {
+      if (part.includes("[")) {
+        const [key, index] = part.split("[");
+        const idx = parseInt(index.replace("]", ""));
+        current = current[key]?.[idx];
+      } else {
+        current = current[part];
+      }
+      if (current === void 0) return void 0;
+    }
+    return current;
+  }
+  parseShopifyWebhook(payload) {
+    return [{
+      id: payload.id?.toString() || payload.handle,
+      content: `${payload.title}
+${payload.body_html || ""}`,
+      metadata: {
+        title: payload.title,
+        price: payload.variants?.[0]?.price,
+        sku: payload.variants?.[0]?.sku,
+        inStock: (payload.variants?.[0]?.inventory_quantity || 0) > 0,
+        vendor: payload.vendor,
+        product_type: payload.product_type,
+        tags: payload.tags
+      }
+    }];
+  }
+  parseWooCommerceWebhook(payload) {
+    return [{
+      id: payload.id?.toString() || payload.sku,
+      content: `${payload.name}
+${payload.description || ""}`,
+      metadata: {
+        title: payload.name,
+        price: payload.price,
+        sku: payload.sku,
+        inStock: payload.stock_status === "instock",
+        categories: payload.categories?.map((c) => c.name)
+      }
+    }];
+  }
+  /**
+   * Cleanup resources and close connections
+   */
+  async disconnect() {
+    if (this.cleanupInterval) {
+      clearInterval(this.cleanupInterval);
+      this.cleanupInterval = void 0;
+    }
+    await this.client.close();
+    this.db = null;
+  }
+};
+export {
+  EcommerceRAGPlugin
+};