@retrivora-ai/rag-engine 0.1.5 → 0.1.6

package/dist/server.mjs

@@ -1,6 +1,6 @@
 import {
   QdrantProvider
-} from "./chunk-6FODXNUF.mjs";
+} from "./chunk-5K23H7JL.mjs";
 import {
   ChromaDBProvider
 } from "./chunk-HUGLYKD6.mjs";
@@ -21,7 +21,7 @@ import {
   createIngestHandler,
   createUploadHandler,
   getRagConfig
-} from "./chunk-BP4U4TT5.mjs";
+} from "./chunk-HSBXE2WV.mjs";
 import {
   AnthropicProvider,
   LLMFactory,
@@ -31,10 +31,10 @@ import {
 import "./chunk-UKDXCXW7.mjs";
 import {
   PineconeProvider
-} from "./chunk-AALIF3AL.mjs";
+} from "./chunk-ZM6TYIDH.mjs";
 import {
   PostgreSQLProvider
-} from "./chunk-S5DRHETN.mjs";
+} from "./chunk-IUTAZ7QR.mjs";
 import {
   MongoDBProvider
 } from "./chunk-QEYVWVT5.mjs";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@retrivora-ai/rag-engine",
-  "version": "0.1.5",
+  "version": "0.1.6",
   "description": "Retrivora AI is a plug-and-play AI engine for RAG chat experiences — generic vector DB + LLM provider, embeddable or standalone.",
   "author": "Abhinav Alkuchi",
   "license": "MIT",
@@ -98,6 +98,7 @@
     "@types/pg": "^8.20.0",
     "@types/react": "^19.2.14",
     "@types/react-dom": "^19.2.3",
+    "dotenv": "^17.4.2",
     "eslint": "^9",
     "eslint-config-next": "16.2.4",
     "tailwindcss": "^4",

package/src/config/serverConfig.ts

@@ -73,6 +73,9 @@ export function getRagConfig(env: Record<string, string | undefined> = process.e
     vectorDbOptions.database = readString(env, 'MONGODB_DB') ?? '';
     vectorDbOptions.collection = readString(env, 'MONGODB_COLLECTION') ?? '';
     vectorDbOptions.indexName = readString(env, 'MONGODB_INDEX_NAME') ?? 'vector_index';
+  } else if (vectorProvider === 'qdrant') {
+    vectorDbOptions.baseUrl = readString(env, 'QDRANT_URL') ?? 'http://localhost:6333';
+    vectorDbOptions.apiKey = readString(env, 'QDRANT_API_KEY');
   }
 
   const llmApiKeyByProvider: Record<string, string | undefined> = {

package/src/core/BatchProcessor.ts

@@ -0,0 +1,347 @@
+/**
+ * BatchProcessor.ts — Handles batch operations with retry logic.
+ *
+ * Provides exponential backoff, partial failure tracking, and
+ * configurable batch sizing for efficient bulk operations.
+ */
+
+export interface BatchOptions {
+  /** Maximum number of items per batch */
+  batchSize?: number;
+  /** Maximum number of retries for transient failures */
+  maxRetries?: number;
+  /** Initial delay before retry in milliseconds */
+  initialDelayMs?: number;
+  /** Maximum delay between retries in milliseconds */
+  maxDelayMs?: number;
+  /** Multiplier for exponential backoff */
+  backoffMultiplier?: number;
+  /** Whether to throw on any partial failure */
+  throwOnPartialFailure?: boolean;
+}
+
+export interface BatchResult<T> {
+  results: T[];
+  errors: Array<{ index: number; error: Error; itemCount?: number }>;
+  totalProcessed: number;
+  totalFailed: number;
+}
+
+/**
+ * Determines if an error is transient (should retry)
+ */
+function isTransientError(error: unknown): boolean {
+  if (!(error instanceof Error)) return false;
+
+  const message = error.message.toLowerCase();
+
+  // Network errors
+  if (
+    message.includes('econnrefused') ||
+    message.includes('econnreset') ||
+    message.includes('timeout') ||
+    message.includes('network') ||
+    message.includes('socket')
+  ) {
+    return true;
+  }
+
+  // HTTP errors (rate limit, service unavailable)
+  if (
+    message.includes('429') ||
+    message.includes('503') ||
+    message.includes('502') ||
+    message.includes('rate limit') ||
+    message.includes('too many requests')
+  ) {
+    return true;
+  }
+
+  return false;
+}
+
+/**
+ * Calculates delay using exponential backoff with jitter
+ */
+function calculateBackoffDelay(
+  attempt: number,
+  initialDelayMs: number,
+  maxDelayMs: number,
+  multiplier: number
+): number {
+  const exponentialDelay = Math.min(
+    initialDelayMs * Math.pow(multiplier, attempt),
+    maxDelayMs
+  );
+  // Add jitter: ±10% of the delay
+  const jitter = exponentialDelay * 0.1 * (Math.random() * 2 - 1);
+  return Math.max(0, exponentialDelay + jitter);
+}
+
+/**
+ * Sleeps for the given duration
+ */
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+export class BatchProcessor {
+  /**
+   * Processes an array of items in configurable batches with retry logic.
+   *
+   * @param items - Items to process
+   * @param processor - Async function that processes a batch of items
+   * @param options - Configuration for batch size, retries, etc.
+   * @returns Object with results, errors, and statistics
+   *
+   * @example
+   * const docs = [...];
+   * const result = await BatchProcessor.processBatch(
+   *   docs,
+   *   (batch) => vectorDB.batchUpsert(batch),
+   *   { batchSize: 100, maxRetries: 3 }
+   * );
+   */
+  static async processBatch<T, R>(
+    items: T[],
+    processor: (batch: T[]) => Promise<R>,
+    options?: BatchOptions
+  ): Promise<BatchResult<R>> {
+    const {
+      batchSize = 100,
+      maxRetries = 3,
+      initialDelayMs = 100,
+      maxDelayMs = 10000,
+      backoffMultiplier = 2,
+      throwOnPartialFailure = false,
+    } = options ?? {};
+
+    const results: R[] = [];
+    const errors: Array<{ index: number; error: Error; itemCount?: number }> = [];
+
+    // Split items into batches
+    const batches: T[][] = [];
+    for (let i = 0; i < items.length; i += batchSize) {
+      batches.push(items.slice(i, i + batchSize));
+    }
+
+    if (batches.length === 0) {
+      return { results: [], errors: [], totalProcessed: 0, totalFailed: 0 };
+    }
+
+    // Process each batch with retry logic
+    for (let batchIndex = 0; batchIndex < batches.length; batchIndex++) {
+      const batch = batches[batchIndex];
+      let success = false;
+      let lastError: Error | undefined;
+
+      for (let attempt = 0; attempt <= maxRetries; attempt++) {
+        try {
+          const result = await processor(batch);
+          results.push(result);
+          success = true;
+          break;
+        } catch (err) {
+          lastError = err instanceof Error ? err : new Error(String(err));
+
+          // If not transient or last attempt, fail immediately
+          if (!isTransientError(err) || attempt === maxRetries) {
+            break;
+          }
+
+          // Wait before retry with exponential backoff
+          const delay = calculateBackoffDelay(
+            attempt,
+            initialDelayMs,
+            maxDelayMs,
+            backoffMultiplier
+          );
+          await sleep(delay);
+        }
+      }
+
+      if (!success && lastError) {
+        errors.push({
+          index: batchIndex,
+          error: lastError,
+          itemCount: batch.length,
+        });
+      }
+    }
+
+    const totalProcessed = items.length - (errors.length > 0 ? errors.reduce((sum, e) => sum + (e.itemCount || 0), 0) : 0);
+    const totalFailed = errors.reduce((sum, e) => sum + (e.itemCount || 0), 0);
+
+    if (throwOnPartialFailure && errors.length > 0) {
+      const errorMessages = errors
+        .map((e) => `Batch ${e.index} (${e.itemCount} items): ${e.error.message}`)
+        .join('\n');
+      throw new Error(
+        `[BatchProcessor] Batch processing failed:\n${errorMessages}`
+      );
+    }
+
+    return { results, errors, totalProcessed, totalFailed };
+  }
+
+  /**
+   * Processes items sequentially (one at a time) with retry logic.
+   * Useful for operations that don't support batching or when granular
+   * error tracking is needed.
+   */
+  static async processSequential<T, R>(
+    items: T[],
+    processor: (item: T) => Promise<R>,
+    options?: BatchOptions
+  ): Promise<BatchResult<R>> {
+    const {
+      maxRetries = 3,
+      initialDelayMs = 100,
+      maxDelayMs = 10000,
+      backoffMultiplier = 2,
+      throwOnPartialFailure = false,
+    } = options ?? {};
+
+    const results: R[] = [];
+    const errors: Array<{ index: number; error: Error; itemCount?: number }> = [];
+
+    for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
+      const item = items[itemIndex];
+      let success = false;
+      let lastError: Error | undefined;
+
+      for (let attempt = 0; attempt <= maxRetries; attempt++) {
+        try {
+          const result = await processor(item);
+          results.push(result);
+          success = true;
+          break;
+        } catch (err) {
+          lastError = err instanceof Error ? err : new Error(String(err));
+
+          if (!isTransientError(err) || attempt === maxRetries) {
+            break;
+          }
+
+          const delay = calculateBackoffDelay(
+            attempt,
+            initialDelayMs,
+            maxDelayMs,
+            backoffMultiplier
+          );
+          await sleep(delay);
+        }
+      }
+
+      if (!success && lastError) {
+        errors.push({ index: itemIndex, error: lastError, itemCount: 1 });
+      }
+    }
+
+    const totalProcessed = items.length - errors.length;
+    const totalFailed = errors.length;
+
+    if (throwOnPartialFailure && errors.length > 0) {
+      const errorMessages = errors
+        .map((e) => `Item ${e.index}: ${e.error.message}`)
+        .join('\n');
+      throw new Error(
+        `[BatchProcessor] Sequential processing failed:\n${errorMessages}`
+      );
+    }
+
+    return { results, errors, totalProcessed, totalFailed };
+  }
+
+  /**
+   * Maps over items with retry logic, returning results in order.
+   * Like Array.map() but with async processing and automatic retries.
+   */
+  static async mapWithRetry<T, R>(
+    items: T[],
+    mapper: (item: T) => Promise<R>,
+    options?: BatchOptions
+  ): Promise<R[]> {
+    const result = await this.processSequential(items, mapper, options);
+
+    if (result.errors.length > 0) {
+      console.warn(
+        `[BatchProcessor] mapWithRetry: ${result.errors.length} items failed`
+      );
+    }
+
+    // Return results in original order (fill gaps with undefined or error value)
+    const orderedResults: (R | undefined)[] = new Array(items.length);
+    let resultIndex = 0;
+    let errorIndex = 0;
+
+    for (let i = 0; i < items.length; i++) {
+      if (errorIndex < result.errors.length && result.errors[errorIndex].index === i) {
+        orderedResults[i] = undefined;
+        errorIndex++;
+      } else {
+        orderedResults[i] = result.results[resultIndex];
+        resultIndex++;
+      }
+    }
+
+    return orderedResults.filter((r) => r !== undefined) as R[];
+  }
+
+  /**
+   * Parallel processor with concurrency limit.
+   * Processes up to `concurrency` items at the same time.
+   */
+  static async processConcurrent<T, R>(
+    items: T[],
+    processor: (item: T) => Promise<R>,
+    concurrency: number = 5,
+    options?: BatchOptions
+  ): Promise<BatchResult<R>> {
+    const { throwOnPartialFailure = false } = options ?? {};
+    const results: R[] = [];
+    const errors: Array<{ index: number; error: Error; itemCount?: number }> = [];
+
+    // Process in chunks of `concurrency` size
+    for (let i = 0; i < items.length; i += concurrency) {
+      const chunk = items.slice(i, i + concurrency).map((item, idx) => ({
+        item,
+        originalIndex: i + idx,
+      }));
+
+      const chunkPromises = chunk.map(async ({ item, originalIndex }) => {
+        try {
+          const result = await processor(item);
+          return { success: true, result, index: originalIndex };
+        } catch (err) {
+          const error = err instanceof Error ? err : new Error(String(err));
+          return { success: false, error, index: originalIndex };
+        }
+      });
+
+      const chunkResults = await Promise.all(chunkPromises);
+
+      for (const { success, result, error, index } of chunkResults) {
+        if (success) {
+          results.push(result as R);
+        } else {
+          errors.push({ index, error: error as Error, itemCount: 1 });
+        }
+      }
+    }
+
+    const totalProcessed = items.length - errors.length;
+    const totalFailed = errors.length;
+
+    if (throwOnPartialFailure && errors.length > 0) {
+      const errorMessages = errors
+        .map((e) => `Item ${e.index}: ${e.error.message}`)
+        .join('\n');
+      throw new Error(
+        `[BatchProcessor] Concurrent processing failed:\n${errorMessages}`
+      );
+    }
+
+    return { results, errors, totalProcessed, totalFailed };
+  }
+}