@juspay/neurolink 9.1.1 → 9.3.0

package/dist/lib/rag/resilience/RetryHandler.js

@@ -0,0 +1,301 @@
+/**
+ * RAG Retry Handler
+ *
+ * Provides retry logic with exponential backoff and jitter
+ * specifically designed for RAG operations including embeddings,
+ * vector queries, and LLM-based extraction.
+ */
+import { withRetry, } from "../../core/infrastructure/index.js";
+import { logger } from "../../utils/logger.js";
+import { EmbeddingError, isRetryableRAGError, MetadataExtractionError, RAGError, RAGErrorCodes, VectorQueryError, } from "../errors/RAGError.js";
+/**
+ * Default retry configuration
+ */
+export const DEFAULT_RAG_RETRY_CONFIG = {
+    maxRetries: 3,
+    initialDelay: 1000,
+    maxDelay: 30000,
+    backoffMultiplier: 2,
+    jitter: true,
+    retryableErrorCodes: [
+        RAGErrorCodes.EMBEDDING_RATE_LIMIT,
+        RAGErrorCodes.VECTOR_QUERY_TIMEOUT,
+        RAGErrorCodes.VECTOR_STORE_CONNECTION_ERROR,
+        RAGErrorCodes.METADATA_EXTRACTION_TIMEOUT,
+        RAGErrorCodes.RERANKER_API_ERROR,
+        RAGErrorCodes.OPERATION_TIMEOUT,
+    ],
+    retryableStatusCodes: [408, 429, 500, 502, 503, 504],
+};
+/**
+ * Calculate delay with exponential backoff
+ */
+function calculateDelay(attempt, config) {
+    const exponentialDelay = config.initialDelay * config.backoffMultiplier ** attempt;
+    const delay = Math.min(exponentialDelay, config.maxDelay);
+    if (config.jitter) {
+        // Add random jitter between 0 and 50% of the delay
+        const jitterFactor = 0.5 * Math.random();
+        return delay * (1 + jitterFactor);
+    }
+    return delay;
+}
+/**
+ * Sleep utility
+ */
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+/**
+ * Check if an error is retryable based on configuration
+ */
+export function isRetryable(error, config = DEFAULT_RAG_RETRY_CONFIG) {
+    // Use custom shouldRetry if provided
+    if (config.shouldRetry) {
+        return config.shouldRetry(error);
+    }
+    // Check if it's a RAG error with retryable flag
+    if (isRetryableRAGError(error)) {
+        return true;
+    }
+    // Check error code
+    if (error instanceof RAGError && config.retryableErrorCodes) {
+        if (config.retryableErrorCodes.includes(error.code)) {
+            return true;
+        }
+    }
+    // Check for common network/timeout errors
+    if (error && typeof error === "object") {
+        const errorObj = error;
+        // Timeout errors
+        if (errorObj.name === "TimeoutError" ||
+            errorObj.code === "TIMEOUT" ||
+            errorObj.code === "ETIMEDOUT" ||
+            errorObj.name === "AbortError") {
+            return true;
+        }
+        // Network errors
+        if (errorObj.code === "ECONNRESET" ||
+            errorObj.code === "ENOTFOUND" ||
+            errorObj.code === "ECONNREFUSED" ||
+            errorObj.code === "ECONNABORTED" ||
+            errorObj.code === "EPIPE" ||
+            errorObj.code === "ENETUNREACH" ||
+            errorObj.code === "EHOSTUNREACH") {
+            return true;
+        }
+        // HTTP status codes
+        if (typeof errorObj.status === "number" &&
+            config.retryableStatusCodes?.includes(errorObj.status)) {
+            return true;
+        }
+        if (typeof errorObj.statusCode === "number" &&
+            config.retryableStatusCodes?.includes(errorObj.statusCode)) {
+            return true;
+        }
+    }
+    return false;
+}
+/**
+ * Execute a RAG operation with retry logic
+ *
+ * Implements exponential backoff with jitter to prevent thundering herd.
+ * Only retries on errors that are considered retryable.
+ *
+ * @param operation - Async operation to execute with retries
+ * @param config - Partial retry configuration (merged with defaults)
+ * @returns Result of the operation
+ * @throws Last error if all retry attempts fail
+ */
+export async function withRAGRetry(operation, config = {}) {
+    const mergedConfig = {
+        ...DEFAULT_RAG_RETRY_CONFIG,
+        ...config,
+    };
+    let lastError;
+    let attempt = 0;
+    while (attempt <= mergedConfig.maxRetries) {
+        try {
+            return await operation();
+        }
+        catch (error) {
+            lastError = error;
+            // Don't retry if this is the last attempt
+            if (attempt === mergedConfig.maxRetries) {
+                logger.debug(`[RAGRetryHandler] All ${mergedConfig.maxRetries} attempts exhausted`);
+                break;
+            }
+            // Check if we should retry this error
+            if (!isRetryable(error, mergedConfig)) {
+                logger.debug(`[RAGRetryHandler] Non-retryable error encountered: ${error instanceof Error ? error.message : String(error)}`);
+                break;
+            }
+            // Calculate delay with backoff and jitter
+            const delay = calculateDelay(attempt, mergedConfig);
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            logger.warn(`[RAGRetryHandler] Attempt ${attempt + 1}/${mergedConfig.maxRetries + 1} failed: ${errorMessage}. Retrying in ${Math.round(delay)}ms...`);
+            await sleep(delay);
+            attempt++;
+        }
+    }
+    // Wrap the last error if it's not already a RAG error
+    if (lastError instanceof RAGError) {
+        throw lastError;
+    }
+    throw new RAGError(`RAG operation failed after ${attempt + 1} attempts: ${lastError instanceof Error ? lastError.message : String(lastError)}`, RAGErrorCodes.RETRY_EXHAUSTED, {
+        retryable: false,
+        cause: lastError instanceof Error ? lastError : undefined,
+        details: {
+            totalAttempts: attempt + 1,
+            maxRetries: mergedConfig.maxRetries,
+        },
+    });
+}
+/**
+ * RAG Retry Handler class for more complex retry scenarios
+ */
+export class RAGRetryHandler {
+    config;
+    constructor(config = {}) {
+        this.config = { ...DEFAULT_RAG_RETRY_CONFIG, ...config };
+    }
+    /**
+     * Execute an operation with retry logic
+     */
+    async executeWithRetry(operation, maxRetries) {
+        const config = maxRetries !== undefined ? { ...this.config, maxRetries } : this.config;
+        return withRAGRetry(operation, config);
+    }
+    /**
+     * Execute multiple operations with retry, collecting results
+     * Returns successful results and failed operations with their errors
+     */
+    async executeBatch(items, operation, options) {
+        const { concurrency = 5, continueOnError = true } = options ?? {};
+        const successful = [];
+        const failed = [];
+        // Process in batches for concurrency control
+        for (let i = 0; i < items.length; i += concurrency) {
+            const batch = items.slice(i, i + concurrency);
+            const batchPromises = batch.map(async (item, batchIndex) => {
+                const index = i + batchIndex;
+                try {
+                    const result = await this.executeWithRetry(() => operation(item, index));
+                    successful.push({ item, result, index });
+                }
+                catch (error) {
+                    const errorObj = error instanceof Error ? error : new Error(String(error));
+                    failed.push({ item, error: errorObj, index });
+                    if (!continueOnError) {
+                        throw error;
+                    }
+                }
+            });
+            await Promise.all(batchPromises);
+        }
+        const total = successful.length + failed.length;
+        const successRate = total > 0 ? successful.length / total : 0;
+        return { successful, failed, successRate };
+    }
+    /**
+     * Get current configuration
+     */
+    getConfig() {
+        return { ...this.config };
+    }
+    /**
+     * Update configuration
+     */
+    updateConfig(config) {
+        this.config = { ...this.config, ...config };
+    }
+}
+/**
+ * Specialized retry handler for embedding operations
+ */
+export class EmbeddingRetryHandler extends RAGRetryHandler {
+    constructor(config) {
+        super({
+            maxRetries: 5, // More retries for rate-limited embedding APIs
+            initialDelay: 2000, // Longer initial delay
+            retryableErrorCodes: [
+                RAGErrorCodes.EMBEDDING_ERROR,
+                RAGErrorCodes.EMBEDDING_RATE_LIMIT,
+                RAGErrorCodes.EMBEDDING_PROVIDER_ERROR,
+            ],
+            shouldRetry: (error) => {
+                // Always retry rate limit errors
+                if (error instanceof EmbeddingError) {
+                    return error.retryable;
+                }
+                return isRetryable(error);
+            },
+            ...config,
+        });
+    }
+}
+/**
+ * Specialized retry handler for vector store operations
+ */
+export class VectorStoreRetryHandler extends RAGRetryHandler {
+    constructor(config) {
+        super({
+            maxRetries: 3,
+            initialDelay: 1000,
+            retryableErrorCodes: [
+                RAGErrorCodes.VECTOR_QUERY_ERROR,
+                RAGErrorCodes.VECTOR_QUERY_TIMEOUT,
+                RAGErrorCodes.VECTOR_STORE_UNAVAILABLE,
+                RAGErrorCodes.VECTOR_STORE_CONNECTION_ERROR,
+            ],
+            shouldRetry: (error) => {
+                if (error instanceof VectorQueryError) {
+                    return error.retryable;
+                }
+                return isRetryable(error);
+            },
+            ...config,
+        });
+    }
+}
+/**
+ * Specialized retry handler for metadata extraction
+ */
+export class MetadataExtractionRetryHandler extends RAGRetryHandler {
+    constructor(config) {
+        super({
+            maxRetries: 3,
+            initialDelay: 1500,
+            retryableErrorCodes: [
+                RAGErrorCodes.METADATA_EXTRACTION_ERROR,
+                RAGErrorCodes.METADATA_EXTRACTION_TIMEOUT,
+            ],
+            shouldRetry: (error) => {
+                if (error instanceof MetadataExtractionError) {
+                    return error.retryable;
+                }
+                return isRetryable(error);
+            },
+            ...config,
+        });
+    }
+}
+/**
+ * Create a retry handler with the core infrastructure withRetry
+ */
+export function createRetryHandler(config = {}) {
+    const mergedConfig = { ...DEFAULT_RAG_RETRY_CONFIG, ...config };
+    return (operation) => withRetry(operation, {
+        maxRetries: mergedConfig.maxRetries,
+        baseDelayMs: mergedConfig.initialDelay,
+        maxDelayMs: mergedConfig.maxDelay,
+        shouldRetry: (error) => isRetryable(error, mergedConfig),
+    });
+}
+/**
+ * Global retry handlers for common RAG operations
+ */
+export const embeddingRetryHandler = new EmbeddingRetryHandler();
+export const vectorStoreRetryHandler = new VectorStoreRetryHandler();
+export const metadataExtractionRetryHandler = new MetadataExtractionRetryHandler();
+//# sourceMappingURL=RetryHandler.js.map

package/dist/lib/rag/resilience/index.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Resilience Index
+ *
+ * Exports resilience patterns for RAG operations.
+ */
+export { RAGCircuitBreaker, RAGCircuitBreakerManager, ragCircuitBreakerManager, getCircuitBreaker, executeWithCircuitBreaker, type CircuitState, type RAGCircuitBreakerConfig, type RAGCircuitBreakerStats, type RAGCircuitBreakerEvents, } from "./CircuitBreaker.js";
+export { RAGRetryHandler, EmbeddingRetryHandler, VectorStoreRetryHandler, MetadataExtractionRetryHandler, withRAGRetry, isRetryable, createRetryHandler, embeddingRetryHandler, vectorStoreRetryHandler, metadataExtractionRetryHandler, DEFAULT_RAG_RETRY_CONFIG, type RAGRetryConfig, } from "./RetryHandler.js";

package/dist/lib/rag/resilience/index.js

@@ -0,0 +1,8 @@
+/**
+ * Resilience Index
+ *
+ * Exports resilience patterns for RAG operations.
+ */
+export { RAGCircuitBreaker, RAGCircuitBreakerManager, ragCircuitBreakerManager, getCircuitBreaker, executeWithCircuitBreaker, } from "./CircuitBreaker.js";
+export { RAGRetryHandler, EmbeddingRetryHandler, VectorStoreRetryHandler, MetadataExtractionRetryHandler, withRAGRetry, isRetryable, createRetryHandler, embeddingRetryHandler, vectorStoreRetryHandler, metadataExtractionRetryHandler, DEFAULT_RAG_RETRY_CONFIG, } from "./RetryHandler.js";
+//# sourceMappingURL=index.js.map

package/dist/lib/rag/retrieval/hybridSearch.d.ts

@@ -0,0 +1,94 @@
+/**
+ * Hybrid Search Implementation
+ *
+ * Combines vector (dense) search with BM25 (sparse) search for improved retrieval.
+ * Supports multiple fusion methods: Reciprocal Rank Fusion (RRF) and Linear Combination.
+ */
+import type { BM25Result, HybridSearchConfig, HybridSearchResult } from "../types.js";
+import type { VectorStore } from "./vectorQueryTool.js";
+/**
+ * BM25 Index interface
+ * Implementations should provide sparse retrieval capabilities
+ */
+export interface BM25Index {
+    /**
+     * Search documents using BM25 algorithm
+     * @param query - Search query string
+     * @param topK - Number of results to return
+     * @returns Array of BM25 results
+     */
+    search(query: string, topK?: number): Promise<BM25Result[]>;
+    /**
+     * Add documents to the index
+     * @param documents - Documents to index
+     */
+    addDocuments(documents: Array<{
+        id: string;
+        text: string;
+        metadata?: Record<string, unknown>;
+    }>): Promise<void>;
+}
+/**
+ * In-memory BM25 implementation for testing and development
+ */
+export declare class InMemoryBM25Index implements BM25Index {
+    private documents;
+    private avgDocLength;
+    private k1;
+    private b;
+    search(query: string, topK?: number): Promise<BM25Result[]>;
+    addDocuments(documents: Array<{
+        id: string;
+        text: string;
+        metadata?: Record<string, unknown>;
+    }>): Promise<void>;
+    private tokenize;
+    private countTermFrequency;
+    private countDocumentsWithTerm;
+}
+/**
+ * Reciprocal Rank Fusion
+ * Combines rankings from multiple retrieval methods
+ *
+ * @param rankings - Array of ranking lists, each with id and rank
+ * @param k - RRF constant (default: 60)
+ * @returns Map of document IDs to fused scores
+ */
+export declare function reciprocalRankFusion(rankings: Array<Array<{
+    id: string;
+    rank: number;
+}>>, k?: number): Map<string, number>;
+/**
+ * Linear Combination of normalized scores
+ *
+ * @param vectorScores - Vector search scores
+ * @param bm25Scores - BM25 search scores
+ * @param alpha - Weight for vector scores (0-1), bm25 gets 1-alpha
+ * @returns Map of document IDs to combined scores
+ */
+export declare function linearCombination(vectorScores: Map<string, number>, bm25Scores: Map<string, number>, alpha?: number): Map<string, number>;
+/**
+ * Hybrid search configuration for creating a search function
+ */
+export interface HybridSearchOptions {
+    /** Vector store instance */
+    vectorStore: VectorStore;
+    /** BM25 index instance */
+    bm25Index: BM25Index;
+    /** Index name for vector store */
+    indexName: string;
+    /** Embedding model configuration (optional - uses defaults from ProviderFactory if not specified) */
+    embeddingModel?: {
+        provider?: string;
+        modelName?: string;
+    };
+    /** Default search configuration */
+    defaultConfig?: HybridSearchConfig;
+}
+/**
+ * Create a hybrid search function
+ *
+ * @param options - Search options
+ * @returns Hybrid search function
+ */
+export declare function createHybridSearch(options: HybridSearchOptions): (query: string, config?: HybridSearchConfig) => Promise<HybridSearchResult[]>;