@juspay/neurolink 9.32.0 → 9.33.0

package/dist/client/rag/resilience/RetryHandler.js

@@ -1,304 +0,0 @@
-/**
- * RAG Retry Handler
- *
- * Provides retry logic with exponential backoff and jitter
- * specifically designed for RAG operations including embeddings,
- * vector queries, and LLM-based extraction.
- */
-import { withRetry } from "../../core/infrastructure/index.js";
-import { isAbortError } from "../../utils/errorHandling.js";
-import { logger } from "../../utils/logger.js";
-import { EmbeddingError, isRetryableRAGError, MetadataExtractionError, RAGError, RAGErrorCodes, VectorQueryError, } from "../errors/RAGError.js";
-/**
- * Default retry configuration
- */
-export const DEFAULT_RAG_RETRY_CONFIG = {
-    maxRetries: 3,
-    initialDelay: 1000,
-    maxDelay: 30000,
-    backoffMultiplier: 2,
-    jitter: true,
-    retryableErrorCodes: [
-        RAGErrorCodes.EMBEDDING_RATE_LIMIT,
-        RAGErrorCodes.VECTOR_QUERY_TIMEOUT,
-        RAGErrorCodes.VECTOR_STORE_CONNECTION_ERROR,
-        RAGErrorCodes.METADATA_EXTRACTION_TIMEOUT,
-        RAGErrorCodes.RERANKER_API_ERROR,
-        RAGErrorCodes.OPERATION_TIMEOUT,
-    ],
-    retryableStatusCodes: [408, 429, 500, 502, 503, 504],
-};
-/**
- * Calculate delay with exponential backoff
- */
-function calculateDelay(attempt, config) {
-    const exponentialDelay = config.initialDelay * config.backoffMultiplier ** attempt;
-    const delay = Math.min(exponentialDelay, config.maxDelay);
-    if (config.jitter) {
-        // Add random jitter between 0 and 50% of the delay
-        const jitterFactor = 0.5 * Math.random();
-        return delay * (1 + jitterFactor);
-    }
-    return delay;
-}
-/**
- * Sleep utility
- */
-function sleep(ms) {
-    return new Promise((resolve) => setTimeout(resolve, ms));
-}
-/**
- * Check if an error is retryable based on configuration
- */
-export function isRetryable(error, config = DEFAULT_RAG_RETRY_CONFIG) {
-    // Never retry abort errors - the operation was intentionally cancelled
-    if (isAbortError(error)) {
-        return false;
-    }
-    // Use custom shouldRetry if provided
-    if (config.shouldRetry) {
-        return config.shouldRetry(error);
-    }
-    // Check if it's a RAG error with retryable flag
-    if (isRetryableRAGError(error)) {
-        return true;
-    }
-    // Check error code
-    if (error instanceof RAGError && config.retryableErrorCodes) {
-        if (config.retryableErrorCodes.includes(error.code)) {
-            return true;
-        }
-    }
-    // Check for common network/timeout errors
-    if (error && typeof error === "object") {
-        const errorObj = error;
-        // Timeout errors
-        if (errorObj.name === "TimeoutError" ||
-            errorObj.code === "TIMEOUT" ||
-            errorObj.code === "ETIMEDOUT") {
-            return true;
-        }
-        // Network errors
-        if (errorObj.code === "ECONNRESET" ||
-            errorObj.code === "ENOTFOUND" ||
-            errorObj.code === "ECONNREFUSED" ||
-            errorObj.code === "ECONNABORTED" ||
-            errorObj.code === "EPIPE" ||
-            errorObj.code === "ENETUNREACH" ||
-            errorObj.code === "EHOSTUNREACH") {
-            return true;
-        }
-        // HTTP status codes
-        if (typeof errorObj.status === "number" &&
-            config.retryableStatusCodes?.includes(errorObj.status)) {
-            return true;
-        }
-        if (typeof errorObj.statusCode === "number" &&
-            config.retryableStatusCodes?.includes(errorObj.statusCode)) {
-            return true;
-        }
-    }
-    return false;
-}
-/**
- * Execute a RAG operation with retry logic
- *
- * Implements exponential backoff with jitter to prevent thundering herd.
- * Only retries on errors that are considered retryable.
- *
- * @param operation - Async operation to execute with retries
- * @param config - Partial retry configuration (merged with defaults)
- * @returns Result of the operation
- * @throws Last error if all retry attempts fail
- */
-export async function withRAGRetry(operation, config = {}) {
-    const mergedConfig = {
-        ...DEFAULT_RAG_RETRY_CONFIG,
-        ...config,
-    };
-    let lastError;
-    let attempt = 0;
-    while (attempt <= mergedConfig.maxRetries) {
-        try {
-            return await operation();
-        }
-        catch (error) {
-            lastError = error;
-            // Don't retry if this is the last attempt
-            if (attempt === mergedConfig.maxRetries) {
-                logger.debug(`[RAGRetryHandler] All ${mergedConfig.maxRetries} attempts exhausted`);
-                break;
-            }
-            // Check if we should retry this error
-            if (!isRetryable(error, mergedConfig)) {
-                logger.debug(`[RAGRetryHandler] Non-retryable error encountered: ${error instanceof Error ? error.message : String(error)}`);
-                break;
-            }
-            // Calculate delay with backoff and jitter
-            const delay = calculateDelay(attempt, mergedConfig);
-            const errorMessage = error instanceof Error ? error.message : String(error);
-            logger.warn(`[RAGRetryHandler] Attempt ${attempt + 1}/${mergedConfig.maxRetries + 1} failed: ${errorMessage}. Retrying in ${Math.round(delay)}ms...`);
-            await sleep(delay);
-            attempt++;
-        }
-    }
-    // Wrap the last error if it's not already a RAG error
-    if (lastError instanceof RAGError) {
-        throw lastError;
-    }
-    throw new RAGError(`RAG operation failed after ${attempt + 1} attempts: ${lastError instanceof Error ? lastError.message : String(lastError)}`, RAGErrorCodes.RETRY_EXHAUSTED, {
-        retryable: false,
-        cause: lastError instanceof Error ? lastError : undefined,
-        details: {
-            totalAttempts: attempt + 1,
-            maxRetries: mergedConfig.maxRetries,
-        },
-    });
-}
-/**
- * RAG Retry Handler class for more complex retry scenarios
- */
-export class RAGRetryHandler {
-    config;
-    constructor(config = {}) {
-        this.config = { ...DEFAULT_RAG_RETRY_CONFIG, ...config };
-    }
-    /**
-     * Execute an operation with retry logic
-     */
-    async executeWithRetry(operation, maxRetries) {
-        const config = maxRetries !== undefined ? { ...this.config, maxRetries } : this.config;
-        return withRAGRetry(operation, config);
-    }
-    /**
-     * Execute multiple operations with retry, collecting results
-     * Returns successful results and failed operations with their errors
-     */
-    async executeBatch(items, operation, options) {
-        const { concurrency = 5, continueOnError = true } = options ?? {};
-        const successful = [];
-        const failed = [];
-        // Process in batches for concurrency control
-        for (let i = 0; i < items.length; i += concurrency) {
-            const batch = items.slice(i, i + concurrency);
-            const batchPromises = batch.map(async (item, batchIndex) => {
-                const index = i + batchIndex;
-                try {
-                    const result = await this.executeWithRetry(() => operation(item, index));
-                    successful.push({ item, result, index });
-                }
-                catch (error) {
-                    const errorObj = error instanceof Error ? error : new Error(String(error));
-                    failed.push({ item, error: errorObj, index });
-                    if (!continueOnError) {
-                        throw error;
-                    }
-                }
-            });
-            await Promise.all(batchPromises);
-        }
-        const total = successful.length + failed.length;
-        const successRate = total > 0 ? successful.length / total : 0;
-        return { successful, failed, successRate };
-    }
-    /**
-     * Get current configuration
-     */
-    getConfig() {
-        return { ...this.config };
-    }
-    /**
-     * Update configuration
-     */
-    updateConfig(config) {
-        this.config = { ...this.config, ...config };
-    }
-}
-/**
- * Specialized retry handler for embedding operations
- */
-export class EmbeddingRetryHandler extends RAGRetryHandler {
-    constructor(config) {
-        super({
-            maxRetries: 5, // More retries for rate-limited embedding APIs
-            initialDelay: 2000, // Longer initial delay
-            retryableErrorCodes: [
-                RAGErrorCodes.EMBEDDING_ERROR,
-                RAGErrorCodes.EMBEDDING_RATE_LIMIT,
-                RAGErrorCodes.EMBEDDING_PROVIDER_ERROR,
-            ],
-            shouldRetry: (error) => {
-                // Always retry rate limit errors
-                if (error instanceof EmbeddingError) {
-                    return error.retryable;
-                }
-                return isRetryable(error);
-            },
-            ...config,
-        });
-    }
-}
-/**
- * Specialized retry handler for vector store operations
- */
-export class VectorStoreRetryHandler extends RAGRetryHandler {
-    constructor(config) {
-        super({
-            maxRetries: 3,
-            initialDelay: 1000,
-            retryableErrorCodes: [
-                RAGErrorCodes.VECTOR_QUERY_ERROR,
-                RAGErrorCodes.VECTOR_QUERY_TIMEOUT,
-                RAGErrorCodes.VECTOR_STORE_UNAVAILABLE,
-                RAGErrorCodes.VECTOR_STORE_CONNECTION_ERROR,
-            ],
-            shouldRetry: (error) => {
-                if (error instanceof VectorQueryError) {
-                    return error.retryable;
-                }
-                return isRetryable(error);
-            },
-            ...config,
-        });
-    }
-}
-/**
- * Specialized retry handler for metadata extraction
- */
-export class MetadataExtractionRetryHandler extends RAGRetryHandler {
-    constructor(config) {
-        super({
-            maxRetries: 3,
-            initialDelay: 1500,
-            retryableErrorCodes: [
-                RAGErrorCodes.METADATA_EXTRACTION_ERROR,
-                RAGErrorCodes.METADATA_EXTRACTION_TIMEOUT,
-            ],
-            shouldRetry: (error) => {
-                if (error instanceof MetadataExtractionError) {
-                    return error.retryable;
-                }
-                return isRetryable(error);
-            },
-            ...config,
-        });
-    }
-}
-/**
- * Create a retry handler with the core infrastructure withRetry
- */
-export function createRetryHandler(config = {}) {
-    const mergedConfig = { ...DEFAULT_RAG_RETRY_CONFIG, ...config };
-    return (operation) => withRetry(operation, {
-        maxRetries: mergedConfig.maxRetries,
-        baseDelayMs: mergedConfig.initialDelay,
-        maxDelayMs: mergedConfig.maxDelay,
-        shouldRetry: (error) => isRetryable(error, mergedConfig),
-    });
-}
-/**
- * Global retry handlers for common RAG operations
- */
-export const embeddingRetryHandler = new EmbeddingRetryHandler();
-export const vectorStoreRetryHandler = new VectorStoreRetryHandler();
-export const metadataExtractionRetryHandler = new MetadataExtractionRetryHandler();

package/dist/client/rag/resilience/index.js

@@ -1,7 +0,0 @@
-/**
- * Resilience Index
- *
- * Exports resilience patterns for RAG operations.
- */
-export { RAGCircuitBreaker, RAGCircuitBreakerManager, ragCircuitBreakerManager, getCircuitBreaker, executeWithCircuitBreaker, } from "./CircuitBreaker.js";
-export { RAGRetryHandler, EmbeddingRetryHandler, VectorStoreRetryHandler, MetadataExtractionRetryHandler, withRAGRetry, isRetryable, createRetryHandler, embeddingRetryHandler, vectorStoreRetryHandler, metadataExtractionRetryHandler, DEFAULT_RAG_RETRY_CONFIG, } from "./RetryHandler.js";

package/dist/client/rag/retrieval/hybridSearch.js

@@ -1,335 +0,0 @@
-/**
- * Hybrid Search Implementation
- *
- * Combines vector (dense) search with BM25 (sparse) search for improved retrieval.
- * Supports multiple fusion methods: Reciprocal Rank Fusion (RRF) and Linear Combination.
- */
-import { ProviderFactory } from "../../factories/providerFactory.js";
-import { SpanSerializer, SpanType, SpanStatus, getMetricsAggregator, } from "../../observability/index.js";
-import { logger } from "../../utils/logger.js";
-import { rerank } from "../reranker/reranker.js";
-/**
- * In-memory BM25 implementation for testing and development
- */
-export class InMemoryBM25Index {
-    documents = new Map();
-    avgDocLength = 0;
-    k1 = 1.5; // BM25 parameter
-    b = 0.75; // BM25 parameter
-    async search(query, topK = 10) {
-        const queryTokens = this.tokenize(query);
-        if (queryTokens.length === 0 || this.documents.size === 0) {
-            return [];
-        }
-        // Calculate IDF for each query term
-        const idfValues = new Map();
-        for (const token of queryTokens) {
-            const docCount = this.countDocumentsWithTerm(token);
-            const idf = Math.log((this.documents.size - docCount + 0.5) / (docCount + 0.5) + 1);
-            idfValues.set(token, idf);
-        }
-        // Calculate BM25 score for each document
-        const scores = [];
-        for (const [id, doc] of this.documents) {
-            let score = 0;
-            const docLength = doc.tokens.length;
-            for (const token of queryTokens) {
-                const tf = this.countTermFrequency(doc.tokens, token);
-                const idf = idfValues.get(token) || 0;
-                // BM25 scoring formula
-                const numerator = tf * (this.k1 + 1);
-                const denominator = tf +
-                    this.k1 * (1 - this.b + this.b * (docLength / this.avgDocLength));
-                score += idf * (numerator / denominator);
-            }
-            if (score > 0) {
-                scores.push({
-                    id,
-                    score,
-                    text: doc.text,
-                    metadata: doc.metadata,
-                });
-            }
-        }
-        // Sort by score descending
-        scores.sort((a, b) => b.score - a.score);
-        return scores.slice(0, topK);
-    }
-    async addDocuments(documents) {
-        for (const doc of documents) {
-            const tokens = this.tokenize(doc.text);
-            this.documents.set(doc.id, {
-                text: doc.text,
-                tokens,
-                metadata: doc.metadata || {},
-            });
-        }
-        // Recalculate average document length
-        let totalLength = 0;
-        for (const doc of this.documents.values()) {
-            totalLength += doc.tokens.length;
-        }
-        this.avgDocLength =
-            this.documents.size > 0 ? totalLength / this.documents.size : 0;
-    }
-    tokenize(text) {
-        return text
-            .toLowerCase()
-            .replace(/[^\w\s]/g, " ")
-            .split(/\s+/)
-            .filter((t) => t.length > 0);
-    }
-    countTermFrequency(tokens, term) {
-        return tokens.filter((t) => t === term).length;
-    }
-    countDocumentsWithTerm(term) {
-        let count = 0;
-        for (const doc of this.documents.values()) {
-            if (doc.tokens.includes(term)) {
-                count++;
-            }
-        }
-        return count;
-    }
-}
-/**
- * Reciprocal Rank Fusion
- * Combines rankings from multiple retrieval methods
- *
- * @param rankings - Array of ranking lists, each with id and rank
- * @param k - RRF constant (default: 60)
- * @returns Map of document IDs to fused scores
- */
-export function reciprocalRankFusion(rankings, k = 60) {
-    const scores = new Map();
-    for (const ranking of rankings) {
-        for (const { id, rank } of ranking) {
-            const currentScore = scores.get(id) || 0;
-            scores.set(id, currentScore + 1 / (k + rank));
-        }
-    }
-    return scores;
-}
-/**
- * Linear Combination of normalized scores
- *
- * @param vectorScores - Vector search scores
- * @param bm25Scores - BM25 search scores
- * @param alpha - Weight for vector scores (0-1), bm25 gets 1-alpha
- * @returns Map of document IDs to combined scores
- */
-export function linearCombination(vectorScores, bm25Scores, alpha = 0.5) {
-    const combined = new Map();
-    // Get all document IDs
-    const allIds = new Set([...vectorScores.keys(), ...bm25Scores.keys()]);
-    // Normalize scores
-    const normalizedVector = normalizeScores(vectorScores);
-    const normalizedBM25 = normalizeScores(bm25Scores);
-    for (const id of allIds) {
-        const vectorScore = normalizedVector.get(id) || 0;
-        const bm25Score = normalizedBM25.get(id) || 0;
-        combined.set(id, alpha * vectorScore + (1 - alpha) * bm25Score);
-    }
-    return combined;
-}
-/**
- * Normalize scores to 0-1 range
- */
-function normalizeScores(scores) {
-    const values = Array.from(scores.values());
-    if (values.length === 0) {
-        return new Map();
-    }
-    const min = Math.min(...values);
-    const max = Math.max(...values);
-    const range = max - min || 1;
-    const normalized = new Map();
-    for (const [id, score] of scores) {
-        normalized.set(id, (score - min) / range);
-    }
-    return normalized;
-}
-/**
- * Create a hybrid search function
- *
- * @param options - Search options
- * @returns Hybrid search function
- */
-export function createHybridSearch(options) {
-    const { vectorStore, bm25Index, indexName, embeddingModel, defaultConfig = {}, } = options;
-    /**
-     * Execute hybrid search combining vector and BM25 retrieval
-     *
-     * @param query - Search query
-     * @param config - Search configuration
-     * @returns Hybrid search results
-     */
-    return async function hybridSearch(query, config) {
-        const startTime = Date.now();
-        const { vectorWeight = defaultConfig.vectorWeight ?? 0.5, bm25Weight = defaultConfig.bm25Weight ?? 0.5, fusionMethod = defaultConfig.fusionMethod ?? "rrf", rrfK = defaultConfig.rrfK ?? 60, topK = defaultConfig.topK ?? 10, enableReranking = defaultConfig.enableReranking ?? false, reranker: rerankerConfig = defaultConfig.reranker, } = config || {};
-        const span = SpanSerializer.createSpan(SpanType.RAG, "rag.search", {
-            "rag.operation": "search",
-            "rag.topK": topK,
-            "rag.fusionMethod": fusionMethod,
-            "rag.query": query.slice(0, 200),
-        });
-        const spanStartTime = Date.now();
-        try {
-            // Generate query embedding
-            const embeddingProvider = await ProviderFactory.createProvider(embeddingModel?.provider, embeddingModel?.modelName);
-            if (typeof embeddingProvider
-                .embed !== "function") {
-                throw new Error(`Embedding provider does not support the embed() method. ` +
-                    `Please use a provider that supports embeddings (e.g., OpenAI text-embedding-3-small, Vertex text-embedding-004).`);
-            }
-            const queryEmbedding = await embeddingProvider.embed(query);
-            // Parallel retrieval
-            const [vectorResults, bm25Results] = await Promise.all([
-                vectorStore.query({
-                    indexName,
-                    queryVector: queryEmbedding,
-                    topK: topK * 2, // Get more for fusion
-                }),
-                bm25Index.search(query, topK * 2),
-            ]);
-            // Fuse results
-            let fusedResults;
-            if (fusionMethod === "rrf") {
-                // Reciprocal Rank Fusion
-                const vectorRanking = vectorResults.map((r, i) => ({
-                    id: r.id,
-                    rank: i + 1,
-                }));
-                const bm25Ranking = bm25Results.map((r, i) => ({
-                    id: r.id,
-                    rank: i + 1,
-                }));
-                const rrfScores = reciprocalRankFusion([vectorRanking, bm25Ranking], rrfK);
-                // Combine with original data
-                const resultMap = new Map();
-                for (const r of vectorResults) {
-                    resultMap.set(r.id, { text: r.text || "", metadata: r.metadata });
-                }
-                for (const r of bm25Results) {
-                    if (!resultMap.has(r.id)) {
-                        resultMap.set(r.id, { text: r.text, metadata: r.metadata });
-                    }
-                }
-                fusedResults = Array.from(rrfScores.entries())
-                    .sort((a, b) => b[1] - a[1])
-                    .slice(0, topK)
-                    .map(([id, score]) => ({
-                    id,
-                    score,
-                    text: resultMap.get(id)?.text || "",
-                    metadata: resultMap.get(id)?.metadata,
-                    scores: {
-                        combined: score,
-                    },
-                }));
-            }
-            else {
-                // Linear combination
-                const vectorScoreMap = new Map(vectorResults.map((r) => [r.id, r.score || 0]));
-                const bm25ScoreMap = new Map(bm25Results.map((r) => [r.id, r.score]));
-                // Adjust weights based on config
-                const totalWeight = vectorWeight + bm25Weight;
-                const normalizedVectorWeight = vectorWeight / totalWeight;
-                const combinedScores = linearCombination(vectorScoreMap, bm25ScoreMap, normalizedVectorWeight);
-                // Combine with original data
-                const resultMap = new Map();
-                for (const r of vectorResults) {
-                    resultMap.set(r.id, {
-                        text: r.text || "",
-                        metadata: r.metadata,
-                        vectorScore: r.score,
-                    });
-                }
-                for (const r of bm25Results) {
-                    const existing = resultMap.get(r.id);
-                    if (existing) {
-                        existing.bm25Score = r.score;
-                    }
-                    else {
-                        resultMap.set(r.id, {
-                            text: r.text,
-                            metadata: r.metadata,
-                            bm25Score: r.score,
-                        });
-                    }
-                }
-                fusedResults = Array.from(combinedScores.entries())
-                    .sort((a, b) => b[1] - a[1])
-                    .slice(0, topK)
-                    .map(([id, score]) => {
-                    const data = resultMap.get(id);
-                    return {
-                        id,
-                        score,
-                        text: data?.text || "",
-                        metadata: data?.metadata,
-                        scores: {
-                            vector: data?.vectorScore,
-                            bm25: data?.bm25Score,
-                            combined: score,
-                        },
-                    };
-                });
-            }
-            // Apply reranking if configured
-            if (enableReranking && rerankerConfig && fusedResults.length > 0) {
-                const rerankerModel = await ProviderFactory.createProvider(rerankerConfig.model.provider, rerankerConfig.model.modelName);
-                const rerankedResults = await rerank(fusedResults.map((r) => ({
-                    id: r.id,
-                    text: r.text,
-                    score: r.score,
-                    metadata: r.metadata,
-                })), query, rerankerModel, {
-                    weights: rerankerConfig.weights,
-                    topK: rerankerConfig.topK || topK,
-                });
-                fusedResults = rerankedResults.map((r) => ({
-                    id: r.result.id,
-                    score: r.score,
-                    text: r.result.text || "",
-                    metadata: r.result.metadata,
-                    scores: {
-                        ...(fusedResults.find((f) => f.id === r.result.id)?.scores || {}),
-                        reranked: r.score,
-                    },
-                }));
-            }
-            const queryTime = Date.now() - startTime;
-            logger.info("[HybridSearch] Search completed", {
-                query: query.slice(0, 50),
-                vectorResults: vectorResults.length,
-                bm25Results: bm25Results.length,
-                fusedResults: fusedResults.length,
-                fusionMethod,
-                queryTime,
-            });
-            span.durationMs = Date.now() - spanStartTime;
-            const endedSpan = SpanSerializer.endSpan(span, SpanStatus.OK);
-            endedSpan.attributes = {
-                ...endedSpan.attributes,
-                "rag.results_count": fusedResults.length,
-                "rag.vector_results": vectorResults.length,
-                "rag.bm25_results": bm25Results.length,
-            };
-            getMetricsAggregator().recordSpan(endedSpan);
-            return fusedResults;
-        }
-        catch (error) {
-            span.durationMs = Date.now() - spanStartTime;
-            const endedSpan = SpanSerializer.endSpan(span, SpanStatus.ERROR);
-            endedSpan.statusMessage =
-                error instanceof Error ? error.message : String(error);
-            getMetricsAggregator().recordSpan(endedSpan);
-            logger.error("[HybridSearch] Search failed", {
-                query: query.slice(0, 50),
-                error: error instanceof Error ? error.message : String(error),
-            });
-            throw error;
-        }
-    };
-}

package/dist/client/rag/retrieval/index.js

@@ -1,5 +0,0 @@
-/**
- * Retrieval Module Exports
- */
-export { createVectorQueryTool, InMemoryVectorStore, } from "./vectorQueryTool.js";
-export { createHybridSearch, InMemoryBM25Index, reciprocalRankFusion, linearCombination, } from "./hybridSearch.js";