@juspay/neurolink 9.30.0 → 9.31.0

package/dist/client/rag/errors/RAGError.js

@@ -0,0 +1,274 @@
+/**
+ * RAG Error Types
+ *
+ * Provides typed errors for all RAG operations including chunking,
+ * metadata extraction, embedding, vector queries, and reranking.
+ * Uses the NeuroLinkFeatureError pattern for consistency.
+ */
+import { NeuroLinkFeatureError, createErrorFactory, } from "../../core/infrastructure/index.js";
+/**
+ * RAG error codes for all RAG-related operations
+ */
+export const RAGErrorCodes = {
+    // Chunking errors
+    CHUNKING_ERROR: "RAG_CHUNKING_ERROR",
+    CHUNKING_INVALID_CONFIG: "RAG_CHUNKING_INVALID_CONFIG",
+    CHUNKING_STRATEGY_NOT_FOUND: "RAG_CHUNKING_STRATEGY_NOT_FOUND",
+    CHUNKING_EMPTY_CONTENT: "RAG_CHUNKING_EMPTY_CONTENT",
+    CHUNKING_SIZE_EXCEEDED: "RAG_CHUNKING_SIZE_EXCEEDED",
+    // Metadata extraction errors
+    METADATA_EXTRACTION_ERROR: "RAG_METADATA_EXTRACTION_ERROR",
+    METADATA_EXTRACTION_TIMEOUT: "RAG_METADATA_EXTRACTION_TIMEOUT",
+    METADATA_SCHEMA_INVALID: "RAG_METADATA_SCHEMA_INVALID",
+    METADATA_EXTRACTOR_NOT_FOUND: "RAG_METADATA_EXTRACTOR_NOT_FOUND",
+    // Embedding errors
+    EMBEDDING_ERROR: "RAG_EMBEDDING_ERROR",
+    EMBEDDING_DIMENSION_MISMATCH: "RAG_EMBEDDING_DIMENSION_MISMATCH",
+    EMBEDDING_RATE_LIMIT: "RAG_EMBEDDING_RATE_LIMIT",
+    EMBEDDING_PROVIDER_ERROR: "RAG_EMBEDDING_PROVIDER_ERROR",
+    // Vector store/query errors
+    VECTOR_QUERY_ERROR: "RAG_VECTOR_QUERY_ERROR",
+    VECTOR_QUERY_TIMEOUT: "RAG_VECTOR_QUERY_TIMEOUT",
+    VECTOR_STORE_UNAVAILABLE: "RAG_VECTOR_STORE_UNAVAILABLE",
+    VECTOR_STORE_CONNECTION_ERROR: "RAG_VECTOR_STORE_CONNECTION_ERROR",
+    VECTOR_INDEX_NOT_FOUND: "RAG_VECTOR_INDEX_NOT_FOUND",
+    // Reranking errors
+    RERANKER_ERROR: "RAG_RERANKER_ERROR",
+    RERANKER_NOT_FOUND: "RAG_RERANKER_NOT_FOUND",
+    RERANKER_API_ERROR: "RAG_RERANKER_API_ERROR",
+    // Graph RAG errors
+    GRAPH_RAG_ERROR: "RAG_GRAPH_ERROR",
+    GRAPH_TRAVERSAL_ERROR: "RAG_GRAPH_TRAVERSAL_ERROR",
+    GRAPH_NODE_NOT_FOUND: "RAG_GRAPH_NODE_NOT_FOUND",
+    // Pipeline errors
+    PIPELINE_ERROR: "RAG_PIPELINE_ERROR",
+    PIPELINE_STAGE_FAILED: "RAG_PIPELINE_STAGE_FAILED",
+    PIPELINE_PARTIAL_FAILURE: "RAG_PIPELINE_PARTIAL_FAILURE",
+    // Circuit breaker errors
+    CIRCUIT_BREAKER_OPEN: "RAG_CIRCUIT_BREAKER_OPEN",
+    CIRCUIT_BREAKER_HALF_OPEN_LIMIT: "RAG_CIRCUIT_BREAKER_HALF_OPEN_LIMIT",
+    // General errors
+    OPERATION_TIMEOUT: "RAG_OPERATION_TIMEOUT",
+    RETRY_EXHAUSTED: "RAG_RETRY_EXHAUSTED",
+    INVALID_CONFIGURATION: "RAG_INVALID_CONFIGURATION",
+};
+/**
+ * RAG error factory using the infrastructure pattern
+ */
+export const RAGErrorFactory = createErrorFactory("RAG", RAGErrorCodes);
+/**
+ * Base RAG error class extending NeuroLinkFeatureError
+ */
+export class RAGError extends NeuroLinkFeatureError {
+    constructor(message, code, options) {
+        super(message, code, "RAG", options);
+    }
+}
+/**
+ * Chunking-specific error
+ */
+export class ChunkingError extends RAGError {
+    strategy;
+    contentLength;
+    constructor(message, options) {
+        super(message, options?.code ?? RAGErrorCodes.CHUNKING_ERROR, {
+            retryable: options?.retryable ?? false,
+            details: {
+                ...options?.details,
+                strategy: options?.strategy,
+                contentLength: options?.contentLength,
+            },
+            cause: options?.cause,
+        });
+        this.strategy = options?.strategy;
+        this.contentLength = options?.contentLength;
+    }
+}
+/**
+ * Metadata extraction error
+ */
+export class MetadataExtractionError extends RAGError {
+    extractorType;
+    chunkId;
+    constructor(message, options) {
+        super(message, options?.code ?? RAGErrorCodes.METADATA_EXTRACTION_ERROR, {
+            retryable: options?.retryable ?? true, // LLM-based extraction is often retryable
+            details: {
+                ...options?.details,
+                extractorType: options?.extractorType,
+                chunkId: options?.chunkId,
+            },
+            cause: options?.cause,
+        });
+        this.extractorType = options?.extractorType;
+        this.chunkId = options?.chunkId;
+    }
+}
+/**
+ * Embedding error
+ */
+export class EmbeddingError extends RAGError {
+    provider;
+    batchSize;
+    constructor(message, options) {
+        super(message, options?.code ?? RAGErrorCodes.EMBEDDING_ERROR, {
+            retryable: options?.retryable ?? true, // Embedding operations are typically retryable
+            details: {
+                ...options?.details,
+                provider: options?.provider,
+                batchSize: options?.batchSize,
+            },
+            cause: options?.cause,
+        });
+        this.provider = options?.provider;
+        this.batchSize = options?.batchSize;
+    }
+}
+/**
+ * Vector query error
+ */
+export class VectorQueryError extends RAGError {
+    storeType;
+    queryLength;
+    indexName;
+    constructor(message, options) {
+        super(message, options?.code ?? RAGErrorCodes.VECTOR_QUERY_ERROR, {
+            retryable: options?.retryable ?? true,
+            details: {
+                ...options?.details,
+                storeType: options?.storeType,
+                queryLength: options?.queryLength,
+                indexName: options?.indexName,
+            },
+            cause: options?.cause,
+        });
+        this.storeType = options?.storeType;
+        this.queryLength = options?.queryLength;
+        this.indexName = options?.indexName;
+    }
+}
+/**
+ * Reranker error
+ */
+export class RerankerError extends RAGError {
+    rerankerType;
+    documentCount;
+    constructor(message, options) {
+        super(message, options?.code ?? RAGErrorCodes.RERANKER_ERROR, {
+            retryable: options?.retryable ?? true,
+            details: {
+                ...options?.details,
+                rerankerType: options?.rerankerType,
+                documentCount: options?.documentCount,
+            },
+            cause: options?.cause,
+        });
+        this.rerankerType = options?.rerankerType;
+        this.documentCount = options?.documentCount;
+    }
+}
+/**
+ * Graph RAG error
+ */
+export class GraphRAGError extends RAGError {
+    traversalStrategy;
+    nodeId;
+    constructor(message, options) {
+        super(message, options?.code ?? RAGErrorCodes.GRAPH_RAG_ERROR, {
+            retryable: options?.retryable ?? false,
+            details: {
+                ...options?.details,
+                traversalStrategy: options?.traversalStrategy,
+                nodeId: options?.nodeId,
+            },
+            cause: options?.cause,
+        });
+        this.traversalStrategy = options?.traversalStrategy;
+        this.nodeId = options?.nodeId;
+    }
+}
+/**
+ * Pipeline error with partial failure support
+ */
+export class PipelineError extends RAGError {
+    stageName;
+    successfulChunks;
+    failedChunks;
+    partialResults;
+    constructor(message, options) {
+        super(message, options?.code ?? RAGErrorCodes.PIPELINE_ERROR, {
+            retryable: options?.retryable ?? false,
+            details: {
+                ...options?.details,
+                stageName: options?.stageName,
+                successfulChunks: options?.successfulChunks,
+                failedChunks: options?.failedChunks,
+            },
+            cause: options?.cause,
+        });
+        this.stageName = options?.stageName;
+        this.successfulChunks = options?.successfulChunks;
+        this.failedChunks = options?.failedChunks;
+        this.partialResults = options?.partialResults;
+    }
+    /**
+     * Check if this is a partial failure (some chunks succeeded)
+     */
+    isPartialFailure() {
+        return (this.successfulChunks !== undefined &&
+            this.successfulChunks > 0 &&
+            this.failedChunks !== undefined &&
+            this.failedChunks > 0);
+    }
+    /**
+     * Get success rate as a percentage
+     */
+    getSuccessRate() {
+        if (this.successfulChunks === undefined ||
+            this.failedChunks === undefined) {
+            return 0;
+        }
+        const total = this.successfulChunks + this.failedChunks;
+        return total > 0 ? (this.successfulChunks / total) * 100 : 0;
+    }
+}
+/**
+ * Circuit breaker specific error for RAG operations
+ */
+export class RAGCircuitBreakerError extends RAGError {
+    circuitName;
+    nextRetryTime;
+    constructor(message, options) {
+        super(message, options?.code ?? RAGErrorCodes.CIRCUIT_BREAKER_OPEN, {
+            retryable: false, // Circuit breaker errors are not immediately retryable
+            details: {
+                ...options?.details,
+                circuitName: options?.circuitName,
+                nextRetryTime: options?.nextRetryTime?.toISOString(),
+            },
+        });
+        this.circuitName = options?.circuitName;
+        this.nextRetryTime = options?.nextRetryTime;
+    }
+}
+/**
+ * Type guard for RAG errors
+ */
+export function isRAGError(error) {
+    return error instanceof RAGError;
+}
+/**
+ * Type guard for retryable errors
+ */
+export function isRetryableRAGError(error) {
+    if (!isRAGError(error)) {
+        return false;
+    }
+    return error.retryable;
+}
+/**
+ * Type guard for partial failure errors
+ */
+export function isPartialFailure(error) {
+    return error instanceof PipelineError && error.isPartialFailure();
+}

package/dist/client/rag/errors/index.js

@@ -0,0 +1,6 @@
+/**
+ * RAG Errors Index
+ *
+ * Exports all RAG error types and utilities.
+ */
+export { RAGErrorCodes, RAGErrorFactory, RAGError, ChunkingError, MetadataExtractionError, EmbeddingError, VectorQueryError, RerankerError, GraphRAGError, PipelineError, RAGCircuitBreakerError, isRAGError, isRetryableRAGError, isPartialFailure, } from "./RAGError.js";

package/dist/client/rag/graphRag/graphRAG.js

@@ -0,0 +1,401 @@
+/**
+ * Graph RAG Implementation
+ *
+ * Knowledge graph-based retrieval augmented generation.
+ * Creates semantic relationships between document chunks and uses
+ * random walk with restart for context-aware retrieval.
+ */
+import { randomUUID } from "crypto";
+import { logger } from "../../utils/logger.js";
+/**
+ * Graph-based Retrieval Augmented Generation
+ *
+ * Creates a knowledge graph from document chunks where nodes represent
+ * documents and edges represent semantic relationships based on
+ * embedding similarity.
+ */
+export class GraphRAG {
+    nodes = new Map();
+    edges = new Map();
+    dimension;
+    threshold;
+    constructor(config) {
+        this.dimension = config?.dimension ?? 1536;
+        this.threshold = config?.threshold ?? 0.7;
+    }
+    /**
+     * Create a knowledge graph from document chunks and embeddings
+     *
+     * @param chunks - Array of document chunks
+     * @param embeddings - Corresponding embedding vectors
+     */
+    createGraph(chunks, embeddings) {
+        if (chunks.length !== embeddings.length) {
+            throw new Error("Chunks and embeddings arrays must have the same length");
+        }
+        // Clear existing graph
+        this.nodes.clear();
+        this.edges.clear();
+        // Create nodes
+        const nodeIds = [];
+        for (let i = 0; i < chunks.length; i++) {
+            const id = randomUUID();
+            nodeIds.push(id);
+            this.nodes.set(id, {
+                id,
+                content: chunks[i].text,
+                metadata: chunks[i].metadata || {},
+                embedding: embeddings[i].vector,
+            });
+        }
+        // Create edges based on semantic similarity
+        for (let i = 0; i < nodeIds.length; i++) {
+            const edges = [];
+            const nodeA = this.nodes.get(nodeIds[i]);
+            if (!nodeA?.embedding) {
+                continue;
+            }
+            for (let j = 0; j < nodeIds.length; j++) {
+                if (i === j) {
+                    continue;
+                }
+                const nodeB = this.nodes.get(nodeIds[j]);
+                if (!nodeB?.embedding) {
+                    continue;
+                }
+                const similarity = this.cosineSimilarity(nodeA.embedding, nodeB.embedding);
+                if (similarity >= this.threshold) {
+                    edges.push({
+                        source: nodeIds[i],
+                        target: nodeIds[j],
+                        weight: similarity,
+                        type: "semantic",
+                    });
+                }
+            }
+            // Sort edges by weight descending
+            edges.sort((a, b) => b.weight - a.weight);
+            this.edges.set(nodeIds[i], edges);
+        }
+        logger.info("[GraphRAG] Graph created", {
+            nodes: this.nodes.size,
+            totalEdges: Array.from(this.edges.values()).reduce((sum, e) => sum + e.length, 0),
+            threshold: this.threshold,
+        });
+    }
+    /**
+     * Query the graph using random walk with restart
+     *
+     * @param params - Query parameters including embedding vector
+     * @returns Ranked nodes by relevance
+     */
+    query(params) {
+        const { query, topK = 10, randomWalkSteps = 100, restartProb = 0.15, } = params;
+        if (this.nodes.size === 0) {
+            return [];
+        }
+        // Calculate initial similarities to query
+        const similarities = new Map();
+        for (const [id, node] of this.nodes) {
+            if (node.embedding) {
+                similarities.set(id, this.cosineSimilarity(query, node.embedding));
+            }
+        }
+        // Find starting nodes (most similar to query)
+        const sortedNodes = Array.from(similarities.entries())
+            .sort((a, b) => b[1] - a[1])
+            .slice(0, Math.min(5, this.nodes.size));
+        if (sortedNodes.length === 0) {
+            return [];
+        }
+        // Random walk with restart
+        const visitCounts = new Map();
+        const startNodeIds = sortedNodes.map(([id]) => id);
+        const startProbs = this.normalizeProbs(sortedNodes.map(([, sim]) => sim));
+        for (let step = 0; step < randomWalkSteps; step++) {
+            // Choose starting node based on query similarity
+            const startIdx = this.weightedRandomChoice(startProbs);
+            let currentNode = startNodeIds[startIdx];
+            // Walk with restart probability
+            if (Math.random() >= restartProb) {
+                const edges = this.edges.get(currentNode) || [];
+                if (edges.length > 0) {
+                    // Choose next node based on edge weights
+                    const edgeWeights = edges.map((e) => e.weight);
+                    const normalizedWeights = this.normalizeProbs(edgeWeights);
+                    const nextIdx = this.weightedRandomChoice(normalizedWeights);
+                    currentNode = edges[nextIdx].target;
+                }
+            }
+            // Update visit count
+            visitCounts.set(currentNode, (visitCounts.get(currentNode) || 0) + 1);
+        }
+        // Combine visit frequency with query similarity for final ranking
+        const scores = new Map();
+        const maxVisits = Math.max(...visitCounts.values());
+        for (const [id] of this.nodes) {
+            const visitScore = (visitCounts.get(id) || 0) / maxVisits;
+            const similarityScore = similarities.get(id) || 0;
+            // Weighted combination
+            scores.set(id, 0.6 * similarityScore + 0.4 * visitScore);
+        }
+        // Sort and return top K
+        const rankedNodes = Array.from(scores.entries())
+            .sort((a, b) => b[1] - a[1])
+            .slice(0, topK)
+            .flatMap(([id, score]) => {
+            const node = this.nodes.get(id);
+            if (!node) {
+                return [];
+            }
+            return [
+                {
+                    id,
+                    content: node.content,
+                    metadata: node.metadata,
+                    score,
+                },
+            ];
+        });
+        logger.debug("[GraphRAG] Query completed", {
+            topK,
+            resultsCount: rankedNodes.length,
+            topScore: rankedNodes[0]?.score,
+        });
+        return rankedNodes;
+    }
+    /**
+     * Add a single node to the graph
+     *
+     * @param chunk - Document chunk
+     * @param embedding - Embedding vector
+     * @returns Node ID
+     */
+    addNode(chunk, embedding) {
+        const id = randomUUID();
+        this.nodes.set(id, {
+            id,
+            content: chunk.text,
+            metadata: chunk.metadata || {},
+            embedding: embedding.vector,
+        });
+        // Create edges to existing nodes
+        const edges = [];
+        for (const [existingId, existingNode] of this.nodes) {
+            if (existingId === id) {
+                continue;
+            }
+            if (!existingNode.embedding) {
+                continue;
+            }
+            const similarity = this.cosineSimilarity(embedding.vector, existingNode.embedding);
+            if (similarity >= this.threshold) {
+                edges.push({
+                    source: id,
+                    target: existingId,
+                    weight: similarity,
+                    type: "semantic",
+                });
+                // Add reverse edge
+                const existingEdges = this.edges.get(existingId) || [];
+                existingEdges.push({
+                    source: existingId,
+                    target: id,
+                    weight: similarity,
+                    type: "semantic",
+                });
+                this.edges.set(existingId, existingEdges);
+            }
+        }
+        this.edges.set(id, edges);
+        return id;
+    }
+    /**
+     * Remove a node and its edges from the graph
+     *
+     * @param id - Node ID to remove
+     * @returns True if node was removed
+     */
+    removeNode(id) {
+        if (!this.nodes.has(id)) {
+            return false;
+        }
+        // Remove node
+        this.nodes.delete(id);
+        this.edges.delete(id);
+        // Remove edges pointing to this node
+        for (const [nodeId, edges] of this.edges) {
+            this.edges.set(nodeId, edges.filter((e) => e.target !== id));
+        }
+        return true;
+    }
+    /**
+     * Get graph statistics
+     */
+    getStats() {
+        const edgeCount = Array.from(this.edges.values()).reduce((sum, e) => sum + e.length, 0);
+        return {
+            nodeCount: this.nodes.size,
+            edgeCount,
+            avgDegree: this.nodes.size > 0 ? edgeCount / this.nodes.size : 0,
+            threshold: this.threshold,
+        };
+    }
+    /**
+     * Get a node by ID
+     */
+    getNode(id) {
+        return this.nodes.get(id);
+    }
+    /**
+     * Get all nodes
+     */
+    getAllNodes() {
+        return Array.from(this.nodes.values());
+    }
+    /**
+     * Get edges for a node
+     */
+    getEdges(nodeId) {
+        return this.edges.get(nodeId) || [];
+    }
+    /**
+     * Find connected components in the graph
+     */
+    findConnectedComponents() {
+        const visited = new Set();
+        const components = [];
+        for (const nodeId of this.nodes.keys()) {
+            if (visited.has(nodeId)) {
+                continue;
+            }
+            const component = [];
+            const queue = [nodeId];
+            while (queue.length > 0) {
+                const current = queue.shift();
+                if (current === undefined || visited.has(current)) {
+                    continue;
+                }
+                visited.add(current);
+                component.push(current);
+                const edges = this.edges.get(current) || [];
+                for (const edge of edges) {
+                    if (!visited.has(edge.target)) {
+                        queue.push(edge.target);
+                    }
+                }
+            }
+            components.push(component);
+        }
+        return components;
+    }
+    /**
+     * Calculate cosine similarity between two vectors
+     */
+    cosineSimilarity(a, b) {
+        if (a.length !== b.length) {
+            throw new Error("Vectors must have the same dimension");
+        }
+        let dotProduct = 0;
+        let normA = 0;
+        let normB = 0;
+        for (let i = 0; i < a.length; i++) {
+            dotProduct += a[i] * b[i];
+            normA += a[i] * a[i];
+            normB += b[i] * b[i];
+        }
+        const denominator = Math.sqrt(normA) * Math.sqrt(normB);
+        return denominator === 0 ? 0 : dotProduct / denominator;
+    }
+    /**
+     * Normalize probabilities to sum to 1
+     */
+    normalizeProbs(probs) {
+        const sum = probs.reduce((a, b) => a + b, 0);
+        return sum === 0
+            ? probs.map(() => 1 / probs.length)
+            : probs.map((p) => p / sum);
+    }
+    /**
+     * Weighted random choice
+     */
+    weightedRandomChoice(weights) {
+        const random = Math.random();
+        let cumulative = 0;
+        for (let i = 0; i < weights.length; i++) {
+            cumulative += weights[i];
+            if (random <= cumulative) {
+                return i;
+            }
+        }
+        return weights.length - 1;
+    }
+    /**
+     * Update similarity threshold and rebuild edges
+     */
+    updateThreshold(threshold) {
+        this.threshold = threshold;
+        // Rebuild edges with new threshold
+        this.edges.clear();
+        const nodeIds = Array.from(this.nodes.keys());
+        for (let i = 0; i < nodeIds.length; i++) {
+            const edges = [];
+            const nodeA = this.nodes.get(nodeIds[i]);
+            if (!nodeA?.embedding) {
+                continue;
+            }
+            for (let j = 0; j < nodeIds.length; j++) {
+                if (i === j) {
+                    continue;
+                }
+                const nodeB = this.nodes.get(nodeIds[j]);
+                if (!nodeB?.embedding) {
+                    continue;
+                }
+                const similarity = this.cosineSimilarity(nodeA.embedding, nodeB.embedding);
+                if (similarity >= threshold) {
+                    edges.push({
+                        source: nodeIds[i],
+                        target: nodeIds[j],
+                        weight: similarity,
+                        type: "semantic",
+                    });
+                }
+            }
+            edges.sort((a, b) => b.weight - a.weight);
+            this.edges.set(nodeIds[i], edges);
+        }
+    }
+    /**
+     * Serialize graph to JSON
+     */
+    toJSON() {
+        return {
+            nodes: Array.from(this.nodes.values()),
+            edges: Array.from(this.edges.entries()).map(([source, edges]) => ({
+                source,
+                edges,
+            })),
+            config: {
+                dimension: this.dimension,
+                threshold: this.threshold,
+            },
+        };
+    }
+    /**
+     * Load graph from JSON
+     */
+    static fromJSON(json) {
+        const graph = new GraphRAG({
+            dimension: json.config.dimension,
+            threshold: json.config.threshold,
+        });
+        for (const node of json.nodes) {
+            graph.nodes.set(node.id, node);
+        }
+        for (const { source, edges } of json.edges) {
+            graph.edges.set(source, edges);
+        }
+        return graph;
+    }
+}

package/dist/client/rag/graphRag/index.js

@@ -0,0 +1,4 @@
+/**
+ * Graph RAG Module Exports
+ */
+export { GraphRAG } from "./graphRAG.js";