@juspay/neurolink 9.1.1 → 9.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/README.md +106 -37
- package/dist/agent/directTools.d.ts +11 -11
- package/dist/cli/commands/config.d.ts +6 -6
- package/dist/cli/commands/rag.d.ts +19 -0
- package/dist/cli/commands/rag.js +756 -0
- package/dist/cli/factories/commandFactory.js +146 -83
- package/dist/cli/parser.js +4 -1
- package/dist/core/baseProvider.d.ts +43 -30
- package/dist/core/baseProvider.js +98 -138
- package/dist/core/conversationMemoryFactory.d.ts +2 -2
- package/dist/core/conversationMemoryFactory.js +2 -2
- package/dist/core/conversationMemoryInitializer.d.ts +1 -2
- package/dist/core/conversationMemoryInitializer.js +2 -2
- package/dist/core/infrastructure/baseError.d.ts +21 -0
- package/dist/core/infrastructure/baseError.js +22 -0
- package/dist/core/infrastructure/baseFactory.d.ts +21 -0
- package/dist/core/infrastructure/baseFactory.js +54 -0
- package/dist/core/infrastructure/baseRegistry.d.ts +21 -0
- package/dist/core/infrastructure/baseRegistry.js +49 -0
- package/dist/core/infrastructure/index.d.ts +5 -0
- package/dist/core/infrastructure/index.js +5 -0
- package/dist/core/infrastructure/retry.d.ts +7 -0
- package/dist/core/infrastructure/retry.js +20 -0
- package/dist/core/infrastructure/typedEventEmitter.d.ts +8 -0
- package/dist/core/infrastructure/typedEventEmitter.js +23 -0
- package/dist/core/redisConversationMemoryManager.d.ts +1 -6
- package/dist/core/redisConversationMemoryManager.js +7 -19
- package/dist/factories/providerFactory.d.ts +5 -3
- package/dist/factories/providerFactory.js +31 -24
- package/dist/image-gen/ImageGenService.d.ts +143 -0
- package/dist/image-gen/ImageGenService.js +345 -0
- package/dist/image-gen/imageGenTools.d.ts +126 -0
- package/dist/image-gen/imageGenTools.js +304 -0
- package/dist/image-gen/index.d.ts +46 -0
- package/dist/image-gen/index.js +48 -0
- package/dist/image-gen/types.d.ts +237 -0
- package/dist/image-gen/types.js +24 -0
- package/dist/index.d.ts +46 -12
- package/dist/index.js +88 -36
- package/dist/lib/agent/directTools.d.ts +8 -8
- package/dist/lib/core/baseProvider.d.ts +43 -30
- package/dist/lib/core/baseProvider.js +98 -138
- package/dist/lib/core/conversationMemoryFactory.d.ts +2 -2
- package/dist/lib/core/conversationMemoryFactory.js +2 -2
- package/dist/lib/core/conversationMemoryInitializer.d.ts +1 -2
- package/dist/lib/core/conversationMemoryInitializer.js +2 -2
- package/dist/lib/core/infrastructure/baseError.d.ts +21 -0
- package/dist/lib/core/infrastructure/baseError.js +23 -0
- package/dist/lib/core/infrastructure/baseFactory.d.ts +21 -0
- package/dist/lib/core/infrastructure/baseFactory.js +55 -0
- package/dist/lib/core/infrastructure/baseRegistry.d.ts +21 -0
- package/dist/lib/core/infrastructure/baseRegistry.js +50 -0
- package/dist/lib/core/infrastructure/index.d.ts +5 -0
- package/dist/lib/core/infrastructure/index.js +6 -0
- package/dist/lib/core/infrastructure/retry.d.ts +7 -0
- package/dist/lib/core/infrastructure/retry.js +21 -0
- package/dist/lib/core/infrastructure/typedEventEmitter.d.ts +8 -0
- package/dist/lib/core/infrastructure/typedEventEmitter.js +24 -0
- package/dist/lib/core/redisConversationMemoryManager.d.ts +1 -6
- package/dist/lib/core/redisConversationMemoryManager.js +7 -19
- package/dist/lib/factories/providerFactory.d.ts +5 -3
- package/dist/lib/factories/providerFactory.js +31 -24
- package/dist/lib/image-gen/ImageGenService.d.ts +143 -0
- package/dist/lib/image-gen/ImageGenService.js +346 -0
- package/dist/lib/image-gen/imageGenTools.d.ts +126 -0
- package/dist/lib/image-gen/imageGenTools.js +305 -0
- package/dist/lib/image-gen/index.d.ts +46 -0
- package/dist/lib/image-gen/index.js +49 -0
- package/dist/lib/image-gen/types.d.ts +237 -0
- package/dist/lib/image-gen/types.js +25 -0
- package/dist/lib/index.d.ts +46 -12
- package/dist/lib/index.js +88 -36
- package/dist/lib/mcp/index.d.ts +6 -5
- package/dist/lib/mcp/index.js +7 -5
- package/dist/lib/neurolink.d.ts +11 -13
- package/dist/lib/neurolink.js +95 -29
- package/dist/lib/processors/base/BaseFileProcessor.d.ts +273 -0
- package/dist/lib/processors/base/BaseFileProcessor.js +614 -0
- package/dist/lib/processors/base/index.d.ts +14 -0
- package/dist/lib/processors/base/index.js +20 -0
- package/dist/lib/processors/base/types.d.ts +593 -0
- package/dist/lib/processors/base/types.js +77 -0
- package/dist/lib/processors/cli/fileProcessorCli.d.ts +163 -0
- package/dist/lib/processors/cli/fileProcessorCli.js +389 -0
- package/dist/lib/processors/cli/index.d.ts +37 -0
- package/dist/lib/processors/cli/index.js +50 -0
- package/dist/lib/processors/code/ConfigProcessor.d.ts +171 -0
- package/dist/lib/processors/code/ConfigProcessor.js +401 -0
- package/dist/lib/processors/code/SourceCodeProcessor.d.ts +174 -0
- package/dist/lib/processors/code/SourceCodeProcessor.js +305 -0
- package/dist/lib/processors/code/index.d.ts +44 -0
- package/dist/lib/processors/code/index.js +61 -0
- package/dist/lib/processors/config/fileTypes.d.ts +283 -0
- package/dist/lib/processors/config/fileTypes.js +521 -0
- package/dist/lib/processors/config/index.d.ts +32 -0
- package/dist/lib/processors/config/index.js +93 -0
- package/dist/lib/processors/config/languageMap.d.ts +66 -0
- package/dist/lib/processors/config/languageMap.js +411 -0
- package/dist/lib/processors/config/mimeTypes.d.ts +376 -0
- package/dist/lib/processors/config/mimeTypes.js +339 -0
- package/dist/lib/processors/config/sizeLimits.d.ts +194 -0
- package/dist/lib/processors/config/sizeLimits.js +247 -0
- package/dist/lib/processors/data/JsonProcessor.d.ts +122 -0
- package/dist/lib/processors/data/JsonProcessor.js +204 -0
- package/dist/lib/processors/data/XmlProcessor.d.ts +160 -0
- package/dist/lib/processors/data/XmlProcessor.js +284 -0
- package/dist/lib/processors/data/YamlProcessor.d.ts +163 -0
- package/dist/lib/processors/data/YamlProcessor.js +295 -0
- package/dist/lib/processors/data/index.d.ts +49 -0
- package/dist/lib/processors/data/index.js +77 -0
- package/dist/lib/processors/document/ExcelProcessor.d.ts +238 -0
- package/dist/lib/processors/document/ExcelProcessor.js +520 -0
- package/dist/lib/processors/document/OpenDocumentProcessor.d.ts +69 -0
- package/dist/lib/processors/document/OpenDocumentProcessor.js +211 -0
- package/dist/lib/processors/document/RtfProcessor.d.ts +152 -0
- package/dist/lib/processors/document/RtfProcessor.js +362 -0
- package/dist/lib/processors/document/WordProcessor.d.ts +168 -0
- package/dist/lib/processors/document/WordProcessor.js +354 -0
- package/dist/lib/processors/document/index.d.ts +54 -0
- package/dist/lib/processors/document/index.js +91 -0
- package/dist/lib/processors/errors/FileErrorCode.d.ts +98 -0
- package/dist/lib/processors/errors/FileErrorCode.js +256 -0
- package/dist/lib/processors/errors/errorHelpers.d.ts +151 -0
- package/dist/lib/processors/errors/errorHelpers.js +379 -0
- package/dist/lib/processors/errors/errorSerializer.d.ts +139 -0
- package/dist/lib/processors/errors/errorSerializer.js +508 -0
- package/dist/lib/processors/errors/index.d.ts +46 -0
- package/dist/lib/processors/errors/index.js +50 -0
- package/dist/lib/processors/index.d.ts +76 -0
- package/dist/lib/processors/index.js +113 -0
- package/dist/lib/processors/integration/FileProcessorIntegration.d.ts +244 -0
- package/dist/lib/processors/integration/FileProcessorIntegration.js +273 -0
- package/dist/lib/processors/integration/index.d.ts +42 -0
- package/dist/lib/processors/integration/index.js +45 -0
- package/dist/lib/processors/markup/HtmlProcessor.d.ts +169 -0
- package/dist/lib/processors/markup/HtmlProcessor.js +250 -0
- package/dist/lib/processors/markup/MarkdownProcessor.d.ts +165 -0
- package/dist/lib/processors/markup/MarkdownProcessor.js +245 -0
- package/dist/lib/processors/markup/SvgProcessor.d.ts +156 -0
- package/dist/lib/processors/markup/SvgProcessor.js +241 -0
- package/dist/lib/processors/markup/TextProcessor.d.ts +135 -0
- package/dist/lib/processors/markup/TextProcessor.js +189 -0
- package/dist/lib/processors/markup/index.d.ts +66 -0
- package/dist/lib/processors/markup/index.js +103 -0
- package/dist/lib/processors/registry/ProcessorRegistry.d.ts +334 -0
- package/dist/lib/processors/registry/ProcessorRegistry.js +609 -0
- package/dist/lib/processors/registry/index.d.ts +12 -0
- package/dist/lib/processors/registry/index.js +17 -0
- package/dist/lib/processors/registry/types.d.ts +53 -0
- package/dist/lib/processors/registry/types.js +11 -0
- package/dist/lib/providers/amazonBedrock.d.ts +15 -2
- package/dist/lib/providers/amazonBedrock.js +65 -8
- package/dist/lib/providers/anthropic.d.ts +3 -3
- package/dist/lib/providers/anthropic.js +10 -7
- package/dist/lib/providers/googleAiStudio.d.ts +5 -5
- package/dist/lib/providers/googleAiStudio.js +10 -7
- package/dist/lib/providers/googleVertex.d.ts +16 -4
- package/dist/lib/providers/googleVertex.js +72 -16
- package/dist/lib/providers/litellm.d.ts +3 -3
- package/dist/lib/providers/litellm.js +10 -10
- package/dist/lib/providers/mistral.d.ts +3 -3
- package/dist/lib/providers/mistral.js +7 -6
- package/dist/lib/providers/ollama.d.ts +3 -4
- package/dist/lib/providers/ollama.js +7 -8
- package/dist/lib/providers/openAI.d.ts +14 -2
- package/dist/lib/providers/openAI.js +60 -6
- package/dist/lib/providers/openRouter.d.ts +2 -2
- package/dist/lib/providers/openRouter.js +10 -6
- package/dist/lib/providers/sagemaker/language-model.d.ts +2 -2
- package/dist/lib/rag/ChunkerFactory.d.ts +91 -0
- package/dist/lib/rag/ChunkerFactory.js +321 -0
- package/dist/lib/rag/ChunkerRegistry.d.ts +91 -0
- package/dist/lib/rag/ChunkerRegistry.js +422 -0
- package/dist/lib/rag/chunkers/BaseChunker.d.ts +53 -0
- package/dist/lib/rag/chunkers/BaseChunker.js +144 -0
- package/dist/lib/rag/chunkers/CharacterChunker.d.ts +18 -0
- package/dist/lib/rag/chunkers/CharacterChunker.js +29 -0
- package/dist/lib/rag/chunkers/HTMLChunker.d.ts +19 -0
- package/dist/lib/rag/chunkers/HTMLChunker.js +39 -0
- package/dist/lib/rag/chunkers/JSONChunker.d.ts +19 -0
- package/dist/lib/rag/chunkers/JSONChunker.js +69 -0
- package/dist/lib/rag/chunkers/LaTeXChunker.d.ts +15 -0
- package/dist/lib/rag/chunkers/LaTeXChunker.js +64 -0
- package/dist/lib/rag/chunkers/MarkdownChunker.d.ts +15 -0
- package/dist/lib/rag/chunkers/MarkdownChunker.js +103 -0
- package/dist/lib/rag/chunkers/RecursiveChunker.d.ts +27 -0
- package/dist/lib/rag/chunkers/RecursiveChunker.js +140 -0
- package/dist/lib/rag/chunkers/SemanticMarkdownChunker.d.ts +22 -0
- package/dist/lib/rag/chunkers/SemanticMarkdownChunker.js +139 -0
- package/dist/lib/rag/chunkers/SentenceChunker.d.ts +19 -0
- package/dist/lib/rag/chunkers/SentenceChunker.js +67 -0
- package/dist/lib/rag/chunkers/TokenChunker.d.ts +19 -0
- package/dist/lib/rag/chunkers/TokenChunker.js +62 -0
- package/dist/lib/rag/chunkers/index.d.ts +15 -0
- package/dist/lib/rag/chunkers/index.js +16 -0
- package/dist/lib/rag/chunking/characterChunker.d.ts +16 -0
- package/dist/lib/rag/chunking/characterChunker.js +143 -0
- package/dist/lib/rag/chunking/chunkerRegistry.d.ts +67 -0
- package/dist/lib/rag/chunking/chunkerRegistry.js +195 -0
- package/dist/lib/rag/chunking/htmlChunker.d.ts +34 -0
- package/dist/lib/rag/chunking/htmlChunker.js +248 -0
- package/dist/lib/rag/chunking/index.d.ts +15 -0
- package/dist/lib/rag/chunking/index.js +18 -0
- package/dist/lib/rag/chunking/jsonChunker.d.ts +20 -0
- package/dist/lib/rag/chunking/jsonChunker.js +282 -0
- package/dist/lib/rag/chunking/latexChunker.d.ts +26 -0
- package/dist/lib/rag/chunking/latexChunker.js +252 -0
- package/dist/lib/rag/chunking/markdownChunker.d.ts +19 -0
- package/dist/lib/rag/chunking/markdownChunker.js +202 -0
- package/dist/lib/rag/chunking/recursiveChunker.d.ts +19 -0
- package/dist/lib/rag/chunking/recursiveChunker.js +149 -0
- package/dist/lib/rag/chunking/semanticChunker.d.ts +41 -0
- package/dist/lib/rag/chunking/semanticChunker.js +307 -0
- package/dist/lib/rag/chunking/sentenceChunker.d.ts +25 -0
- package/dist/lib/rag/chunking/sentenceChunker.js +231 -0
- package/dist/lib/rag/chunking/tokenChunker.d.ts +36 -0
- package/dist/lib/rag/chunking/tokenChunker.js +184 -0
- package/dist/lib/rag/document/MDocument.d.ts +198 -0
- package/dist/lib/rag/document/MDocument.js +393 -0
- package/dist/lib/rag/document/index.d.ts +5 -0
- package/dist/lib/rag/document/index.js +6 -0
- package/dist/lib/rag/document/loaders.d.ts +201 -0
- package/dist/lib/rag/document/loaders.js +501 -0
- package/dist/lib/rag/errors/RAGError.d.ts +244 -0
- package/dist/lib/rag/errors/RAGError.js +275 -0
- package/dist/lib/rag/errors/index.d.ts +6 -0
- package/dist/lib/rag/errors/index.js +7 -0
- package/dist/lib/rag/graphRag/graphRAG.d.ts +115 -0
- package/dist/lib/rag/graphRag/graphRAG.js +385 -0
- package/dist/lib/rag/graphRag/index.d.ts +4 -0
- package/dist/lib/rag/graphRag/index.js +5 -0
- package/dist/lib/rag/index.d.ts +103 -0
- package/dist/lib/rag/index.js +142 -0
- package/dist/lib/rag/metadata/MetadataExtractorFactory.d.ts +157 -0
- package/dist/lib/rag/metadata/MetadataExtractorFactory.js +419 -0
- package/dist/lib/rag/metadata/MetadataExtractorRegistry.d.ts +99 -0
- package/dist/lib/rag/metadata/MetadataExtractorRegistry.js +363 -0
- package/dist/lib/rag/metadata/index.d.ts +6 -0
- package/dist/lib/rag/metadata/index.js +10 -0
- package/dist/lib/rag/metadata/metadataExtractor.d.ts +69 -0
- package/dist/lib/rag/metadata/metadataExtractor.js +278 -0
- package/dist/lib/rag/pipeline/RAGPipeline.d.ts +235 -0
- package/dist/lib/rag/pipeline/RAGPipeline.js +402 -0
- package/dist/lib/rag/pipeline/contextAssembly.d.ts +126 -0
- package/dist/lib/rag/pipeline/contextAssembly.js +338 -0
- package/dist/lib/rag/pipeline/index.d.ts +5 -0
- package/dist/lib/rag/pipeline/index.js +6 -0
- package/dist/lib/rag/ragIntegration.d.ts +38 -0
- package/dist/lib/rag/ragIntegration.js +212 -0
- package/dist/lib/rag/reranker/RerankerFactory.d.ts +184 -0
- package/dist/lib/rag/reranker/RerankerFactory.js +431 -0
- package/dist/lib/rag/reranker/RerankerRegistry.d.ts +119 -0
- package/dist/lib/rag/reranker/RerankerRegistry.js +403 -0
- package/dist/lib/rag/reranker/index.d.ts +6 -0
- package/dist/lib/rag/reranker/index.js +10 -0
- package/dist/lib/rag/reranker/reranker.d.ts +71 -0
- package/dist/lib/rag/reranker/reranker.js +278 -0
- package/dist/lib/rag/resilience/CircuitBreaker.d.ts +215 -0
- package/dist/lib/rag/resilience/CircuitBreaker.js +432 -0
- package/dist/lib/rag/resilience/RetryHandler.d.ts +115 -0
- package/dist/lib/rag/resilience/RetryHandler.js +301 -0
- package/dist/lib/rag/resilience/index.d.ts +7 -0
- package/dist/lib/rag/resilience/index.js +8 -0
- package/dist/lib/rag/retrieval/hybridSearch.d.ts +94 -0
- package/dist/lib/rag/retrieval/hybridSearch.js +314 -0
- package/dist/lib/rag/retrieval/index.d.ts +5 -0
- package/dist/lib/rag/retrieval/index.js +6 -0
- package/dist/lib/rag/retrieval/vectorQueryTool.d.ts +93 -0
- package/dist/lib/rag/retrieval/vectorQueryTool.js +290 -0
- package/dist/lib/rag/types.d.ts +768 -0
- package/dist/lib/rag/types.js +9 -0
- package/dist/lib/server/index.d.ts +15 -11
- package/dist/lib/server/index.js +55 -51
- package/dist/lib/server/utils/validation.d.ts +2 -2
- package/dist/lib/types/common.d.ts +0 -1
- package/dist/lib/types/fileTypes.d.ts +1 -1
- package/dist/lib/types/generateTypes.d.ts +42 -8
- package/dist/lib/types/generateTypes.js +1 -1
- package/dist/lib/types/index.d.ts +25 -24
- package/dist/lib/types/index.js +21 -20
- package/dist/lib/types/modelTypes.d.ts +16 -16
- package/dist/lib/types/pptTypes.d.ts +14 -2
- package/dist/lib/types/pptTypes.js +16 -0
- package/dist/lib/types/streamTypes.d.ts +28 -8
- package/dist/lib/types/streamTypes.js +1 -1
- package/dist/lib/utils/async/delay.d.ts +40 -0
- package/dist/lib/utils/async/delay.js +43 -0
- package/dist/lib/utils/async/index.d.ts +23 -0
- package/dist/lib/utils/async/index.js +24 -0
- package/dist/lib/utils/async/retry.d.ts +141 -0
- package/dist/lib/utils/async/retry.js +172 -0
- package/dist/lib/utils/async/withTimeout.d.ts +73 -0
- package/dist/lib/utils/async/withTimeout.js +97 -0
- package/dist/lib/utils/fileDetector.d.ts +7 -1
- package/dist/lib/utils/fileDetector.js +91 -18
- package/dist/lib/utils/json/extract.d.ts +103 -0
- package/dist/lib/utils/json/extract.js +249 -0
- package/dist/lib/utils/json/index.d.ts +36 -0
- package/dist/lib/utils/json/index.js +37 -0
- package/dist/lib/utils/json/safeParse.d.ts +137 -0
- package/dist/lib/utils/json/safeParse.js +191 -0
- package/dist/lib/utils/messageBuilder.d.ts +2 -2
- package/dist/lib/utils/messageBuilder.js +15 -7
- package/dist/lib/utils/modelRouter.d.ts +4 -4
- package/dist/lib/utils/modelRouter.js +4 -4
- package/dist/lib/utils/sanitizers/filename.d.ts +137 -0
- package/dist/lib/utils/sanitizers/filename.js +366 -0
- package/dist/lib/utils/sanitizers/html.d.ts +170 -0
- package/dist/lib/utils/sanitizers/html.js +326 -0
- package/dist/lib/utils/sanitizers/index.d.ts +26 -0
- package/dist/lib/utils/sanitizers/index.js +30 -0
- package/dist/lib/utils/sanitizers/svg.d.ts +81 -0
- package/dist/lib/utils/sanitizers/svg.js +483 -0
- package/dist/mcp/index.d.ts +6 -5
- package/dist/mcp/index.js +7 -5
- package/dist/neurolink.d.ts +11 -13
- package/dist/neurolink.js +95 -29
- package/dist/processors/base/BaseFileProcessor.d.ts +273 -0
- package/dist/processors/base/BaseFileProcessor.js +613 -0
- package/dist/processors/base/index.d.ts +14 -0
- package/dist/processors/base/index.js +19 -0
- package/dist/processors/base/types.d.ts +593 -0
- package/dist/processors/base/types.js +76 -0
- package/dist/processors/cli/fileProcessorCli.d.ts +163 -0
- package/dist/processors/cli/fileProcessorCli.js +388 -0
- package/dist/processors/cli/index.d.ts +37 -0
- package/dist/processors/cli/index.js +49 -0
- package/dist/processors/code/ConfigProcessor.d.ts +171 -0
- package/dist/processors/code/ConfigProcessor.js +400 -0
- package/dist/processors/code/SourceCodeProcessor.d.ts +174 -0
- package/dist/processors/code/SourceCodeProcessor.js +304 -0
- package/dist/processors/code/index.d.ts +44 -0
- package/dist/processors/code/index.js +60 -0
- package/dist/processors/config/fileTypes.d.ts +283 -0
- package/dist/processors/config/fileTypes.js +520 -0
- package/dist/processors/config/index.d.ts +32 -0
- package/dist/processors/config/index.js +92 -0
- package/dist/processors/config/languageMap.d.ts +66 -0
- package/dist/processors/config/languageMap.js +410 -0
- package/dist/processors/config/mimeTypes.d.ts +376 -0
- package/dist/processors/config/mimeTypes.js +338 -0
- package/dist/processors/config/sizeLimits.d.ts +194 -0
- package/dist/processors/config/sizeLimits.js +246 -0
- package/dist/processors/data/JsonProcessor.d.ts +122 -0
- package/dist/processors/data/JsonProcessor.js +203 -0
- package/dist/processors/data/XmlProcessor.d.ts +160 -0
- package/dist/processors/data/XmlProcessor.js +283 -0
- package/dist/processors/data/YamlProcessor.d.ts +163 -0
- package/dist/processors/data/YamlProcessor.js +294 -0
- package/dist/processors/data/index.d.ts +49 -0
- package/dist/processors/data/index.js +76 -0
- package/dist/processors/document/ExcelProcessor.d.ts +238 -0
- package/dist/processors/document/ExcelProcessor.js +519 -0
- package/dist/processors/document/OpenDocumentProcessor.d.ts +69 -0
- package/dist/processors/document/OpenDocumentProcessor.js +210 -0
- package/dist/processors/document/RtfProcessor.d.ts +152 -0
- package/dist/processors/document/RtfProcessor.js +361 -0
- package/dist/processors/document/WordProcessor.d.ts +168 -0
- package/dist/processors/document/WordProcessor.js +353 -0
- package/dist/processors/document/index.d.ts +54 -0
- package/dist/processors/document/index.js +90 -0
- package/dist/processors/errors/FileErrorCode.d.ts +98 -0
- package/dist/processors/errors/FileErrorCode.js +255 -0
- package/dist/processors/errors/errorHelpers.d.ts +151 -0
- package/dist/processors/errors/errorHelpers.js +378 -0
- package/dist/processors/errors/errorSerializer.d.ts +139 -0
- package/dist/processors/errors/errorSerializer.js +507 -0
- package/dist/processors/errors/index.d.ts +46 -0
- package/dist/processors/errors/index.js +49 -0
- package/dist/processors/index.d.ts +76 -0
- package/dist/processors/index.js +112 -0
- package/dist/processors/integration/FileProcessorIntegration.d.ts +244 -0
- package/dist/processors/integration/FileProcessorIntegration.js +272 -0
- package/dist/processors/integration/index.d.ts +42 -0
- package/dist/processors/integration/index.js +44 -0
- package/dist/processors/markup/HtmlProcessor.d.ts +169 -0
- package/dist/processors/markup/HtmlProcessor.js +249 -0
- package/dist/processors/markup/MarkdownProcessor.d.ts +165 -0
- package/dist/processors/markup/MarkdownProcessor.js +244 -0
- package/dist/processors/markup/SvgProcessor.d.ts +156 -0
- package/dist/processors/markup/SvgProcessor.js +240 -0
- package/dist/processors/markup/TextProcessor.d.ts +135 -0
- package/dist/processors/markup/TextProcessor.js +188 -0
- package/dist/processors/markup/index.d.ts +66 -0
- package/dist/processors/markup/index.js +102 -0
- package/dist/processors/registry/ProcessorRegistry.d.ts +334 -0
- package/dist/processors/registry/ProcessorRegistry.js +608 -0
- package/dist/processors/registry/index.d.ts +12 -0
- package/dist/processors/registry/index.js +16 -0
- package/dist/processors/registry/types.d.ts +53 -0
- package/dist/processors/registry/types.js +10 -0
- package/dist/providers/amazonBedrock.d.ts +15 -2
- package/dist/providers/amazonBedrock.js +65 -8
- package/dist/providers/anthropic.d.ts +3 -3
- package/dist/providers/anthropic.js +10 -7
- package/dist/providers/googleAiStudio.d.ts +5 -5
- package/dist/providers/googleAiStudio.js +10 -7
- package/dist/providers/googleVertex.d.ts +16 -4
- package/dist/providers/googleVertex.js +72 -16
- package/dist/providers/litellm.d.ts +3 -3
- package/dist/providers/litellm.js +10 -10
- package/dist/providers/mistral.d.ts +3 -3
- package/dist/providers/mistral.js +7 -6
- package/dist/providers/ollama.d.ts +3 -4
- package/dist/providers/ollama.js +7 -8
- package/dist/providers/openAI.d.ts +14 -2
- package/dist/providers/openAI.js +60 -6
- package/dist/providers/openRouter.d.ts +2 -2
- package/dist/providers/openRouter.js +10 -6
- package/dist/rag/ChunkerFactory.d.ts +91 -0
- package/dist/rag/ChunkerFactory.js +320 -0
- package/dist/rag/ChunkerRegistry.d.ts +91 -0
- package/dist/rag/ChunkerRegistry.js +421 -0
- package/dist/rag/chunkers/BaseChunker.d.ts +53 -0
- package/dist/rag/chunkers/BaseChunker.js +143 -0
- package/dist/rag/chunkers/CharacterChunker.d.ts +18 -0
- package/dist/rag/chunkers/CharacterChunker.js +28 -0
- package/dist/rag/chunkers/HTMLChunker.d.ts +19 -0
- package/dist/rag/chunkers/HTMLChunker.js +38 -0
- package/dist/rag/chunkers/JSONChunker.d.ts +19 -0
- package/dist/rag/chunkers/JSONChunker.js +68 -0
- package/dist/rag/chunkers/LaTeXChunker.d.ts +15 -0
- package/dist/rag/chunkers/LaTeXChunker.js +63 -0
- package/dist/rag/chunkers/MarkdownChunker.d.ts +15 -0
- package/dist/rag/chunkers/MarkdownChunker.js +102 -0
- package/dist/rag/chunkers/RecursiveChunker.d.ts +27 -0
- package/dist/rag/chunkers/RecursiveChunker.js +139 -0
- package/dist/rag/chunkers/SemanticMarkdownChunker.d.ts +22 -0
- package/dist/rag/chunkers/SemanticMarkdownChunker.js +138 -0
- package/dist/rag/chunkers/SentenceChunker.d.ts +19 -0
- package/dist/rag/chunkers/SentenceChunker.js +66 -0
- package/dist/rag/chunkers/TokenChunker.d.ts +19 -0
- package/dist/rag/chunkers/TokenChunker.js +61 -0
- package/dist/rag/chunkers/index.d.ts +15 -0
- package/dist/rag/chunkers/index.js +15 -0
- package/dist/rag/chunking/characterChunker.d.ts +16 -0
- package/dist/rag/chunking/characterChunker.js +142 -0
- package/dist/rag/chunking/chunkerRegistry.d.ts +67 -0
- package/dist/rag/chunking/chunkerRegistry.js +194 -0
- package/dist/rag/chunking/htmlChunker.d.ts +34 -0
- package/dist/rag/chunking/htmlChunker.js +247 -0
- package/dist/rag/chunking/index.d.ts +15 -0
- package/dist/rag/chunking/index.js +17 -0
- package/dist/rag/chunking/jsonChunker.d.ts +20 -0
- package/dist/rag/chunking/jsonChunker.js +281 -0
- package/dist/rag/chunking/latexChunker.d.ts +26 -0
- package/dist/rag/chunking/latexChunker.js +251 -0
- package/dist/rag/chunking/markdownChunker.d.ts +19 -0
- package/dist/rag/chunking/markdownChunker.js +201 -0
- package/dist/rag/chunking/recursiveChunker.d.ts +19 -0
- package/dist/rag/chunking/recursiveChunker.js +148 -0
- package/dist/rag/chunking/semanticChunker.d.ts +41 -0
- package/dist/rag/chunking/semanticChunker.js +306 -0
- package/dist/rag/chunking/sentenceChunker.d.ts +25 -0
- package/dist/rag/chunking/sentenceChunker.js +230 -0
- package/dist/rag/chunking/tokenChunker.d.ts +36 -0
- package/dist/rag/chunking/tokenChunker.js +183 -0
- package/dist/rag/document/MDocument.d.ts +198 -0
- package/dist/rag/document/MDocument.js +392 -0
- package/dist/rag/document/index.d.ts +5 -0
- package/dist/rag/document/index.js +5 -0
- package/dist/rag/document/loaders.d.ts +201 -0
- package/dist/rag/document/loaders.js +500 -0
- package/dist/rag/errors/RAGError.d.ts +244 -0
- package/dist/rag/errors/RAGError.js +274 -0
- package/dist/rag/errors/index.d.ts +6 -0
- package/dist/rag/errors/index.js +6 -0
- package/dist/rag/graphRag/graphRAG.d.ts +115 -0
- package/dist/rag/graphRag/graphRAG.js +384 -0
- package/dist/rag/graphRag/index.d.ts +4 -0
- package/dist/rag/graphRag/index.js +4 -0
- package/dist/rag/index.d.ts +103 -0
- package/dist/rag/index.js +141 -0
- package/dist/rag/metadata/MetadataExtractorFactory.d.ts +157 -0
- package/dist/rag/metadata/MetadataExtractorFactory.js +418 -0
- package/dist/rag/metadata/MetadataExtractorRegistry.d.ts +99 -0
- package/dist/rag/metadata/MetadataExtractorRegistry.js +362 -0
- package/dist/rag/metadata/index.d.ts +6 -0
- package/dist/rag/metadata/index.js +9 -0
- package/dist/rag/metadata/metadataExtractor.d.ts +69 -0
- package/dist/rag/metadata/metadataExtractor.js +277 -0
- package/dist/rag/pipeline/RAGPipeline.d.ts +235 -0
- package/dist/rag/pipeline/RAGPipeline.js +401 -0
- package/dist/rag/pipeline/contextAssembly.d.ts +126 -0
- package/dist/rag/pipeline/contextAssembly.js +337 -0
- package/dist/rag/pipeline/index.d.ts +5 -0
- package/dist/rag/pipeline/index.js +5 -0
- package/dist/rag/ragIntegration.d.ts +38 -0
- package/dist/rag/ragIntegration.js +211 -0
- package/dist/rag/reranker/RerankerFactory.d.ts +184 -0
- package/dist/rag/reranker/RerankerFactory.js +430 -0
- package/dist/rag/reranker/RerankerRegistry.d.ts +119 -0
- package/dist/rag/reranker/RerankerRegistry.js +402 -0
- package/dist/rag/reranker/index.d.ts +6 -0
- package/dist/rag/reranker/index.js +9 -0
- package/dist/rag/reranker/reranker.d.ts +71 -0
- package/dist/rag/reranker/reranker.js +277 -0
- package/dist/rag/resilience/CircuitBreaker.d.ts +215 -0
- package/dist/rag/resilience/CircuitBreaker.js +431 -0
- package/dist/rag/resilience/RetryHandler.d.ts +115 -0
- package/dist/rag/resilience/RetryHandler.js +300 -0
- package/dist/rag/resilience/index.d.ts +7 -0
- package/dist/rag/resilience/index.js +7 -0
- package/dist/rag/retrieval/hybridSearch.d.ts +94 -0
- package/dist/rag/retrieval/hybridSearch.js +313 -0
- package/dist/rag/retrieval/index.d.ts +5 -0
- package/dist/rag/retrieval/index.js +5 -0
- package/dist/rag/retrieval/vectorQueryTool.d.ts +93 -0
- package/dist/rag/retrieval/vectorQueryTool.js +289 -0
- package/dist/rag/types.d.ts +768 -0
- package/dist/rag/types.js +8 -0
- package/dist/server/index.d.ts +15 -11
- package/dist/server/index.js +55 -51
- package/dist/server/utils/validation.d.ts +8 -8
- package/dist/types/common.d.ts +0 -1
- package/dist/types/fileTypes.d.ts +1 -1
- package/dist/types/generateTypes.d.ts +42 -8
- package/dist/types/generateTypes.js +1 -1
- package/dist/types/index.d.ts +25 -24
- package/dist/types/index.js +21 -20
- package/dist/types/modelTypes.d.ts +10 -10
- package/dist/types/pptTypes.d.ts +14 -2
- package/dist/types/pptTypes.js +16 -0
- package/dist/types/streamTypes.d.ts +28 -8
- package/dist/types/streamTypes.js +1 -1
- package/dist/utils/async/delay.d.ts +40 -0
- package/dist/utils/async/delay.js +42 -0
- package/dist/utils/async/index.d.ts +23 -0
- package/dist/utils/async/index.js +23 -0
- package/dist/utils/async/retry.d.ts +141 -0
- package/dist/utils/async/retry.js +171 -0
- package/dist/utils/async/withTimeout.d.ts +73 -0
- package/dist/utils/async/withTimeout.js +96 -0
- package/dist/utils/fileDetector.d.ts +7 -1
- package/dist/utils/fileDetector.js +91 -18
- package/dist/utils/json/extract.d.ts +103 -0
- package/dist/utils/json/extract.js +248 -0
- package/dist/utils/json/index.d.ts +36 -0
- package/dist/utils/json/index.js +36 -0
- package/dist/utils/json/safeParse.d.ts +137 -0
- package/dist/utils/json/safeParse.js +190 -0
- package/dist/utils/messageBuilder.d.ts +2 -2
- package/dist/utils/messageBuilder.js +15 -7
- package/dist/utils/modelRouter.d.ts +4 -4
- package/dist/utils/modelRouter.js +4 -4
- package/dist/utils/sanitizers/filename.d.ts +137 -0
- package/dist/utils/sanitizers/filename.js +365 -0
- package/dist/utils/sanitizers/html.d.ts +170 -0
- package/dist/utils/sanitizers/html.js +325 -0
- package/dist/utils/sanitizers/index.d.ts +26 -0
- package/dist/utils/sanitizers/index.js +29 -0
- package/dist/utils/sanitizers/svg.d.ts +81 -0
- package/dist/utils/sanitizers/svg.js +482 -0
- package/package.json +2 -2
|
@@ -0,0 +1,385 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Graph RAG Implementation
|
|
3
|
+
*
|
|
4
|
+
* Knowledge graph-based retrieval augmented generation.
|
|
5
|
+
* Creates semantic relationships between document chunks and uses
|
|
6
|
+
* random walk with restart for context-aware retrieval.
|
|
7
|
+
*/
|
|
8
|
+
import { randomUUID } from "crypto";
|
|
9
|
+
import { logger } from "../../utils/logger.js";
|
|
10
|
+
/**
|
|
11
|
+
* Graph-based Retrieval Augmented Generation
|
|
12
|
+
*
|
|
13
|
+
* Creates a knowledge graph from document chunks where nodes represent
|
|
14
|
+
* documents and edges represent semantic relationships based on
|
|
15
|
+
* embedding similarity.
|
|
16
|
+
*/
|
|
17
|
+
export class GraphRAG {
|
|
18
|
+
nodes = new Map();
|
|
19
|
+
edges = new Map();
|
|
20
|
+
dimension;
|
|
21
|
+
threshold;
|
|
22
|
+
constructor(config) {
|
|
23
|
+
this.dimension = config?.dimension ?? 1536;
|
|
24
|
+
this.threshold = config?.threshold ?? 0.7;
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Create a knowledge graph from document chunks and embeddings
|
|
28
|
+
*
|
|
29
|
+
* @param chunks - Array of document chunks
|
|
30
|
+
* @param embeddings - Corresponding embedding vectors
|
|
31
|
+
*/
|
|
32
|
+
createGraph(chunks, embeddings) {
|
|
33
|
+
if (chunks.length !== embeddings.length) {
|
|
34
|
+
throw new Error("Chunks and embeddings arrays must have the same length");
|
|
35
|
+
}
|
|
36
|
+
// Clear existing graph
|
|
37
|
+
this.nodes.clear();
|
|
38
|
+
this.edges.clear();
|
|
39
|
+
// Create nodes
|
|
40
|
+
const nodeIds = [];
|
|
41
|
+
for (let i = 0; i < chunks.length; i++) {
|
|
42
|
+
const id = randomUUID();
|
|
43
|
+
nodeIds.push(id);
|
|
44
|
+
this.nodes.set(id, {
|
|
45
|
+
id,
|
|
46
|
+
content: chunks[i].text,
|
|
47
|
+
metadata: chunks[i].metadata || {},
|
|
48
|
+
embedding: embeddings[i].vector,
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
// Create edges based on semantic similarity
|
|
52
|
+
for (let i = 0; i < nodeIds.length; i++) {
|
|
53
|
+
const edges = [];
|
|
54
|
+
const nodeA = this.nodes.get(nodeIds[i]);
|
|
55
|
+
for (let j = 0; j < nodeIds.length; j++) {
|
|
56
|
+
if (i === j) {
|
|
57
|
+
continue;
|
|
58
|
+
}
|
|
59
|
+
const nodeB = this.nodes.get(nodeIds[j]);
|
|
60
|
+
const similarity = this.cosineSimilarity(nodeA.embedding, nodeB.embedding);
|
|
61
|
+
if (similarity >= this.threshold) {
|
|
62
|
+
edges.push({
|
|
63
|
+
source: nodeIds[i],
|
|
64
|
+
target: nodeIds[j],
|
|
65
|
+
weight: similarity,
|
|
66
|
+
type: "semantic",
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
// Sort edges by weight descending
|
|
71
|
+
edges.sort((a, b) => b.weight - a.weight);
|
|
72
|
+
this.edges.set(nodeIds[i], edges);
|
|
73
|
+
}
|
|
74
|
+
logger.info("[GraphRAG] Graph created", {
|
|
75
|
+
nodes: this.nodes.size,
|
|
76
|
+
totalEdges: Array.from(this.edges.values()).reduce((sum, e) => sum + e.length, 0),
|
|
77
|
+
threshold: this.threshold,
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Query the graph using random walk with restart
|
|
82
|
+
*
|
|
83
|
+
* @param params - Query parameters including embedding vector
|
|
84
|
+
* @returns Ranked nodes by relevance
|
|
85
|
+
*/
|
|
86
|
+
query(params) {
|
|
87
|
+
const { query, topK = 10, randomWalkSteps = 100, restartProb = 0.15, } = params;
|
|
88
|
+
if (this.nodes.size === 0) {
|
|
89
|
+
return [];
|
|
90
|
+
}
|
|
91
|
+
// Calculate initial similarities to query
|
|
92
|
+
const similarities = new Map();
|
|
93
|
+
for (const [id, node] of this.nodes) {
|
|
94
|
+
if (node.embedding) {
|
|
95
|
+
similarities.set(id, this.cosineSimilarity(query, node.embedding));
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
// Find starting nodes (most similar to query)
|
|
99
|
+
const sortedNodes = Array.from(similarities.entries())
|
|
100
|
+
.sort((a, b) => b[1] - a[1])
|
|
101
|
+
.slice(0, Math.min(5, this.nodes.size));
|
|
102
|
+
if (sortedNodes.length === 0) {
|
|
103
|
+
return [];
|
|
104
|
+
}
|
|
105
|
+
// Random walk with restart
|
|
106
|
+
const visitCounts = new Map();
|
|
107
|
+
const startNodeIds = sortedNodes.map(([id]) => id);
|
|
108
|
+
const startProbs = this.normalizeProbs(sortedNodes.map(([, sim]) => sim));
|
|
109
|
+
for (let step = 0; step < randomWalkSteps; step++) {
|
|
110
|
+
// Choose starting node based on query similarity
|
|
111
|
+
const startIdx = this.weightedRandomChoice(startProbs);
|
|
112
|
+
let currentNode = startNodeIds[startIdx];
|
|
113
|
+
// Walk with restart probability
|
|
114
|
+
if (Math.random() >= restartProb) {
|
|
115
|
+
const edges = this.edges.get(currentNode) || [];
|
|
116
|
+
if (edges.length > 0) {
|
|
117
|
+
// Choose next node based on edge weights
|
|
118
|
+
const edgeWeights = edges.map((e) => e.weight);
|
|
119
|
+
const normalizedWeights = this.normalizeProbs(edgeWeights);
|
|
120
|
+
const nextIdx = this.weightedRandomChoice(normalizedWeights);
|
|
121
|
+
currentNode = edges[nextIdx].target;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
// Update visit count
|
|
125
|
+
visitCounts.set(currentNode, (visitCounts.get(currentNode) || 0) + 1);
|
|
126
|
+
}
|
|
127
|
+
// Combine visit frequency with query similarity for final ranking
|
|
128
|
+
const scores = new Map();
|
|
129
|
+
const maxVisits = Math.max(...visitCounts.values());
|
|
130
|
+
for (const [id] of this.nodes) {
|
|
131
|
+
const visitScore = (visitCounts.get(id) || 0) / maxVisits;
|
|
132
|
+
const similarityScore = similarities.get(id) || 0;
|
|
133
|
+
// Weighted combination
|
|
134
|
+
scores.set(id, 0.6 * similarityScore + 0.4 * visitScore);
|
|
135
|
+
}
|
|
136
|
+
// Sort and return top K
|
|
137
|
+
const rankedNodes = Array.from(scores.entries())
|
|
138
|
+
.sort((a, b) => b[1] - a[1])
|
|
139
|
+
.slice(0, topK)
|
|
140
|
+
.map(([id, score]) => {
|
|
141
|
+
const node = this.nodes.get(id);
|
|
142
|
+
return {
|
|
143
|
+
id,
|
|
144
|
+
content: node.content,
|
|
145
|
+
metadata: node.metadata,
|
|
146
|
+
score,
|
|
147
|
+
};
|
|
148
|
+
});
|
|
149
|
+
logger.debug("[GraphRAG] Query completed", {
|
|
150
|
+
topK,
|
|
151
|
+
resultsCount: rankedNodes.length,
|
|
152
|
+
topScore: rankedNodes[0]?.score,
|
|
153
|
+
});
|
|
154
|
+
return rankedNodes;
|
|
155
|
+
}
|
|
156
|
+
/**
|
|
157
|
+
* Add a single node to the graph
|
|
158
|
+
*
|
|
159
|
+
* @param chunk - Document chunk
|
|
160
|
+
* @param embedding - Embedding vector
|
|
161
|
+
* @returns Node ID
|
|
162
|
+
*/
|
|
163
|
+
addNode(chunk, embedding) {
|
|
164
|
+
const id = randomUUID();
|
|
165
|
+
this.nodes.set(id, {
|
|
166
|
+
id,
|
|
167
|
+
content: chunk.text,
|
|
168
|
+
metadata: chunk.metadata || {},
|
|
169
|
+
embedding: embedding.vector,
|
|
170
|
+
});
|
|
171
|
+
// Create edges to existing nodes
|
|
172
|
+
const edges = [];
|
|
173
|
+
for (const [existingId, existingNode] of this.nodes) {
|
|
174
|
+
if (existingId === id) {
|
|
175
|
+
continue;
|
|
176
|
+
}
|
|
177
|
+
const similarity = this.cosineSimilarity(embedding.vector, existingNode.embedding);
|
|
178
|
+
if (similarity >= this.threshold) {
|
|
179
|
+
edges.push({
|
|
180
|
+
source: id,
|
|
181
|
+
target: existingId,
|
|
182
|
+
weight: similarity,
|
|
183
|
+
type: "semantic",
|
|
184
|
+
});
|
|
185
|
+
// Add reverse edge
|
|
186
|
+
const existingEdges = this.edges.get(existingId) || [];
|
|
187
|
+
existingEdges.push({
|
|
188
|
+
source: existingId,
|
|
189
|
+
target: id,
|
|
190
|
+
weight: similarity,
|
|
191
|
+
type: "semantic",
|
|
192
|
+
});
|
|
193
|
+
this.edges.set(existingId, existingEdges);
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
this.edges.set(id, edges);
|
|
197
|
+
return id;
|
|
198
|
+
}
|
|
199
|
+
/**
|
|
200
|
+
* Remove a node and its edges from the graph
|
|
201
|
+
*
|
|
202
|
+
* @param id - Node ID to remove
|
|
203
|
+
* @returns True if node was removed
|
|
204
|
+
*/
|
|
205
|
+
removeNode(id) {
|
|
206
|
+
if (!this.nodes.has(id)) {
|
|
207
|
+
return false;
|
|
208
|
+
}
|
|
209
|
+
// Remove node
|
|
210
|
+
this.nodes.delete(id);
|
|
211
|
+
this.edges.delete(id);
|
|
212
|
+
// Remove edges pointing to this node
|
|
213
|
+
for (const [nodeId, edges] of this.edges) {
|
|
214
|
+
this.edges.set(nodeId, edges.filter((e) => e.target !== id));
|
|
215
|
+
}
|
|
216
|
+
return true;
|
|
217
|
+
}
|
|
218
|
+
/**
|
|
219
|
+
* Get graph statistics
|
|
220
|
+
*/
|
|
221
|
+
getStats() {
|
|
222
|
+
const edgeCount = Array.from(this.edges.values()).reduce((sum, e) => sum + e.length, 0);
|
|
223
|
+
return {
|
|
224
|
+
nodeCount: this.nodes.size,
|
|
225
|
+
edgeCount,
|
|
226
|
+
avgDegree: this.nodes.size > 0 ? edgeCount / this.nodes.size : 0,
|
|
227
|
+
threshold: this.threshold,
|
|
228
|
+
};
|
|
229
|
+
}
|
|
230
|
+
/**
|
|
231
|
+
* Get a node by ID
|
|
232
|
+
*/
|
|
233
|
+
getNode(id) {
|
|
234
|
+
return this.nodes.get(id);
|
|
235
|
+
}
|
|
236
|
+
/**
|
|
237
|
+
* Get all nodes
|
|
238
|
+
*/
|
|
239
|
+
getAllNodes() {
|
|
240
|
+
return Array.from(this.nodes.values());
|
|
241
|
+
}
|
|
242
|
+
/**
|
|
243
|
+
* Get edges for a node
|
|
244
|
+
*/
|
|
245
|
+
getEdges(nodeId) {
|
|
246
|
+
return this.edges.get(nodeId) || [];
|
|
247
|
+
}
|
|
248
|
+
/**
|
|
249
|
+
* Find connected components in the graph
|
|
250
|
+
*/
|
|
251
|
+
findConnectedComponents() {
|
|
252
|
+
const visited = new Set();
|
|
253
|
+
const components = [];
|
|
254
|
+
for (const nodeId of this.nodes.keys()) {
|
|
255
|
+
if (visited.has(nodeId)) {
|
|
256
|
+
continue;
|
|
257
|
+
}
|
|
258
|
+
const component = [];
|
|
259
|
+
const queue = [nodeId];
|
|
260
|
+
while (queue.length > 0) {
|
|
261
|
+
const current = queue.shift();
|
|
262
|
+
if (visited.has(current)) {
|
|
263
|
+
continue;
|
|
264
|
+
}
|
|
265
|
+
visited.add(current);
|
|
266
|
+
component.push(current);
|
|
267
|
+
const edges = this.edges.get(current) || [];
|
|
268
|
+
for (const edge of edges) {
|
|
269
|
+
if (!visited.has(edge.target)) {
|
|
270
|
+
queue.push(edge.target);
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
components.push(component);
|
|
275
|
+
}
|
|
276
|
+
return components;
|
|
277
|
+
}
|
|
278
|
+
/**
|
|
279
|
+
* Calculate cosine similarity between two vectors
|
|
280
|
+
*/
|
|
281
|
+
cosineSimilarity(a, b) {
|
|
282
|
+
if (a.length !== b.length) {
|
|
283
|
+
throw new Error("Vectors must have the same dimension");
|
|
284
|
+
}
|
|
285
|
+
let dotProduct = 0;
|
|
286
|
+
let normA = 0;
|
|
287
|
+
let normB = 0;
|
|
288
|
+
for (let i = 0; i < a.length; i++) {
|
|
289
|
+
dotProduct += a[i] * b[i];
|
|
290
|
+
normA += a[i] * a[i];
|
|
291
|
+
normB += b[i] * b[i];
|
|
292
|
+
}
|
|
293
|
+
const denominator = Math.sqrt(normA) * Math.sqrt(normB);
|
|
294
|
+
return denominator === 0 ? 0 : dotProduct / denominator;
|
|
295
|
+
}
|
|
296
|
+
/**
|
|
297
|
+
* Normalize probabilities to sum to 1
|
|
298
|
+
*/
|
|
299
|
+
normalizeProbs(probs) {
|
|
300
|
+
const sum = probs.reduce((a, b) => a + b, 0);
|
|
301
|
+
return sum === 0
|
|
302
|
+
? probs.map(() => 1 / probs.length)
|
|
303
|
+
: probs.map((p) => p / sum);
|
|
304
|
+
}
|
|
305
|
+
/**
|
|
306
|
+
* Weighted random choice
|
|
307
|
+
*/
|
|
308
|
+
weightedRandomChoice(weights) {
|
|
309
|
+
const random = Math.random();
|
|
310
|
+
let cumulative = 0;
|
|
311
|
+
for (let i = 0; i < weights.length; i++) {
|
|
312
|
+
cumulative += weights[i];
|
|
313
|
+
if (random <= cumulative) {
|
|
314
|
+
return i;
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
return weights.length - 1;
|
|
318
|
+
}
|
|
319
|
+
/**
|
|
320
|
+
* Update similarity threshold and rebuild edges
|
|
321
|
+
*/
|
|
322
|
+
updateThreshold(threshold) {
|
|
323
|
+
this.threshold = threshold;
|
|
324
|
+
// Rebuild edges with new threshold
|
|
325
|
+
this.edges.clear();
|
|
326
|
+
const nodeIds = Array.from(this.nodes.keys());
|
|
327
|
+
for (let i = 0; i < nodeIds.length; i++) {
|
|
328
|
+
const edges = [];
|
|
329
|
+
const nodeA = this.nodes.get(nodeIds[i]);
|
|
330
|
+
for (let j = 0; j < nodeIds.length; j++) {
|
|
331
|
+
if (i === j) {
|
|
332
|
+
continue;
|
|
333
|
+
}
|
|
334
|
+
const nodeB = this.nodes.get(nodeIds[j]);
|
|
335
|
+
if (!nodeA.embedding || !nodeB.embedding) {
|
|
336
|
+
continue;
|
|
337
|
+
}
|
|
338
|
+
const similarity = this.cosineSimilarity(nodeA.embedding, nodeB.embedding);
|
|
339
|
+
if (similarity >= threshold) {
|
|
340
|
+
edges.push({
|
|
341
|
+
source: nodeIds[i],
|
|
342
|
+
target: nodeIds[j],
|
|
343
|
+
weight: similarity,
|
|
344
|
+
type: "semantic",
|
|
345
|
+
});
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
edges.sort((a, b) => b.weight - a.weight);
|
|
349
|
+
this.edges.set(nodeIds[i], edges);
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
/**
|
|
353
|
+
* Serialize graph to JSON
|
|
354
|
+
*/
|
|
355
|
+
toJSON() {
|
|
356
|
+
return {
|
|
357
|
+
nodes: Array.from(this.nodes.values()),
|
|
358
|
+
edges: Array.from(this.edges.entries()).map(([source, edges]) => ({
|
|
359
|
+
source,
|
|
360
|
+
edges,
|
|
361
|
+
})),
|
|
362
|
+
config: {
|
|
363
|
+
dimension: this.dimension,
|
|
364
|
+
threshold: this.threshold,
|
|
365
|
+
},
|
|
366
|
+
};
|
|
367
|
+
}
|
|
368
|
+
/**
|
|
369
|
+
* Load graph from JSON
|
|
370
|
+
*/
|
|
371
|
+
static fromJSON(json) {
|
|
372
|
+
const graph = new GraphRAG({
|
|
373
|
+
dimension: json.config.dimension,
|
|
374
|
+
threshold: json.config.threshold,
|
|
375
|
+
});
|
|
376
|
+
for (const node of json.nodes) {
|
|
377
|
+
graph.nodes.set(node.id, node);
|
|
378
|
+
}
|
|
379
|
+
for (const { source, edges } of json.edges) {
|
|
380
|
+
graph.edges.set(source, edges);
|
|
381
|
+
}
|
|
382
|
+
return graph;
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
//# sourceMappingURL=graphRAG.js.map
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* RAG Document Processing Module
|
|
3
|
+
*
|
|
4
|
+
* Provides comprehensive RAG (Retrieval-Augmented Generation) capabilities:
|
|
5
|
+
* - Document loading (text, markdown, HTML, JSON, CSV, PDF, web)
|
|
6
|
+
* - MDocument class for fluent document processing
|
|
7
|
+
* - 10 chunking strategies (character, recursive, sentence, token, markdown, html, json, latex, semantic, semantic-markdown)
|
|
8
|
+
* - LLM-powered metadata extraction (title, summary, keywords, Q&A)
|
|
9
|
+
* - Vector query tools with metadata filtering and reranking
|
|
10
|
+
* - Hybrid search (BM25 + vector fusion)
|
|
11
|
+
* - Graph RAG for knowledge graph-based retrieval
|
|
12
|
+
* - RAG pipeline orchestration
|
|
13
|
+
* - Context assembly and formatting
|
|
14
|
+
* - ChunkerFactory and ChunkerRegistry patterns for extensibility
|
|
15
|
+
* - Error handling and resilience (CircuitBreaker, RetryHandler)
|
|
16
|
+
*
|
|
17
|
+
* @example
|
|
18
|
+
* ```typescript
|
|
19
|
+
* import {
|
|
20
|
+
* MDocument,
|
|
21
|
+
* loadDocument,
|
|
22
|
+
* RAGPipeline,
|
|
23
|
+
* ChunkerRegistry,
|
|
24
|
+
* ChunkerFactory,
|
|
25
|
+
* CircuitBreaker
|
|
26
|
+
* } from '@juspay/neurolink';
|
|
27
|
+
*
|
|
28
|
+
* // Load and process a document
|
|
29
|
+
* const doc = await loadDocument('/path/to/document.md');
|
|
30
|
+
* await doc.chunk({ strategy: 'markdown', config: { maxSize: 1000 } });
|
|
31
|
+
* await doc.embed('openai', 'text-embedding-3-small');
|
|
32
|
+
*
|
|
33
|
+
* // Or use the full RAG pipeline
|
|
34
|
+
* const pipeline = new RAGPipeline({
|
|
35
|
+
* embeddingModel: { provider: 'openai', modelName: 'text-embedding-3-small' },
|
|
36
|
+
* generationModel: { provider: 'openai', modelName: 'gpt-4o-mini' }
|
|
37
|
+
* });
|
|
38
|
+
* await pipeline.ingest(['/path/to/docs/*.md']);
|
|
39
|
+
* const response = await pipeline.query('What are the key features?');
|
|
40
|
+
*
|
|
41
|
+
* // Use factory pattern for chunker creation
|
|
42
|
+
* const chunker = await ChunkerFactory.createChunker('semantic', { maxSize: 500 });
|
|
43
|
+
* const chunks = await chunker.chunk(text);
|
|
44
|
+
* ```
|
|
45
|
+
*/
|
|
46
|
+
export { ChunkerFactory, chunkerFactory, createChunker, getAvailableStrategies as getFactoryStrategies, getDefaultConfig as getFactoryDefaultConfig, } from "./ChunkerFactory.js";
|
|
47
|
+
export { ChunkerRegistry as ChunkerRegistryV2, chunkerRegistry, getAvailableChunkers, getChunker, getChunkerMetadata, } from "./ChunkerRegistry.js";
|
|
48
|
+
export * from "./chunkers/index.js";
|
|
49
|
+
export { CharacterChunker, ChunkerRegistry, chunkText, HTMLChunker, JSONChunker, LaTeXChunker, MarkdownChunker, RecursiveChunker, SemanticChunker, SentenceChunker, TokenChunker, } from "./chunking/index.js";
|
|
50
|
+
export { CSVLoader, type CSVLoaderOptions, type DocumentLoader, HTMLLoader, JSONLoader, type LoaderOptions, loadDocument, loadDocuments, MarkdownLoader, MDocument, PDFLoader, type PDFLoaderOptions, TextLoader, WebLoader, type WebLoaderOptions, } from "./document/index.js";
|
|
51
|
+
export * from "./errors/index.js";
|
|
52
|
+
export { GraphRAG } from "./graphRag/index.js";
|
|
53
|
+
export { createMetadataExtractor, extractMetadata, getAvailableExtractors, getAvailableExtractorTypes, getExtractor, getExtractorDefaultConfig, getExtractorMetadata, getRegisteredExtractorMetadata, LLMMetadataExtractor, type MetadataExtractor, type MetadataExtractorConfig, MetadataExtractorFactory, type MetadataExtractorMetadata, MetadataExtractorRegistry, type MetadataExtractorType, metadataExtractorFactory, metadataExtractorRegistry, } from "./metadata/index.js";
|
|
54
|
+
export { assembleContext, type CitationFormat, type ContextAssemblyOptions, type ContextWindow, createContextWindow, createRAGPipeline, type EmbeddingModelConfig, extractKeySentences, formatContextWithCitations, type GenerationModelConfig, type IngestOptions, orderByDocumentStructure, type PipelineStats, type QueryOptions, RAGPipeline, type RAGPipelineConfig, type RAGResponse, summarizeContext, } from "./pipeline/index.js";
|
|
55
|
+
export { prepareRAGTool, type RAGPreparedTool } from "./ragIntegration.js";
|
|
56
|
+
export { batchRerank, CohereRelevanceScorer, CrossEncoderReranker, createReranker, getAvailableRerankers, getAvailableRerankerTypes, getRegisteredRerankerMetadata, getReranker, getRerankerDefaultConfig, getRerankerMetadata, type Reranker, type RerankerConfig, RerankerFactory, type RerankerMetadata, RerankerRegistry, type RerankerType, rerank, rerankerFactory, rerankerRegistry, simpleRerank, } from "./reranker/index.js";
|
|
57
|
+
export * from "./resilience/index.js";
|
|
58
|
+
export { type BM25Index, createHybridSearch, createVectorQueryTool, type HybridSearchOptions, InMemoryBM25Index, InMemoryVectorStore, linearCombination, reciprocalRankFusion, type VectorStore, } from "./retrieval/index.js";
|
|
59
|
+
export * from "./types.js";
|
|
60
|
+
import type { Chunk, ChunkingStrategy, ExtractParams } from "./types.js";
|
|
61
|
+
/**
|
|
62
|
+
* Process a document through the full RAG pipeline
|
|
63
|
+
*
|
|
64
|
+
* @param text - Document text to process
|
|
65
|
+
* @param options - Processing options
|
|
66
|
+
* @returns Processed chunks with optional metadata
|
|
67
|
+
*/
|
|
68
|
+
export declare function processDocument(text: string, options?: {
|
|
69
|
+
/** Chunking strategy (default: recursive) */
|
|
70
|
+
strategy?: ChunkingStrategy;
|
|
71
|
+
/** Maximum chunk size */
|
|
72
|
+
maxSize?: number;
|
|
73
|
+
/** Chunk overlap */
|
|
74
|
+
overlap?: number;
|
|
75
|
+
/** Metadata extraction options */
|
|
76
|
+
extract?: ExtractParams;
|
|
77
|
+
/** Provider for metadata extraction */
|
|
78
|
+
provider?: string;
|
|
79
|
+
/** Model for metadata extraction */
|
|
80
|
+
model?: string;
|
|
81
|
+
/** Custom metadata to add */
|
|
82
|
+
metadata?: Record<string, unknown>;
|
|
83
|
+
}): Promise<Chunk[]>;
|
|
84
|
+
/**
|
|
85
|
+
* Get recommended chunking strategy based on content type
|
|
86
|
+
*
|
|
87
|
+
* @param contentType - MIME type or file extension
|
|
88
|
+
* @returns Recommended chunking strategy
|
|
89
|
+
*/
|
|
90
|
+
export declare function getRecommendedStrategy(contentType: string): ChunkingStrategy;
|
|
91
|
+
/**
|
|
92
|
+
* Get available chunking strategies
|
|
93
|
+
*
|
|
94
|
+
* @returns Array of available strategy names
|
|
95
|
+
*/
|
|
96
|
+
export declare function getAvailableStrategies(): ChunkingStrategy[];
|
|
97
|
+
/**
|
|
98
|
+
* Get default configuration for a chunking strategy
|
|
99
|
+
*
|
|
100
|
+
* @param strategy - Chunking strategy name
|
|
101
|
+
* @returns Default configuration object
|
|
102
|
+
*/
|
|
103
|
+
export declare function getDefaultChunkerConfig(strategy: ChunkingStrategy): Record<string, unknown>;
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* RAG Document Processing Module
|
|
3
|
+
*
|
|
4
|
+
* Provides comprehensive RAG (Retrieval-Augmented Generation) capabilities:
|
|
5
|
+
* - Document loading (text, markdown, HTML, JSON, CSV, PDF, web)
|
|
6
|
+
* - MDocument class for fluent document processing
|
|
7
|
+
* - 10 chunking strategies (character, recursive, sentence, token, markdown, html, json, latex, semantic, semantic-markdown)
|
|
8
|
+
* - LLM-powered metadata extraction (title, summary, keywords, Q&A)
|
|
9
|
+
* - Vector query tools with metadata filtering and reranking
|
|
10
|
+
* - Hybrid search (BM25 + vector fusion)
|
|
11
|
+
* - Graph RAG for knowledge graph-based retrieval
|
|
12
|
+
* - RAG pipeline orchestration
|
|
13
|
+
* - Context assembly and formatting
|
|
14
|
+
* - ChunkerFactory and ChunkerRegistry patterns for extensibility
|
|
15
|
+
* - Error handling and resilience (CircuitBreaker, RetryHandler)
|
|
16
|
+
*
|
|
17
|
+
* @example
|
|
18
|
+
* ```typescript
|
|
19
|
+
* import {
|
|
20
|
+
* MDocument,
|
|
21
|
+
* loadDocument,
|
|
22
|
+
* RAGPipeline,
|
|
23
|
+
* ChunkerRegistry,
|
|
24
|
+
* ChunkerFactory,
|
|
25
|
+
* CircuitBreaker
|
|
26
|
+
* } from '@juspay/neurolink';
|
|
27
|
+
*
|
|
28
|
+
* // Load and process a document
|
|
29
|
+
* const doc = await loadDocument('/path/to/document.md');
|
|
30
|
+
* await doc.chunk({ strategy: 'markdown', config: { maxSize: 1000 } });
|
|
31
|
+
* await doc.embed('openai', 'text-embedding-3-small');
|
|
32
|
+
*
|
|
33
|
+
* // Or use the full RAG pipeline
|
|
34
|
+
* const pipeline = new RAGPipeline({
|
|
35
|
+
* embeddingModel: { provider: 'openai', modelName: 'text-embedding-3-small' },
|
|
36
|
+
* generationModel: { provider: 'openai', modelName: 'gpt-4o-mini' }
|
|
37
|
+
* });
|
|
38
|
+
* await pipeline.ingest(['/path/to/docs/*.md']);
|
|
39
|
+
* const response = await pipeline.query('What are the key features?');
|
|
40
|
+
*
|
|
41
|
+
* // Use factory pattern for chunker creation
|
|
42
|
+
* const chunker = await ChunkerFactory.createChunker('semantic', { maxSize: 500 });
|
|
43
|
+
* const chunks = await chunker.chunk(text);
|
|
44
|
+
* ```
|
|
45
|
+
*/
|
|
46
|
+
export { ChunkerFactory, chunkerFactory, createChunker, getAvailableStrategies as getFactoryStrategies, getDefaultConfig as getFactoryDefaultConfig, } from "./ChunkerFactory.js";
|
|
47
|
+
// ChunkerFactory and ChunkerRegistry patterns (from main worktree)
|
|
48
|
+
export { ChunkerRegistry as ChunkerRegistryV2, chunkerRegistry, getAvailableChunkers, getChunker, getChunkerMetadata, } from "./ChunkerRegistry.js";
|
|
49
|
+
// Base chunker and chunker implementations (from main worktree)
|
|
50
|
+
export * from "./chunkers/index.js";
|
|
51
|
+
// Chunking
|
|
52
|
+
export { CharacterChunker, ChunkerRegistry, chunkText, HTMLChunker, JSONChunker, LaTeXChunker, MarkdownChunker, RecursiveChunker, SemanticChunker, SentenceChunker, TokenChunker, } from "./chunking/index.js";
|
|
53
|
+
// Document Processing
|
|
54
|
+
export { CSVLoader, HTMLLoader, JSONLoader, loadDocument, loadDocuments, MarkdownLoader, MDocument, PDFLoader, TextLoader, WebLoader, } from "./document/index.js";
|
|
55
|
+
// Error handling
|
|
56
|
+
export * from "./errors/index.js";
|
|
57
|
+
// Graph RAG
|
|
58
|
+
export { GraphRAG } from "./graphRag/index.js";
|
|
59
|
+
// Metadata Extraction
|
|
60
|
+
export { createMetadataExtractor, extractMetadata, getAvailableExtractors, getAvailableExtractorTypes, getExtractor, getExtractorDefaultConfig, getExtractorMetadata, getRegisteredExtractorMetadata, LLMMetadataExtractor,
|
|
61
|
+
// Factory pattern
|
|
62
|
+
MetadataExtractorFactory,
|
|
63
|
+
// Registry pattern
|
|
64
|
+
MetadataExtractorRegistry, metadataExtractorFactory, metadataExtractorRegistry, } from "./metadata/index.js";
|
|
65
|
+
// Pipeline
|
|
66
|
+
export { assembleContext, createContextWindow, createRAGPipeline, extractKeySentences, formatContextWithCitations, orderByDocumentStructure, RAGPipeline, summarizeContext, } from "./pipeline/index.js";
|
|
67
|
+
// RAG Integration (for generate/stream)
|
|
68
|
+
export { prepareRAGTool } from "./ragIntegration.js";
|
|
69
|
+
// Reranker
|
|
70
|
+
export { batchRerank, CohereRelevanceScorer, CrossEncoderReranker, createReranker, getAvailableRerankers, getAvailableRerankerTypes, getRegisteredRerankerMetadata, getReranker, getRerankerDefaultConfig, getRerankerMetadata,
|
|
71
|
+
// Factory pattern
|
|
72
|
+
RerankerFactory,
|
|
73
|
+
// Registry pattern
|
|
74
|
+
RerankerRegistry, rerank, rerankerFactory, rerankerRegistry, simpleRerank, } from "./reranker/index.js";
|
|
75
|
+
// Resilience patterns
|
|
76
|
+
export * from "./resilience/index.js";
|
|
77
|
+
// Retrieval
|
|
78
|
+
export { createHybridSearch, createVectorQueryTool, InMemoryBM25Index, InMemoryVectorStore, linearCombination, reciprocalRankFusion, } from "./retrieval/index.js";
|
|
79
|
+
// Types
|
|
80
|
+
export * from "./types.js";
|
|
81
|
+
// Convenience functions
|
|
82
|
+
import { ChunkerRegistry } from "./chunking/index.js";
|
|
83
|
+
import { LLMMetadataExtractor } from "./metadata/index.js";
|
|
84
|
+
/**
|
|
85
|
+
* Process a document through the full RAG pipeline
|
|
86
|
+
*
|
|
87
|
+
* @param text - Document text to process
|
|
88
|
+
* @param options - Processing options
|
|
89
|
+
* @returns Processed chunks with optional metadata
|
|
90
|
+
*/
|
|
91
|
+
export async function processDocument(text, options) {
|
|
92
|
+
const { strategy = "recursive", maxSize = 1000, overlap = 200, extract, provider, model, metadata = {}, } = options || {};
|
|
93
|
+
// Chunk the document
|
|
94
|
+
const chunker = ChunkerRegistry.get(strategy);
|
|
95
|
+
const chunks = await chunker.chunk(text, { maxSize, overlap, metadata });
|
|
96
|
+
// Extract metadata if requested
|
|
97
|
+
if (extract) {
|
|
98
|
+
const extractor = new LLMMetadataExtractor({ provider, modelName: model });
|
|
99
|
+
const results = await extractor.extract(chunks, extract);
|
|
100
|
+
// Merge metadata into chunks
|
|
101
|
+
for (let i = 0; i < chunks.length && i < results.length; i++) {
|
|
102
|
+
const result = results[i];
|
|
103
|
+
if (result.title) {
|
|
104
|
+
chunks[i].metadata.title = result.title;
|
|
105
|
+
}
|
|
106
|
+
if (result.summary) {
|
|
107
|
+
chunks[i].metadata.summary = result.summary;
|
|
108
|
+
}
|
|
109
|
+
if (result.keywords) {
|
|
110
|
+
chunks[i].metadata.keywords = result.keywords;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
return chunks;
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* Get recommended chunking strategy based on content type
|
|
118
|
+
*
|
|
119
|
+
* @param contentType - MIME type or file extension
|
|
120
|
+
* @returns Recommended chunking strategy
|
|
121
|
+
*/
|
|
122
|
+
export function getRecommendedStrategy(contentType) {
|
|
123
|
+
return ChunkerRegistry.getRecommendedStrategy(contentType);
|
|
124
|
+
}
|
|
125
|
+
/**
|
|
126
|
+
* Get available chunking strategies
|
|
127
|
+
*
|
|
128
|
+
* @returns Array of available strategy names
|
|
129
|
+
*/
|
|
130
|
+
export function getAvailableStrategies() {
|
|
131
|
+
return ChunkerRegistry.getAvailableStrategies();
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* Get default configuration for a chunking strategy
|
|
135
|
+
*
|
|
136
|
+
* @param strategy - Chunking strategy name
|
|
137
|
+
* @returns Default configuration object
|
|
138
|
+
*/
|
|
139
|
+
export function getDefaultChunkerConfig(strategy) {
|
|
140
|
+
return ChunkerRegistry.getDefaultConfig(strategy);
|
|
141
|
+
}
|
|
142
|
+
//# sourceMappingURL=index.js.map
|