@juspay/neurolink 9.1.1 → 9.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (555) hide show
  1. package/CHANGELOG.md +27 -0
  2. package/README.md +106 -37
  3. package/dist/agent/directTools.d.ts +11 -11
  4. package/dist/cli/commands/config.d.ts +6 -6
  5. package/dist/cli/commands/rag.d.ts +19 -0
  6. package/dist/cli/commands/rag.js +756 -0
  7. package/dist/cli/factories/commandFactory.js +146 -83
  8. package/dist/cli/parser.js +4 -1
  9. package/dist/core/baseProvider.d.ts +43 -30
  10. package/dist/core/baseProvider.js +98 -138
  11. package/dist/core/conversationMemoryFactory.d.ts +2 -2
  12. package/dist/core/conversationMemoryFactory.js +2 -2
  13. package/dist/core/conversationMemoryInitializer.d.ts +1 -2
  14. package/dist/core/conversationMemoryInitializer.js +2 -2
  15. package/dist/core/infrastructure/baseError.d.ts +21 -0
  16. package/dist/core/infrastructure/baseError.js +22 -0
  17. package/dist/core/infrastructure/baseFactory.d.ts +21 -0
  18. package/dist/core/infrastructure/baseFactory.js +54 -0
  19. package/dist/core/infrastructure/baseRegistry.d.ts +21 -0
  20. package/dist/core/infrastructure/baseRegistry.js +49 -0
  21. package/dist/core/infrastructure/index.d.ts +5 -0
  22. package/dist/core/infrastructure/index.js +5 -0
  23. package/dist/core/infrastructure/retry.d.ts +7 -0
  24. package/dist/core/infrastructure/retry.js +20 -0
  25. package/dist/core/infrastructure/typedEventEmitter.d.ts +8 -0
  26. package/dist/core/infrastructure/typedEventEmitter.js +23 -0
  27. package/dist/core/redisConversationMemoryManager.d.ts +1 -6
  28. package/dist/core/redisConversationMemoryManager.js +7 -19
  29. package/dist/factories/providerFactory.d.ts +5 -3
  30. package/dist/factories/providerFactory.js +31 -24
  31. package/dist/image-gen/ImageGenService.d.ts +143 -0
  32. package/dist/image-gen/ImageGenService.js +345 -0
  33. package/dist/image-gen/imageGenTools.d.ts +126 -0
  34. package/dist/image-gen/imageGenTools.js +304 -0
  35. package/dist/image-gen/index.d.ts +46 -0
  36. package/dist/image-gen/index.js +48 -0
  37. package/dist/image-gen/types.d.ts +237 -0
  38. package/dist/image-gen/types.js +24 -0
  39. package/dist/index.d.ts +46 -12
  40. package/dist/index.js +88 -36
  41. package/dist/lib/agent/directTools.d.ts +8 -8
  42. package/dist/lib/core/baseProvider.d.ts +43 -30
  43. package/dist/lib/core/baseProvider.js +98 -138
  44. package/dist/lib/core/conversationMemoryFactory.d.ts +2 -2
  45. package/dist/lib/core/conversationMemoryFactory.js +2 -2
  46. package/dist/lib/core/conversationMemoryInitializer.d.ts +1 -2
  47. package/dist/lib/core/conversationMemoryInitializer.js +2 -2
  48. package/dist/lib/core/infrastructure/baseError.d.ts +21 -0
  49. package/dist/lib/core/infrastructure/baseError.js +23 -0
  50. package/dist/lib/core/infrastructure/baseFactory.d.ts +21 -0
  51. package/dist/lib/core/infrastructure/baseFactory.js +55 -0
  52. package/dist/lib/core/infrastructure/baseRegistry.d.ts +21 -0
  53. package/dist/lib/core/infrastructure/baseRegistry.js +50 -0
  54. package/dist/lib/core/infrastructure/index.d.ts +5 -0
  55. package/dist/lib/core/infrastructure/index.js +6 -0
  56. package/dist/lib/core/infrastructure/retry.d.ts +7 -0
  57. package/dist/lib/core/infrastructure/retry.js +21 -0
  58. package/dist/lib/core/infrastructure/typedEventEmitter.d.ts +8 -0
  59. package/dist/lib/core/infrastructure/typedEventEmitter.js +24 -0
  60. package/dist/lib/core/redisConversationMemoryManager.d.ts +1 -6
  61. package/dist/lib/core/redisConversationMemoryManager.js +7 -19
  62. package/dist/lib/factories/providerFactory.d.ts +5 -3
  63. package/dist/lib/factories/providerFactory.js +31 -24
  64. package/dist/lib/image-gen/ImageGenService.d.ts +143 -0
  65. package/dist/lib/image-gen/ImageGenService.js +346 -0
  66. package/dist/lib/image-gen/imageGenTools.d.ts +126 -0
  67. package/dist/lib/image-gen/imageGenTools.js +305 -0
  68. package/dist/lib/image-gen/index.d.ts +46 -0
  69. package/dist/lib/image-gen/index.js +49 -0
  70. package/dist/lib/image-gen/types.d.ts +237 -0
  71. package/dist/lib/image-gen/types.js +25 -0
  72. package/dist/lib/index.d.ts +46 -12
  73. package/dist/lib/index.js +88 -36
  74. package/dist/lib/mcp/index.d.ts +6 -5
  75. package/dist/lib/mcp/index.js +7 -5
  76. package/dist/lib/neurolink.d.ts +11 -13
  77. package/dist/lib/neurolink.js +95 -29
  78. package/dist/lib/processors/base/BaseFileProcessor.d.ts +273 -0
  79. package/dist/lib/processors/base/BaseFileProcessor.js +614 -0
  80. package/dist/lib/processors/base/index.d.ts +14 -0
  81. package/dist/lib/processors/base/index.js +20 -0
  82. package/dist/lib/processors/base/types.d.ts +593 -0
  83. package/dist/lib/processors/base/types.js +77 -0
  84. package/dist/lib/processors/cli/fileProcessorCli.d.ts +163 -0
  85. package/dist/lib/processors/cli/fileProcessorCli.js +389 -0
  86. package/dist/lib/processors/cli/index.d.ts +37 -0
  87. package/dist/lib/processors/cli/index.js +50 -0
  88. package/dist/lib/processors/code/ConfigProcessor.d.ts +171 -0
  89. package/dist/lib/processors/code/ConfigProcessor.js +401 -0
  90. package/dist/lib/processors/code/SourceCodeProcessor.d.ts +174 -0
  91. package/dist/lib/processors/code/SourceCodeProcessor.js +305 -0
  92. package/dist/lib/processors/code/index.d.ts +44 -0
  93. package/dist/lib/processors/code/index.js +61 -0
  94. package/dist/lib/processors/config/fileTypes.d.ts +283 -0
  95. package/dist/lib/processors/config/fileTypes.js +521 -0
  96. package/dist/lib/processors/config/index.d.ts +32 -0
  97. package/dist/lib/processors/config/index.js +93 -0
  98. package/dist/lib/processors/config/languageMap.d.ts +66 -0
  99. package/dist/lib/processors/config/languageMap.js +411 -0
  100. package/dist/lib/processors/config/mimeTypes.d.ts +376 -0
  101. package/dist/lib/processors/config/mimeTypes.js +339 -0
  102. package/dist/lib/processors/config/sizeLimits.d.ts +194 -0
  103. package/dist/lib/processors/config/sizeLimits.js +247 -0
  104. package/dist/lib/processors/data/JsonProcessor.d.ts +122 -0
  105. package/dist/lib/processors/data/JsonProcessor.js +204 -0
  106. package/dist/lib/processors/data/XmlProcessor.d.ts +160 -0
  107. package/dist/lib/processors/data/XmlProcessor.js +284 -0
  108. package/dist/lib/processors/data/YamlProcessor.d.ts +163 -0
  109. package/dist/lib/processors/data/YamlProcessor.js +295 -0
  110. package/dist/lib/processors/data/index.d.ts +49 -0
  111. package/dist/lib/processors/data/index.js +77 -0
  112. package/dist/lib/processors/document/ExcelProcessor.d.ts +238 -0
  113. package/dist/lib/processors/document/ExcelProcessor.js +520 -0
  114. package/dist/lib/processors/document/OpenDocumentProcessor.d.ts +69 -0
  115. package/dist/lib/processors/document/OpenDocumentProcessor.js +211 -0
  116. package/dist/lib/processors/document/RtfProcessor.d.ts +152 -0
  117. package/dist/lib/processors/document/RtfProcessor.js +362 -0
  118. package/dist/lib/processors/document/WordProcessor.d.ts +168 -0
  119. package/dist/lib/processors/document/WordProcessor.js +354 -0
  120. package/dist/lib/processors/document/index.d.ts +54 -0
  121. package/dist/lib/processors/document/index.js +91 -0
  122. package/dist/lib/processors/errors/FileErrorCode.d.ts +98 -0
  123. package/dist/lib/processors/errors/FileErrorCode.js +256 -0
  124. package/dist/lib/processors/errors/errorHelpers.d.ts +151 -0
  125. package/dist/lib/processors/errors/errorHelpers.js +379 -0
  126. package/dist/lib/processors/errors/errorSerializer.d.ts +139 -0
  127. package/dist/lib/processors/errors/errorSerializer.js +508 -0
  128. package/dist/lib/processors/errors/index.d.ts +46 -0
  129. package/dist/lib/processors/errors/index.js +50 -0
  130. package/dist/lib/processors/index.d.ts +76 -0
  131. package/dist/lib/processors/index.js +113 -0
  132. package/dist/lib/processors/integration/FileProcessorIntegration.d.ts +244 -0
  133. package/dist/lib/processors/integration/FileProcessorIntegration.js +273 -0
  134. package/dist/lib/processors/integration/index.d.ts +42 -0
  135. package/dist/lib/processors/integration/index.js +45 -0
  136. package/dist/lib/processors/markup/HtmlProcessor.d.ts +169 -0
  137. package/dist/lib/processors/markup/HtmlProcessor.js +250 -0
  138. package/dist/lib/processors/markup/MarkdownProcessor.d.ts +165 -0
  139. package/dist/lib/processors/markup/MarkdownProcessor.js +245 -0
  140. package/dist/lib/processors/markup/SvgProcessor.d.ts +156 -0
  141. package/dist/lib/processors/markup/SvgProcessor.js +241 -0
  142. package/dist/lib/processors/markup/TextProcessor.d.ts +135 -0
  143. package/dist/lib/processors/markup/TextProcessor.js +189 -0
  144. package/dist/lib/processors/markup/index.d.ts +66 -0
  145. package/dist/lib/processors/markup/index.js +103 -0
  146. package/dist/lib/processors/registry/ProcessorRegistry.d.ts +334 -0
  147. package/dist/lib/processors/registry/ProcessorRegistry.js +609 -0
  148. package/dist/lib/processors/registry/index.d.ts +12 -0
  149. package/dist/lib/processors/registry/index.js +17 -0
  150. package/dist/lib/processors/registry/types.d.ts +53 -0
  151. package/dist/lib/processors/registry/types.js +11 -0
  152. package/dist/lib/providers/amazonBedrock.d.ts +15 -2
  153. package/dist/lib/providers/amazonBedrock.js +65 -8
  154. package/dist/lib/providers/anthropic.d.ts +3 -3
  155. package/dist/lib/providers/anthropic.js +10 -7
  156. package/dist/lib/providers/googleAiStudio.d.ts +5 -5
  157. package/dist/lib/providers/googleAiStudio.js +10 -7
  158. package/dist/lib/providers/googleVertex.d.ts +16 -4
  159. package/dist/lib/providers/googleVertex.js +72 -16
  160. package/dist/lib/providers/litellm.d.ts +3 -3
  161. package/dist/lib/providers/litellm.js +10 -10
  162. package/dist/lib/providers/mistral.d.ts +3 -3
  163. package/dist/lib/providers/mistral.js +7 -6
  164. package/dist/lib/providers/ollama.d.ts +3 -4
  165. package/dist/lib/providers/ollama.js +7 -8
  166. package/dist/lib/providers/openAI.d.ts +14 -2
  167. package/dist/lib/providers/openAI.js +60 -6
  168. package/dist/lib/providers/openRouter.d.ts +2 -2
  169. package/dist/lib/providers/openRouter.js +10 -6
  170. package/dist/lib/providers/sagemaker/language-model.d.ts +2 -2
  171. package/dist/lib/rag/ChunkerFactory.d.ts +91 -0
  172. package/dist/lib/rag/ChunkerFactory.js +321 -0
  173. package/dist/lib/rag/ChunkerRegistry.d.ts +91 -0
  174. package/dist/lib/rag/ChunkerRegistry.js +422 -0
  175. package/dist/lib/rag/chunkers/BaseChunker.d.ts +53 -0
  176. package/dist/lib/rag/chunkers/BaseChunker.js +144 -0
  177. package/dist/lib/rag/chunkers/CharacterChunker.d.ts +18 -0
  178. package/dist/lib/rag/chunkers/CharacterChunker.js +29 -0
  179. package/dist/lib/rag/chunkers/HTMLChunker.d.ts +19 -0
  180. package/dist/lib/rag/chunkers/HTMLChunker.js +39 -0
  181. package/dist/lib/rag/chunkers/JSONChunker.d.ts +19 -0
  182. package/dist/lib/rag/chunkers/JSONChunker.js +69 -0
  183. package/dist/lib/rag/chunkers/LaTeXChunker.d.ts +15 -0
  184. package/dist/lib/rag/chunkers/LaTeXChunker.js +64 -0
  185. package/dist/lib/rag/chunkers/MarkdownChunker.d.ts +15 -0
  186. package/dist/lib/rag/chunkers/MarkdownChunker.js +103 -0
  187. package/dist/lib/rag/chunkers/RecursiveChunker.d.ts +27 -0
  188. package/dist/lib/rag/chunkers/RecursiveChunker.js +140 -0
  189. package/dist/lib/rag/chunkers/SemanticMarkdownChunker.d.ts +22 -0
  190. package/dist/lib/rag/chunkers/SemanticMarkdownChunker.js +139 -0
  191. package/dist/lib/rag/chunkers/SentenceChunker.d.ts +19 -0
  192. package/dist/lib/rag/chunkers/SentenceChunker.js +67 -0
  193. package/dist/lib/rag/chunkers/TokenChunker.d.ts +19 -0
  194. package/dist/lib/rag/chunkers/TokenChunker.js +62 -0
  195. package/dist/lib/rag/chunkers/index.d.ts +15 -0
  196. package/dist/lib/rag/chunkers/index.js +16 -0
  197. package/dist/lib/rag/chunking/characterChunker.d.ts +16 -0
  198. package/dist/lib/rag/chunking/characterChunker.js +143 -0
  199. package/dist/lib/rag/chunking/chunkerRegistry.d.ts +67 -0
  200. package/dist/lib/rag/chunking/chunkerRegistry.js +195 -0
  201. package/dist/lib/rag/chunking/htmlChunker.d.ts +34 -0
  202. package/dist/lib/rag/chunking/htmlChunker.js +248 -0
  203. package/dist/lib/rag/chunking/index.d.ts +15 -0
  204. package/dist/lib/rag/chunking/index.js +18 -0
  205. package/dist/lib/rag/chunking/jsonChunker.d.ts +20 -0
  206. package/dist/lib/rag/chunking/jsonChunker.js +282 -0
  207. package/dist/lib/rag/chunking/latexChunker.d.ts +26 -0
  208. package/dist/lib/rag/chunking/latexChunker.js +252 -0
  209. package/dist/lib/rag/chunking/markdownChunker.d.ts +19 -0
  210. package/dist/lib/rag/chunking/markdownChunker.js +202 -0
  211. package/dist/lib/rag/chunking/recursiveChunker.d.ts +19 -0
  212. package/dist/lib/rag/chunking/recursiveChunker.js +149 -0
  213. package/dist/lib/rag/chunking/semanticChunker.d.ts +41 -0
  214. package/dist/lib/rag/chunking/semanticChunker.js +307 -0
  215. package/dist/lib/rag/chunking/sentenceChunker.d.ts +25 -0
  216. package/dist/lib/rag/chunking/sentenceChunker.js +231 -0
  217. package/dist/lib/rag/chunking/tokenChunker.d.ts +36 -0
  218. package/dist/lib/rag/chunking/tokenChunker.js +184 -0
  219. package/dist/lib/rag/document/MDocument.d.ts +198 -0
  220. package/dist/lib/rag/document/MDocument.js +393 -0
  221. package/dist/lib/rag/document/index.d.ts +5 -0
  222. package/dist/lib/rag/document/index.js +6 -0
  223. package/dist/lib/rag/document/loaders.d.ts +201 -0
  224. package/dist/lib/rag/document/loaders.js +501 -0
  225. package/dist/lib/rag/errors/RAGError.d.ts +244 -0
  226. package/dist/lib/rag/errors/RAGError.js +275 -0
  227. package/dist/lib/rag/errors/index.d.ts +6 -0
  228. package/dist/lib/rag/errors/index.js +7 -0
  229. package/dist/lib/rag/graphRag/graphRAG.d.ts +115 -0
  230. package/dist/lib/rag/graphRag/graphRAG.js +385 -0
  231. package/dist/lib/rag/graphRag/index.d.ts +4 -0
  232. package/dist/lib/rag/graphRag/index.js +5 -0
  233. package/dist/lib/rag/index.d.ts +103 -0
  234. package/dist/lib/rag/index.js +142 -0
  235. package/dist/lib/rag/metadata/MetadataExtractorFactory.d.ts +157 -0
  236. package/dist/lib/rag/metadata/MetadataExtractorFactory.js +419 -0
  237. package/dist/lib/rag/metadata/MetadataExtractorRegistry.d.ts +99 -0
  238. package/dist/lib/rag/metadata/MetadataExtractorRegistry.js +363 -0
  239. package/dist/lib/rag/metadata/index.d.ts +6 -0
  240. package/dist/lib/rag/metadata/index.js +10 -0
  241. package/dist/lib/rag/metadata/metadataExtractor.d.ts +69 -0
  242. package/dist/lib/rag/metadata/metadataExtractor.js +278 -0
  243. package/dist/lib/rag/pipeline/RAGPipeline.d.ts +235 -0
  244. package/dist/lib/rag/pipeline/RAGPipeline.js +402 -0
  245. package/dist/lib/rag/pipeline/contextAssembly.d.ts +126 -0
  246. package/dist/lib/rag/pipeline/contextAssembly.js +338 -0
  247. package/dist/lib/rag/pipeline/index.d.ts +5 -0
  248. package/dist/lib/rag/pipeline/index.js +6 -0
  249. package/dist/lib/rag/ragIntegration.d.ts +38 -0
  250. package/dist/lib/rag/ragIntegration.js +212 -0
  251. package/dist/lib/rag/reranker/RerankerFactory.d.ts +184 -0
  252. package/dist/lib/rag/reranker/RerankerFactory.js +431 -0
  253. package/dist/lib/rag/reranker/RerankerRegistry.d.ts +119 -0
  254. package/dist/lib/rag/reranker/RerankerRegistry.js +403 -0
  255. package/dist/lib/rag/reranker/index.d.ts +6 -0
  256. package/dist/lib/rag/reranker/index.js +10 -0
  257. package/dist/lib/rag/reranker/reranker.d.ts +71 -0
  258. package/dist/lib/rag/reranker/reranker.js +278 -0
  259. package/dist/lib/rag/resilience/CircuitBreaker.d.ts +215 -0
  260. package/dist/lib/rag/resilience/CircuitBreaker.js +432 -0
  261. package/dist/lib/rag/resilience/RetryHandler.d.ts +115 -0
  262. package/dist/lib/rag/resilience/RetryHandler.js +301 -0
  263. package/dist/lib/rag/resilience/index.d.ts +7 -0
  264. package/dist/lib/rag/resilience/index.js +8 -0
  265. package/dist/lib/rag/retrieval/hybridSearch.d.ts +94 -0
  266. package/dist/lib/rag/retrieval/hybridSearch.js +314 -0
  267. package/dist/lib/rag/retrieval/index.d.ts +5 -0
  268. package/dist/lib/rag/retrieval/index.js +6 -0
  269. package/dist/lib/rag/retrieval/vectorQueryTool.d.ts +93 -0
  270. package/dist/lib/rag/retrieval/vectorQueryTool.js +290 -0
  271. package/dist/lib/rag/types.d.ts +768 -0
  272. package/dist/lib/rag/types.js +9 -0
  273. package/dist/lib/server/index.d.ts +15 -11
  274. package/dist/lib/server/index.js +55 -51
  275. package/dist/lib/server/utils/validation.d.ts +2 -2
  276. package/dist/lib/types/common.d.ts +0 -1
  277. package/dist/lib/types/fileTypes.d.ts +1 -1
  278. package/dist/lib/types/generateTypes.d.ts +42 -8
  279. package/dist/lib/types/generateTypes.js +1 -1
  280. package/dist/lib/types/index.d.ts +25 -24
  281. package/dist/lib/types/index.js +21 -20
  282. package/dist/lib/types/modelTypes.d.ts +16 -16
  283. package/dist/lib/types/pptTypes.d.ts +14 -2
  284. package/dist/lib/types/pptTypes.js +16 -0
  285. package/dist/lib/types/streamTypes.d.ts +28 -8
  286. package/dist/lib/types/streamTypes.js +1 -1
  287. package/dist/lib/utils/async/delay.d.ts +40 -0
  288. package/dist/lib/utils/async/delay.js +43 -0
  289. package/dist/lib/utils/async/index.d.ts +23 -0
  290. package/dist/lib/utils/async/index.js +24 -0
  291. package/dist/lib/utils/async/retry.d.ts +141 -0
  292. package/dist/lib/utils/async/retry.js +172 -0
  293. package/dist/lib/utils/async/withTimeout.d.ts +73 -0
  294. package/dist/lib/utils/async/withTimeout.js +97 -0
  295. package/dist/lib/utils/fileDetector.d.ts +7 -1
  296. package/dist/lib/utils/fileDetector.js +91 -18
  297. package/dist/lib/utils/json/extract.d.ts +103 -0
  298. package/dist/lib/utils/json/extract.js +249 -0
  299. package/dist/lib/utils/json/index.d.ts +36 -0
  300. package/dist/lib/utils/json/index.js +37 -0
  301. package/dist/lib/utils/json/safeParse.d.ts +137 -0
  302. package/dist/lib/utils/json/safeParse.js +191 -0
  303. package/dist/lib/utils/messageBuilder.d.ts +2 -2
  304. package/dist/lib/utils/messageBuilder.js +15 -7
  305. package/dist/lib/utils/modelRouter.d.ts +4 -4
  306. package/dist/lib/utils/modelRouter.js +4 -4
  307. package/dist/lib/utils/sanitizers/filename.d.ts +137 -0
  308. package/dist/lib/utils/sanitizers/filename.js +366 -0
  309. package/dist/lib/utils/sanitizers/html.d.ts +170 -0
  310. package/dist/lib/utils/sanitizers/html.js +326 -0
  311. package/dist/lib/utils/sanitizers/index.d.ts +26 -0
  312. package/dist/lib/utils/sanitizers/index.js +30 -0
  313. package/dist/lib/utils/sanitizers/svg.d.ts +81 -0
  314. package/dist/lib/utils/sanitizers/svg.js +483 -0
  315. package/dist/mcp/index.d.ts +6 -5
  316. package/dist/mcp/index.js +7 -5
  317. package/dist/neurolink.d.ts +11 -13
  318. package/dist/neurolink.js +95 -29
  319. package/dist/processors/base/BaseFileProcessor.d.ts +273 -0
  320. package/dist/processors/base/BaseFileProcessor.js +613 -0
  321. package/dist/processors/base/index.d.ts +14 -0
  322. package/dist/processors/base/index.js +19 -0
  323. package/dist/processors/base/types.d.ts +593 -0
  324. package/dist/processors/base/types.js +76 -0
  325. package/dist/processors/cli/fileProcessorCli.d.ts +163 -0
  326. package/dist/processors/cli/fileProcessorCli.js +388 -0
  327. package/dist/processors/cli/index.d.ts +37 -0
  328. package/dist/processors/cli/index.js +49 -0
  329. package/dist/processors/code/ConfigProcessor.d.ts +171 -0
  330. package/dist/processors/code/ConfigProcessor.js +400 -0
  331. package/dist/processors/code/SourceCodeProcessor.d.ts +174 -0
  332. package/dist/processors/code/SourceCodeProcessor.js +304 -0
  333. package/dist/processors/code/index.d.ts +44 -0
  334. package/dist/processors/code/index.js +60 -0
  335. package/dist/processors/config/fileTypes.d.ts +283 -0
  336. package/dist/processors/config/fileTypes.js +520 -0
  337. package/dist/processors/config/index.d.ts +32 -0
  338. package/dist/processors/config/index.js +92 -0
  339. package/dist/processors/config/languageMap.d.ts +66 -0
  340. package/dist/processors/config/languageMap.js +410 -0
  341. package/dist/processors/config/mimeTypes.d.ts +376 -0
  342. package/dist/processors/config/mimeTypes.js +338 -0
  343. package/dist/processors/config/sizeLimits.d.ts +194 -0
  344. package/dist/processors/config/sizeLimits.js +246 -0
  345. package/dist/processors/data/JsonProcessor.d.ts +122 -0
  346. package/dist/processors/data/JsonProcessor.js +203 -0
  347. package/dist/processors/data/XmlProcessor.d.ts +160 -0
  348. package/dist/processors/data/XmlProcessor.js +283 -0
  349. package/dist/processors/data/YamlProcessor.d.ts +163 -0
  350. package/dist/processors/data/YamlProcessor.js +294 -0
  351. package/dist/processors/data/index.d.ts +49 -0
  352. package/dist/processors/data/index.js +76 -0
  353. package/dist/processors/document/ExcelProcessor.d.ts +238 -0
  354. package/dist/processors/document/ExcelProcessor.js +519 -0
  355. package/dist/processors/document/OpenDocumentProcessor.d.ts +69 -0
  356. package/dist/processors/document/OpenDocumentProcessor.js +210 -0
  357. package/dist/processors/document/RtfProcessor.d.ts +152 -0
  358. package/dist/processors/document/RtfProcessor.js +361 -0
  359. package/dist/processors/document/WordProcessor.d.ts +168 -0
  360. package/dist/processors/document/WordProcessor.js +353 -0
  361. package/dist/processors/document/index.d.ts +54 -0
  362. package/dist/processors/document/index.js +90 -0
  363. package/dist/processors/errors/FileErrorCode.d.ts +98 -0
  364. package/dist/processors/errors/FileErrorCode.js +255 -0
  365. package/dist/processors/errors/errorHelpers.d.ts +151 -0
  366. package/dist/processors/errors/errorHelpers.js +378 -0
  367. package/dist/processors/errors/errorSerializer.d.ts +139 -0
  368. package/dist/processors/errors/errorSerializer.js +507 -0
  369. package/dist/processors/errors/index.d.ts +46 -0
  370. package/dist/processors/errors/index.js +49 -0
  371. package/dist/processors/index.d.ts +76 -0
  372. package/dist/processors/index.js +112 -0
  373. package/dist/processors/integration/FileProcessorIntegration.d.ts +244 -0
  374. package/dist/processors/integration/FileProcessorIntegration.js +272 -0
  375. package/dist/processors/integration/index.d.ts +42 -0
  376. package/dist/processors/integration/index.js +44 -0
  377. package/dist/processors/markup/HtmlProcessor.d.ts +169 -0
  378. package/dist/processors/markup/HtmlProcessor.js +249 -0
  379. package/dist/processors/markup/MarkdownProcessor.d.ts +165 -0
  380. package/dist/processors/markup/MarkdownProcessor.js +244 -0
  381. package/dist/processors/markup/SvgProcessor.d.ts +156 -0
  382. package/dist/processors/markup/SvgProcessor.js +240 -0
  383. package/dist/processors/markup/TextProcessor.d.ts +135 -0
  384. package/dist/processors/markup/TextProcessor.js +188 -0
  385. package/dist/processors/markup/index.d.ts +66 -0
  386. package/dist/processors/markup/index.js +102 -0
  387. package/dist/processors/registry/ProcessorRegistry.d.ts +334 -0
  388. package/dist/processors/registry/ProcessorRegistry.js +608 -0
  389. package/dist/processors/registry/index.d.ts +12 -0
  390. package/dist/processors/registry/index.js +16 -0
  391. package/dist/processors/registry/types.d.ts +53 -0
  392. package/dist/processors/registry/types.js +10 -0
  393. package/dist/providers/amazonBedrock.d.ts +15 -2
  394. package/dist/providers/amazonBedrock.js +65 -8
  395. package/dist/providers/anthropic.d.ts +3 -3
  396. package/dist/providers/anthropic.js +10 -7
  397. package/dist/providers/googleAiStudio.d.ts +5 -5
  398. package/dist/providers/googleAiStudio.js +10 -7
  399. package/dist/providers/googleVertex.d.ts +16 -4
  400. package/dist/providers/googleVertex.js +72 -16
  401. package/dist/providers/litellm.d.ts +3 -3
  402. package/dist/providers/litellm.js +10 -10
  403. package/dist/providers/mistral.d.ts +3 -3
  404. package/dist/providers/mistral.js +7 -6
  405. package/dist/providers/ollama.d.ts +3 -4
  406. package/dist/providers/ollama.js +7 -8
  407. package/dist/providers/openAI.d.ts +14 -2
  408. package/dist/providers/openAI.js +60 -6
  409. package/dist/providers/openRouter.d.ts +2 -2
  410. package/dist/providers/openRouter.js +10 -6
  411. package/dist/rag/ChunkerFactory.d.ts +91 -0
  412. package/dist/rag/ChunkerFactory.js +320 -0
  413. package/dist/rag/ChunkerRegistry.d.ts +91 -0
  414. package/dist/rag/ChunkerRegistry.js +421 -0
  415. package/dist/rag/chunkers/BaseChunker.d.ts +53 -0
  416. package/dist/rag/chunkers/BaseChunker.js +143 -0
  417. package/dist/rag/chunkers/CharacterChunker.d.ts +18 -0
  418. package/dist/rag/chunkers/CharacterChunker.js +28 -0
  419. package/dist/rag/chunkers/HTMLChunker.d.ts +19 -0
  420. package/dist/rag/chunkers/HTMLChunker.js +38 -0
  421. package/dist/rag/chunkers/JSONChunker.d.ts +19 -0
  422. package/dist/rag/chunkers/JSONChunker.js +68 -0
  423. package/dist/rag/chunkers/LaTeXChunker.d.ts +15 -0
  424. package/dist/rag/chunkers/LaTeXChunker.js +63 -0
  425. package/dist/rag/chunkers/MarkdownChunker.d.ts +15 -0
  426. package/dist/rag/chunkers/MarkdownChunker.js +102 -0
  427. package/dist/rag/chunkers/RecursiveChunker.d.ts +27 -0
  428. package/dist/rag/chunkers/RecursiveChunker.js +139 -0
  429. package/dist/rag/chunkers/SemanticMarkdownChunker.d.ts +22 -0
  430. package/dist/rag/chunkers/SemanticMarkdownChunker.js +138 -0
  431. package/dist/rag/chunkers/SentenceChunker.d.ts +19 -0
  432. package/dist/rag/chunkers/SentenceChunker.js +66 -0
  433. package/dist/rag/chunkers/TokenChunker.d.ts +19 -0
  434. package/dist/rag/chunkers/TokenChunker.js +61 -0
  435. package/dist/rag/chunkers/index.d.ts +15 -0
  436. package/dist/rag/chunkers/index.js +15 -0
  437. package/dist/rag/chunking/characterChunker.d.ts +16 -0
  438. package/dist/rag/chunking/characterChunker.js +142 -0
  439. package/dist/rag/chunking/chunkerRegistry.d.ts +67 -0
  440. package/dist/rag/chunking/chunkerRegistry.js +194 -0
  441. package/dist/rag/chunking/htmlChunker.d.ts +34 -0
  442. package/dist/rag/chunking/htmlChunker.js +247 -0
  443. package/dist/rag/chunking/index.d.ts +15 -0
  444. package/dist/rag/chunking/index.js +17 -0
  445. package/dist/rag/chunking/jsonChunker.d.ts +20 -0
  446. package/dist/rag/chunking/jsonChunker.js +281 -0
  447. package/dist/rag/chunking/latexChunker.d.ts +26 -0
  448. package/dist/rag/chunking/latexChunker.js +251 -0
  449. package/dist/rag/chunking/markdownChunker.d.ts +19 -0
  450. package/dist/rag/chunking/markdownChunker.js +201 -0
  451. package/dist/rag/chunking/recursiveChunker.d.ts +19 -0
  452. package/dist/rag/chunking/recursiveChunker.js +148 -0
  453. package/dist/rag/chunking/semanticChunker.d.ts +41 -0
  454. package/dist/rag/chunking/semanticChunker.js +306 -0
  455. package/dist/rag/chunking/sentenceChunker.d.ts +25 -0
  456. package/dist/rag/chunking/sentenceChunker.js +230 -0
  457. package/dist/rag/chunking/tokenChunker.d.ts +36 -0
  458. package/dist/rag/chunking/tokenChunker.js +183 -0
  459. package/dist/rag/document/MDocument.d.ts +198 -0
  460. package/dist/rag/document/MDocument.js +392 -0
  461. package/dist/rag/document/index.d.ts +5 -0
  462. package/dist/rag/document/index.js +5 -0
  463. package/dist/rag/document/loaders.d.ts +201 -0
  464. package/dist/rag/document/loaders.js +500 -0
  465. package/dist/rag/errors/RAGError.d.ts +244 -0
  466. package/dist/rag/errors/RAGError.js +274 -0
  467. package/dist/rag/errors/index.d.ts +6 -0
  468. package/dist/rag/errors/index.js +6 -0
  469. package/dist/rag/graphRag/graphRAG.d.ts +115 -0
  470. package/dist/rag/graphRag/graphRAG.js +384 -0
  471. package/dist/rag/graphRag/index.d.ts +4 -0
  472. package/dist/rag/graphRag/index.js +4 -0
  473. package/dist/rag/index.d.ts +103 -0
  474. package/dist/rag/index.js +141 -0
  475. package/dist/rag/metadata/MetadataExtractorFactory.d.ts +157 -0
  476. package/dist/rag/metadata/MetadataExtractorFactory.js +418 -0
  477. package/dist/rag/metadata/MetadataExtractorRegistry.d.ts +99 -0
  478. package/dist/rag/metadata/MetadataExtractorRegistry.js +362 -0
  479. package/dist/rag/metadata/index.d.ts +6 -0
  480. package/dist/rag/metadata/index.js +9 -0
  481. package/dist/rag/metadata/metadataExtractor.d.ts +69 -0
  482. package/dist/rag/metadata/metadataExtractor.js +277 -0
  483. package/dist/rag/pipeline/RAGPipeline.d.ts +235 -0
  484. package/dist/rag/pipeline/RAGPipeline.js +401 -0
  485. package/dist/rag/pipeline/contextAssembly.d.ts +126 -0
  486. package/dist/rag/pipeline/contextAssembly.js +337 -0
  487. package/dist/rag/pipeline/index.d.ts +5 -0
  488. package/dist/rag/pipeline/index.js +5 -0
  489. package/dist/rag/ragIntegration.d.ts +38 -0
  490. package/dist/rag/ragIntegration.js +211 -0
  491. package/dist/rag/reranker/RerankerFactory.d.ts +184 -0
  492. package/dist/rag/reranker/RerankerFactory.js +430 -0
  493. package/dist/rag/reranker/RerankerRegistry.d.ts +119 -0
  494. package/dist/rag/reranker/RerankerRegistry.js +402 -0
  495. package/dist/rag/reranker/index.d.ts +6 -0
  496. package/dist/rag/reranker/index.js +9 -0
  497. package/dist/rag/reranker/reranker.d.ts +71 -0
  498. package/dist/rag/reranker/reranker.js +277 -0
  499. package/dist/rag/resilience/CircuitBreaker.d.ts +215 -0
  500. package/dist/rag/resilience/CircuitBreaker.js +431 -0
  501. package/dist/rag/resilience/RetryHandler.d.ts +115 -0
  502. package/dist/rag/resilience/RetryHandler.js +300 -0
  503. package/dist/rag/resilience/index.d.ts +7 -0
  504. package/dist/rag/resilience/index.js +7 -0
  505. package/dist/rag/retrieval/hybridSearch.d.ts +94 -0
  506. package/dist/rag/retrieval/hybridSearch.js +313 -0
  507. package/dist/rag/retrieval/index.d.ts +5 -0
  508. package/dist/rag/retrieval/index.js +5 -0
  509. package/dist/rag/retrieval/vectorQueryTool.d.ts +93 -0
  510. package/dist/rag/retrieval/vectorQueryTool.js +289 -0
  511. package/dist/rag/types.d.ts +768 -0
  512. package/dist/rag/types.js +8 -0
  513. package/dist/server/index.d.ts +15 -11
  514. package/dist/server/index.js +55 -51
  515. package/dist/server/utils/validation.d.ts +8 -8
  516. package/dist/types/common.d.ts +0 -1
  517. package/dist/types/fileTypes.d.ts +1 -1
  518. package/dist/types/generateTypes.d.ts +42 -8
  519. package/dist/types/generateTypes.js +1 -1
  520. package/dist/types/index.d.ts +25 -24
  521. package/dist/types/index.js +21 -20
  522. package/dist/types/modelTypes.d.ts +10 -10
  523. package/dist/types/pptTypes.d.ts +14 -2
  524. package/dist/types/pptTypes.js +16 -0
  525. package/dist/types/streamTypes.d.ts +28 -8
  526. package/dist/types/streamTypes.js +1 -1
  527. package/dist/utils/async/delay.d.ts +40 -0
  528. package/dist/utils/async/delay.js +42 -0
  529. package/dist/utils/async/index.d.ts +23 -0
  530. package/dist/utils/async/index.js +23 -0
  531. package/dist/utils/async/retry.d.ts +141 -0
  532. package/dist/utils/async/retry.js +171 -0
  533. package/dist/utils/async/withTimeout.d.ts +73 -0
  534. package/dist/utils/async/withTimeout.js +96 -0
  535. package/dist/utils/fileDetector.d.ts +7 -1
  536. package/dist/utils/fileDetector.js +91 -18
  537. package/dist/utils/json/extract.d.ts +103 -0
  538. package/dist/utils/json/extract.js +248 -0
  539. package/dist/utils/json/index.d.ts +36 -0
  540. package/dist/utils/json/index.js +36 -0
  541. package/dist/utils/json/safeParse.d.ts +137 -0
  542. package/dist/utils/json/safeParse.js +190 -0
  543. package/dist/utils/messageBuilder.d.ts +2 -2
  544. package/dist/utils/messageBuilder.js +15 -7
  545. package/dist/utils/modelRouter.d.ts +4 -4
  546. package/dist/utils/modelRouter.js +4 -4
  547. package/dist/utils/sanitizers/filename.d.ts +137 -0
  548. package/dist/utils/sanitizers/filename.js +365 -0
  549. package/dist/utils/sanitizers/html.d.ts +170 -0
  550. package/dist/utils/sanitizers/html.js +325 -0
  551. package/dist/utils/sanitizers/index.d.ts +26 -0
  552. package/dist/utils/sanitizers/index.js +29 -0
  553. package/dist/utils/sanitizers/svg.d.ts +81 -0
  554. package/dist/utils/sanitizers/svg.js +482 -0
  555. package/package.json +2 -2
@@ -0,0 +1,385 @@
1
+ /**
2
+ * Graph RAG Implementation
3
+ *
4
+ * Knowledge graph-based retrieval augmented generation.
5
+ * Creates semantic relationships between document chunks and uses
6
+ * random walk with restart for context-aware retrieval.
7
+ */
8
+ import { randomUUID } from "crypto";
9
+ import { logger } from "../../utils/logger.js";
10
+ /**
11
+ * Graph-based Retrieval Augmented Generation
12
+ *
13
+ * Creates a knowledge graph from document chunks where nodes represent
14
+ * documents and edges represent semantic relationships based on
15
+ * embedding similarity.
16
+ */
17
+ export class GraphRAG {
18
+ nodes = new Map();
19
+ edges = new Map();
20
+ dimension;
21
+ threshold;
22
+ constructor(config) {
23
+ this.dimension = config?.dimension ?? 1536;
24
+ this.threshold = config?.threshold ?? 0.7;
25
+ }
26
+ /**
27
+ * Create a knowledge graph from document chunks and embeddings
28
+ *
29
+ * @param chunks - Array of document chunks
30
+ * @param embeddings - Corresponding embedding vectors
31
+ */
32
+ createGraph(chunks, embeddings) {
33
+ if (chunks.length !== embeddings.length) {
34
+ throw new Error("Chunks and embeddings arrays must have the same length");
35
+ }
36
+ // Clear existing graph
37
+ this.nodes.clear();
38
+ this.edges.clear();
39
+ // Create nodes
40
+ const nodeIds = [];
41
+ for (let i = 0; i < chunks.length; i++) {
42
+ const id = randomUUID();
43
+ nodeIds.push(id);
44
+ this.nodes.set(id, {
45
+ id,
46
+ content: chunks[i].text,
47
+ metadata: chunks[i].metadata || {},
48
+ embedding: embeddings[i].vector,
49
+ });
50
+ }
51
+ // Create edges based on semantic similarity
52
+ for (let i = 0; i < nodeIds.length; i++) {
53
+ const edges = [];
54
+ const nodeA = this.nodes.get(nodeIds[i]);
55
+ for (let j = 0; j < nodeIds.length; j++) {
56
+ if (i === j) {
57
+ continue;
58
+ }
59
+ const nodeB = this.nodes.get(nodeIds[j]);
60
+ const similarity = this.cosineSimilarity(nodeA.embedding, nodeB.embedding);
61
+ if (similarity >= this.threshold) {
62
+ edges.push({
63
+ source: nodeIds[i],
64
+ target: nodeIds[j],
65
+ weight: similarity,
66
+ type: "semantic",
67
+ });
68
+ }
69
+ }
70
+ // Sort edges by weight descending
71
+ edges.sort((a, b) => b.weight - a.weight);
72
+ this.edges.set(nodeIds[i], edges);
73
+ }
74
+ logger.info("[GraphRAG] Graph created", {
75
+ nodes: this.nodes.size,
76
+ totalEdges: Array.from(this.edges.values()).reduce((sum, e) => sum + e.length, 0),
77
+ threshold: this.threshold,
78
+ });
79
+ }
80
+ /**
81
+ * Query the graph using random walk with restart
82
+ *
83
+ * @param params - Query parameters including embedding vector
84
+ * @returns Ranked nodes by relevance
85
+ */
86
+ query(params) {
87
+ const { query, topK = 10, randomWalkSteps = 100, restartProb = 0.15, } = params;
88
+ if (this.nodes.size === 0) {
89
+ return [];
90
+ }
91
+ // Calculate initial similarities to query
92
+ const similarities = new Map();
93
+ for (const [id, node] of this.nodes) {
94
+ if (node.embedding) {
95
+ similarities.set(id, this.cosineSimilarity(query, node.embedding));
96
+ }
97
+ }
98
+ // Find starting nodes (most similar to query)
99
+ const sortedNodes = Array.from(similarities.entries())
100
+ .sort((a, b) => b[1] - a[1])
101
+ .slice(0, Math.min(5, this.nodes.size));
102
+ if (sortedNodes.length === 0) {
103
+ return [];
104
+ }
105
+ // Random walk with restart
106
+ const visitCounts = new Map();
107
+ const startNodeIds = sortedNodes.map(([id]) => id);
108
+ const startProbs = this.normalizeProbs(sortedNodes.map(([, sim]) => sim));
109
+ for (let step = 0; step < randomWalkSteps; step++) {
110
+ // Choose starting node based on query similarity
111
+ const startIdx = this.weightedRandomChoice(startProbs);
112
+ let currentNode = startNodeIds[startIdx];
113
+ // Walk with restart probability
114
+ if (Math.random() >= restartProb) {
115
+ const edges = this.edges.get(currentNode) || [];
116
+ if (edges.length > 0) {
117
+ // Choose next node based on edge weights
118
+ const edgeWeights = edges.map((e) => e.weight);
119
+ const normalizedWeights = this.normalizeProbs(edgeWeights);
120
+ const nextIdx = this.weightedRandomChoice(normalizedWeights);
121
+ currentNode = edges[nextIdx].target;
122
+ }
123
+ }
124
+ // Update visit count
125
+ visitCounts.set(currentNode, (visitCounts.get(currentNode) || 0) + 1);
126
+ }
127
+ // Combine visit frequency with query similarity for final ranking
128
+ const scores = new Map();
129
+ const maxVisits = Math.max(...visitCounts.values());
130
+ for (const [id] of this.nodes) {
131
+ const visitScore = (visitCounts.get(id) || 0) / maxVisits;
132
+ const similarityScore = similarities.get(id) || 0;
133
+ // Weighted combination
134
+ scores.set(id, 0.6 * similarityScore + 0.4 * visitScore);
135
+ }
136
+ // Sort and return top K
137
+ const rankedNodes = Array.from(scores.entries())
138
+ .sort((a, b) => b[1] - a[1])
139
+ .slice(0, topK)
140
+ .map(([id, score]) => {
141
+ const node = this.nodes.get(id);
142
+ return {
143
+ id,
144
+ content: node.content,
145
+ metadata: node.metadata,
146
+ score,
147
+ };
148
+ });
149
+ logger.debug("[GraphRAG] Query completed", {
150
+ topK,
151
+ resultsCount: rankedNodes.length,
152
+ topScore: rankedNodes[0]?.score,
153
+ });
154
+ return rankedNodes;
155
+ }
156
+ /**
157
+ * Add a single node to the graph
158
+ *
159
+ * @param chunk - Document chunk
160
+ * @param embedding - Embedding vector
161
+ * @returns Node ID
162
+ */
163
+ addNode(chunk, embedding) {
164
+ const id = randomUUID();
165
+ this.nodes.set(id, {
166
+ id,
167
+ content: chunk.text,
168
+ metadata: chunk.metadata || {},
169
+ embedding: embedding.vector,
170
+ });
171
+ // Create edges to existing nodes
172
+ const edges = [];
173
+ for (const [existingId, existingNode] of this.nodes) {
174
+ if (existingId === id) {
175
+ continue;
176
+ }
177
+ const similarity = this.cosineSimilarity(embedding.vector, existingNode.embedding);
178
+ if (similarity >= this.threshold) {
179
+ edges.push({
180
+ source: id,
181
+ target: existingId,
182
+ weight: similarity,
183
+ type: "semantic",
184
+ });
185
+ // Add reverse edge
186
+ const existingEdges = this.edges.get(existingId) || [];
187
+ existingEdges.push({
188
+ source: existingId,
189
+ target: id,
190
+ weight: similarity,
191
+ type: "semantic",
192
+ });
193
+ this.edges.set(existingId, existingEdges);
194
+ }
195
+ }
196
+ this.edges.set(id, edges);
197
+ return id;
198
+ }
199
+ /**
200
+ * Remove a node and its edges from the graph
201
+ *
202
+ * @param id - Node ID to remove
203
+ * @returns True if node was removed
204
+ */
205
+ removeNode(id) {
206
+ if (!this.nodes.has(id)) {
207
+ return false;
208
+ }
209
+ // Remove node
210
+ this.nodes.delete(id);
211
+ this.edges.delete(id);
212
+ // Remove edges pointing to this node
213
+ for (const [nodeId, edges] of this.edges) {
214
+ this.edges.set(nodeId, edges.filter((e) => e.target !== id));
215
+ }
216
+ return true;
217
+ }
218
+ /**
219
+ * Get graph statistics
220
+ */
221
+ getStats() {
222
+ const edgeCount = Array.from(this.edges.values()).reduce((sum, e) => sum + e.length, 0);
223
+ return {
224
+ nodeCount: this.nodes.size,
225
+ edgeCount,
226
+ avgDegree: this.nodes.size > 0 ? edgeCount / this.nodes.size : 0,
227
+ threshold: this.threshold,
228
+ };
229
+ }
230
+ /**
231
+ * Get a node by ID
232
+ */
233
+ getNode(id) {
234
+ return this.nodes.get(id);
235
+ }
236
+ /**
237
+ * Get all nodes
238
+ */
239
+ getAllNodes() {
240
+ return Array.from(this.nodes.values());
241
+ }
242
+ /**
243
+ * Get edges for a node
244
+ */
245
+ getEdges(nodeId) {
246
+ return this.edges.get(nodeId) || [];
247
+ }
248
+ /**
249
+ * Find connected components in the graph
250
+ */
251
+ findConnectedComponents() {
252
+ const visited = new Set();
253
+ const components = [];
254
+ for (const nodeId of this.nodes.keys()) {
255
+ if (visited.has(nodeId)) {
256
+ continue;
257
+ }
258
+ const component = [];
259
+ const queue = [nodeId];
260
+ while (queue.length > 0) {
261
+ const current = queue.shift();
262
+ if (visited.has(current)) {
263
+ continue;
264
+ }
265
+ visited.add(current);
266
+ component.push(current);
267
+ const edges = this.edges.get(current) || [];
268
+ for (const edge of edges) {
269
+ if (!visited.has(edge.target)) {
270
+ queue.push(edge.target);
271
+ }
272
+ }
273
+ }
274
+ components.push(component);
275
+ }
276
+ return components;
277
+ }
278
+ /**
279
+ * Calculate cosine similarity between two vectors
280
+ */
281
+ cosineSimilarity(a, b) {
282
+ if (a.length !== b.length) {
283
+ throw new Error("Vectors must have the same dimension");
284
+ }
285
+ let dotProduct = 0;
286
+ let normA = 0;
287
+ let normB = 0;
288
+ for (let i = 0; i < a.length; i++) {
289
+ dotProduct += a[i] * b[i];
290
+ normA += a[i] * a[i];
291
+ normB += b[i] * b[i];
292
+ }
293
+ const denominator = Math.sqrt(normA) * Math.sqrt(normB);
294
+ return denominator === 0 ? 0 : dotProduct / denominator;
295
+ }
296
+ /**
297
+ * Normalize probabilities to sum to 1
298
+ */
299
+ normalizeProbs(probs) {
300
+ const sum = probs.reduce((a, b) => a + b, 0);
301
+ return sum === 0
302
+ ? probs.map(() => 1 / probs.length)
303
+ : probs.map((p) => p / sum);
304
+ }
305
+ /**
306
+ * Weighted random choice
307
+ */
308
+ weightedRandomChoice(weights) {
309
+ const random = Math.random();
310
+ let cumulative = 0;
311
+ for (let i = 0; i < weights.length; i++) {
312
+ cumulative += weights[i];
313
+ if (random <= cumulative) {
314
+ return i;
315
+ }
316
+ }
317
+ return weights.length - 1;
318
+ }
319
+ /**
320
+ * Update similarity threshold and rebuild edges
321
+ */
322
+ updateThreshold(threshold) {
323
+ this.threshold = threshold;
324
+ // Rebuild edges with new threshold
325
+ this.edges.clear();
326
+ const nodeIds = Array.from(this.nodes.keys());
327
+ for (let i = 0; i < nodeIds.length; i++) {
328
+ const edges = [];
329
+ const nodeA = this.nodes.get(nodeIds[i]);
330
+ for (let j = 0; j < nodeIds.length; j++) {
331
+ if (i === j) {
332
+ continue;
333
+ }
334
+ const nodeB = this.nodes.get(nodeIds[j]);
335
+ if (!nodeA.embedding || !nodeB.embedding) {
336
+ continue;
337
+ }
338
+ const similarity = this.cosineSimilarity(nodeA.embedding, nodeB.embedding);
339
+ if (similarity >= threshold) {
340
+ edges.push({
341
+ source: nodeIds[i],
342
+ target: nodeIds[j],
343
+ weight: similarity,
344
+ type: "semantic",
345
+ });
346
+ }
347
+ }
348
+ edges.sort((a, b) => b.weight - a.weight);
349
+ this.edges.set(nodeIds[i], edges);
350
+ }
351
+ }
352
+ /**
353
+ * Serialize graph to JSON
354
+ */
355
+ toJSON() {
356
+ return {
357
+ nodes: Array.from(this.nodes.values()),
358
+ edges: Array.from(this.edges.entries()).map(([source, edges]) => ({
359
+ source,
360
+ edges,
361
+ })),
362
+ config: {
363
+ dimension: this.dimension,
364
+ threshold: this.threshold,
365
+ },
366
+ };
367
+ }
368
+ /**
369
+ * Load graph from JSON
370
+ */
371
+ static fromJSON(json) {
372
+ const graph = new GraphRAG({
373
+ dimension: json.config.dimension,
374
+ threshold: json.config.threshold,
375
+ });
376
+ for (const node of json.nodes) {
377
+ graph.nodes.set(node.id, node);
378
+ }
379
+ for (const { source, edges } of json.edges) {
380
+ graph.edges.set(source, edges);
381
+ }
382
+ return graph;
383
+ }
384
+ }
385
+ //# sourceMappingURL=graphRAG.js.map
@@ -0,0 +1,4 @@
1
+ /**
2
+ * Graph RAG Module Exports
3
+ */
4
+ export { GraphRAG } from "./graphRAG.js";
@@ -0,0 +1,5 @@
1
+ /**
2
+ * Graph RAG Module Exports
3
+ */
4
+ export { GraphRAG } from "./graphRAG.js";
5
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1,103 @@
1
+ /**
2
+ * RAG Document Processing Module
3
+ *
4
+ * Provides comprehensive RAG (Retrieval-Augmented Generation) capabilities:
5
+ * - Document loading (text, markdown, HTML, JSON, CSV, PDF, web)
6
+ * - MDocument class for fluent document processing
7
+ * - 10 chunking strategies (character, recursive, sentence, token, markdown, html, json, latex, semantic, semantic-markdown)
8
+ * - LLM-powered metadata extraction (title, summary, keywords, Q&A)
9
+ * - Vector query tools with metadata filtering and reranking
10
+ * - Hybrid search (BM25 + vector fusion)
11
+ * - Graph RAG for knowledge graph-based retrieval
12
+ * - RAG pipeline orchestration
13
+ * - Context assembly and formatting
14
+ * - ChunkerFactory and ChunkerRegistry patterns for extensibility
15
+ * - Error handling and resilience (CircuitBreaker, RetryHandler)
16
+ *
17
+ * @example
18
+ * ```typescript
19
+ * import {
20
+ * MDocument,
21
+ * loadDocument,
22
+ * RAGPipeline,
23
+ * ChunkerRegistry,
24
+ * ChunkerFactory,
25
+ * CircuitBreaker
26
+ * } from '@juspay/neurolink';
27
+ *
28
+ * // Load and process a document
29
+ * const doc = await loadDocument('/path/to/document.md');
30
+ * await doc.chunk({ strategy: 'markdown', config: { maxSize: 1000 } });
31
+ * await doc.embed('openai', 'text-embedding-3-small');
32
+ *
33
+ * // Or use the full RAG pipeline
34
+ * const pipeline = new RAGPipeline({
35
+ * embeddingModel: { provider: 'openai', modelName: 'text-embedding-3-small' },
36
+ * generationModel: { provider: 'openai', modelName: 'gpt-4o-mini' }
37
+ * });
38
+ * await pipeline.ingest(['/path/to/docs/*.md']);
39
+ * const response = await pipeline.query('What are the key features?');
40
+ *
41
+ * // Use factory pattern for chunker creation
42
+ * const chunker = await ChunkerFactory.createChunker('semantic', { maxSize: 500 });
43
+ * const chunks = await chunker.chunk(text);
44
+ * ```
45
+ */
46
+ export { ChunkerFactory, chunkerFactory, createChunker, getAvailableStrategies as getFactoryStrategies, getDefaultConfig as getFactoryDefaultConfig, } from "./ChunkerFactory.js";
47
+ export { ChunkerRegistry as ChunkerRegistryV2, chunkerRegistry, getAvailableChunkers, getChunker, getChunkerMetadata, } from "./ChunkerRegistry.js";
48
+ export * from "./chunkers/index.js";
49
+ export { CharacterChunker, ChunkerRegistry, chunkText, HTMLChunker, JSONChunker, LaTeXChunker, MarkdownChunker, RecursiveChunker, SemanticChunker, SentenceChunker, TokenChunker, } from "./chunking/index.js";
50
+ export { CSVLoader, type CSVLoaderOptions, type DocumentLoader, HTMLLoader, JSONLoader, type LoaderOptions, loadDocument, loadDocuments, MarkdownLoader, MDocument, PDFLoader, type PDFLoaderOptions, TextLoader, WebLoader, type WebLoaderOptions, } from "./document/index.js";
51
+ export * from "./errors/index.js";
52
+ export { GraphRAG } from "./graphRag/index.js";
53
+ export { createMetadataExtractor, extractMetadata, getAvailableExtractors, getAvailableExtractorTypes, getExtractor, getExtractorDefaultConfig, getExtractorMetadata, getRegisteredExtractorMetadata, LLMMetadataExtractor, type MetadataExtractor, type MetadataExtractorConfig, MetadataExtractorFactory, type MetadataExtractorMetadata, MetadataExtractorRegistry, type MetadataExtractorType, metadataExtractorFactory, metadataExtractorRegistry, } from "./metadata/index.js";
54
+ export { assembleContext, type CitationFormat, type ContextAssemblyOptions, type ContextWindow, createContextWindow, createRAGPipeline, type EmbeddingModelConfig, extractKeySentences, formatContextWithCitations, type GenerationModelConfig, type IngestOptions, orderByDocumentStructure, type PipelineStats, type QueryOptions, RAGPipeline, type RAGPipelineConfig, type RAGResponse, summarizeContext, } from "./pipeline/index.js";
55
+ export { prepareRAGTool, type RAGPreparedTool } from "./ragIntegration.js";
56
+ export { batchRerank, CohereRelevanceScorer, CrossEncoderReranker, createReranker, getAvailableRerankers, getAvailableRerankerTypes, getRegisteredRerankerMetadata, getReranker, getRerankerDefaultConfig, getRerankerMetadata, type Reranker, type RerankerConfig, RerankerFactory, type RerankerMetadata, RerankerRegistry, type RerankerType, rerank, rerankerFactory, rerankerRegistry, simpleRerank, } from "./reranker/index.js";
57
+ export * from "./resilience/index.js";
58
+ export { type BM25Index, createHybridSearch, createVectorQueryTool, type HybridSearchOptions, InMemoryBM25Index, InMemoryVectorStore, linearCombination, reciprocalRankFusion, type VectorStore, } from "./retrieval/index.js";
59
+ export * from "./types.js";
60
+ import type { Chunk, ChunkingStrategy, ExtractParams } from "./types.js";
61
+ /**
62
+ * Process a document through the full RAG pipeline
63
+ *
64
+ * @param text - Document text to process
65
+ * @param options - Processing options
66
+ * @returns Processed chunks with optional metadata
67
+ */
68
+ export declare function processDocument(text: string, options?: {
69
+ /** Chunking strategy (default: recursive) */
70
+ strategy?: ChunkingStrategy;
71
+ /** Maximum chunk size */
72
+ maxSize?: number;
73
+ /** Chunk overlap */
74
+ overlap?: number;
75
+ /** Metadata extraction options */
76
+ extract?: ExtractParams;
77
+ /** Provider for metadata extraction */
78
+ provider?: string;
79
+ /** Model for metadata extraction */
80
+ model?: string;
81
+ /** Custom metadata to add */
82
+ metadata?: Record<string, unknown>;
83
+ }): Promise<Chunk[]>;
84
+ /**
85
+ * Get recommended chunking strategy based on content type
86
+ *
87
+ * @param contentType - MIME type or file extension
88
+ * @returns Recommended chunking strategy
89
+ */
90
+ export declare function getRecommendedStrategy(contentType: string): ChunkingStrategy;
91
+ /**
92
+ * Get available chunking strategies
93
+ *
94
+ * @returns Array of available strategy names
95
+ */
96
+ export declare function getAvailableStrategies(): ChunkingStrategy[];
97
+ /**
98
+ * Get default configuration for a chunking strategy
99
+ *
100
+ * @param strategy - Chunking strategy name
101
+ * @returns Default configuration object
102
+ */
103
+ export declare function getDefaultChunkerConfig(strategy: ChunkingStrategy): Record<string, unknown>;
@@ -0,0 +1,142 @@
1
+ /**
2
+ * RAG Document Processing Module
3
+ *
4
+ * Provides comprehensive RAG (Retrieval-Augmented Generation) capabilities:
5
+ * - Document loading (text, markdown, HTML, JSON, CSV, PDF, web)
6
+ * - MDocument class for fluent document processing
7
+ * - 10 chunking strategies (character, recursive, sentence, token, markdown, html, json, latex, semantic, semantic-markdown)
8
+ * - LLM-powered metadata extraction (title, summary, keywords, Q&A)
9
+ * - Vector query tools with metadata filtering and reranking
10
+ * - Hybrid search (BM25 + vector fusion)
11
+ * - Graph RAG for knowledge graph-based retrieval
12
+ * - RAG pipeline orchestration
13
+ * - Context assembly and formatting
14
+ * - ChunkerFactory and ChunkerRegistry patterns for extensibility
15
+ * - Error handling and resilience (CircuitBreaker, RetryHandler)
16
+ *
17
+ * @example
18
+ * ```typescript
19
+ * import {
20
+ * MDocument,
21
+ * loadDocument,
22
+ * RAGPipeline,
23
+ * ChunkerRegistry,
24
+ * ChunkerFactory,
25
+ * CircuitBreaker
26
+ * } from '@juspay/neurolink';
27
+ *
28
+ * // Load and process a document
29
+ * const doc = await loadDocument('/path/to/document.md');
30
+ * await doc.chunk({ strategy: 'markdown', config: { maxSize: 1000 } });
31
+ * await doc.embed('openai', 'text-embedding-3-small');
32
+ *
33
+ * // Or use the full RAG pipeline
34
+ * const pipeline = new RAGPipeline({
35
+ * embeddingModel: { provider: 'openai', modelName: 'text-embedding-3-small' },
36
+ * generationModel: { provider: 'openai', modelName: 'gpt-4o-mini' }
37
+ * });
38
+ * await pipeline.ingest(['/path/to/docs/*.md']);
39
+ * const response = await pipeline.query('What are the key features?');
40
+ *
41
+ * // Use factory pattern for chunker creation
42
+ * const chunker = await ChunkerFactory.createChunker('semantic', { maxSize: 500 });
43
+ * const chunks = await chunker.chunk(text);
44
+ * ```
45
+ */
46
+ export { ChunkerFactory, chunkerFactory, createChunker, getAvailableStrategies as getFactoryStrategies, getDefaultConfig as getFactoryDefaultConfig, } from "./ChunkerFactory.js";
47
+ // ChunkerFactory and ChunkerRegistry patterns (from main worktree)
48
+ export { ChunkerRegistry as ChunkerRegistryV2, chunkerRegistry, getAvailableChunkers, getChunker, getChunkerMetadata, } from "./ChunkerRegistry.js";
49
+ // Base chunker and chunker implementations (from main worktree)
50
+ export * from "./chunkers/index.js";
51
+ // Chunking
52
+ export { CharacterChunker, ChunkerRegistry, chunkText, HTMLChunker, JSONChunker, LaTeXChunker, MarkdownChunker, RecursiveChunker, SemanticChunker, SentenceChunker, TokenChunker, } from "./chunking/index.js";
53
+ // Document Processing
54
+ export { CSVLoader, HTMLLoader, JSONLoader, loadDocument, loadDocuments, MarkdownLoader, MDocument, PDFLoader, TextLoader, WebLoader, } from "./document/index.js";
55
+ // Error handling
56
+ export * from "./errors/index.js";
57
+ // Graph RAG
58
+ export { GraphRAG } from "./graphRag/index.js";
59
+ // Metadata Extraction
60
+ export { createMetadataExtractor, extractMetadata, getAvailableExtractors, getAvailableExtractorTypes, getExtractor, getExtractorDefaultConfig, getExtractorMetadata, getRegisteredExtractorMetadata, LLMMetadataExtractor,
61
+ // Factory pattern
62
+ MetadataExtractorFactory,
63
+ // Registry pattern
64
+ MetadataExtractorRegistry, metadataExtractorFactory, metadataExtractorRegistry, } from "./metadata/index.js";
65
+ // Pipeline
66
+ export { assembleContext, createContextWindow, createRAGPipeline, extractKeySentences, formatContextWithCitations, orderByDocumentStructure, RAGPipeline, summarizeContext, } from "./pipeline/index.js";
67
+ // RAG Integration (for generate/stream)
68
+ export { prepareRAGTool } from "./ragIntegration.js";
69
+ // Reranker
70
+ export { batchRerank, CohereRelevanceScorer, CrossEncoderReranker, createReranker, getAvailableRerankers, getAvailableRerankerTypes, getRegisteredRerankerMetadata, getReranker, getRerankerDefaultConfig, getRerankerMetadata,
71
+ // Factory pattern
72
+ RerankerFactory,
73
+ // Registry pattern
74
+ RerankerRegistry, rerank, rerankerFactory, rerankerRegistry, simpleRerank, } from "./reranker/index.js";
75
+ // Resilience patterns
76
+ export * from "./resilience/index.js";
77
+ // Retrieval
78
+ export { createHybridSearch, createVectorQueryTool, InMemoryBM25Index, InMemoryVectorStore, linearCombination, reciprocalRankFusion, } from "./retrieval/index.js";
79
+ // Types
80
+ export * from "./types.js";
81
+ // Convenience functions
82
+ import { ChunkerRegistry } from "./chunking/index.js";
83
+ import { LLMMetadataExtractor } from "./metadata/index.js";
84
+ /**
85
+ * Process a document through the full RAG pipeline
86
+ *
87
+ * @param text - Document text to process
88
+ * @param options - Processing options
89
+ * @returns Processed chunks with optional metadata
90
+ */
91
+ export async function processDocument(text, options) {
92
+ const { strategy = "recursive", maxSize = 1000, overlap = 200, extract, provider, model, metadata = {}, } = options || {};
93
+ // Chunk the document
94
+ const chunker = ChunkerRegistry.get(strategy);
95
+ const chunks = await chunker.chunk(text, { maxSize, overlap, metadata });
96
+ // Extract metadata if requested
97
+ if (extract) {
98
+ const extractor = new LLMMetadataExtractor({ provider, modelName: model });
99
+ const results = await extractor.extract(chunks, extract);
100
+ // Merge metadata into chunks
101
+ for (let i = 0; i < chunks.length && i < results.length; i++) {
102
+ const result = results[i];
103
+ if (result.title) {
104
+ chunks[i].metadata.title = result.title;
105
+ }
106
+ if (result.summary) {
107
+ chunks[i].metadata.summary = result.summary;
108
+ }
109
+ if (result.keywords) {
110
+ chunks[i].metadata.keywords = result.keywords;
111
+ }
112
+ }
113
+ }
114
+ return chunks;
115
+ }
116
+ /**
117
+ * Get recommended chunking strategy based on content type
118
+ *
119
+ * @param contentType - MIME type or file extension
120
+ * @returns Recommended chunking strategy
121
+ */
122
+ export function getRecommendedStrategy(contentType) {
123
+ return ChunkerRegistry.getRecommendedStrategy(contentType);
124
+ }
125
+ /**
126
+ * Get available chunking strategies
127
+ *
128
+ * @returns Array of available strategy names
129
+ */
130
+ export function getAvailableStrategies() {
131
+ return ChunkerRegistry.getAvailableStrategies();
132
+ }
133
+ /**
134
+ * Get default configuration for a chunking strategy
135
+ *
136
+ * @param strategy - Chunking strategy name
137
+ * @returns Default configuration object
138
+ */
139
+ export function getDefaultChunkerConfig(strategy) {
140
+ return ChunkerRegistry.getDefaultConfig(strategy);
141
+ }
142
+ //# sourceMappingURL=index.js.map