@juspay/neurolink 9.5.3 → 9.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (356) hide show
  1. package/CHANGELOG.md +10 -0
  2. package/README.md +29 -25
  3. package/dist/agent/directTools.d.ts +5 -5
  4. package/dist/cli/commands/config.d.ts +9 -9
  5. package/dist/cli/commands/serve.d.ts +37 -0
  6. package/dist/cli/commands/serve.js +302 -229
  7. package/dist/cli/commands/setup-anthropic.d.ts +2 -2
  8. package/dist/cli/commands/setup-azure.d.ts +2 -2
  9. package/dist/cli/commands/setup-bedrock.d.ts +2 -2
  10. package/dist/cli/commands/setup-gcp.d.ts +2 -2
  11. package/dist/cli/commands/setup-google-ai.d.ts +2 -2
  12. package/dist/cli/commands/setup-huggingface.d.ts +2 -2
  13. package/dist/cli/commands/setup-mistral.d.ts +2 -2
  14. package/dist/cli/commands/setup-openai.d.ts +2 -2
  15. package/dist/cli/commands/setup.d.ts +2 -2
  16. package/dist/cli/factories/commandFactory.js +16 -2
  17. package/dist/cli/loop/optionsSchema.d.ts +2 -2
  18. package/dist/cli/loop/session.d.ts +4 -0
  19. package/dist/cli/loop/session.js +49 -4
  20. package/dist/cli/utils/interactiveSetup.d.ts +4 -4
  21. package/dist/config/conversationMemory.d.ts +2 -0
  22. package/dist/config/conversationMemory.js +5 -5
  23. package/dist/constants/contextWindows.d.ts +46 -0
  24. package/dist/constants/contextWindows.js +156 -0
  25. package/dist/context/budgetChecker.d.ts +18 -0
  26. package/dist/context/budgetChecker.js +71 -0
  27. package/dist/context/contextCompactor.d.ts +22 -0
  28. package/dist/context/contextCompactor.js +106 -0
  29. package/dist/context/effectiveHistory.d.ts +52 -0
  30. package/dist/context/effectiveHistory.js +105 -0
  31. package/dist/context/errorDetection.d.ts +14 -0
  32. package/dist/context/errorDetection.js +124 -0
  33. package/dist/context/fileSummarizationService.d.ts +54 -0
  34. package/dist/context/fileSummarizationService.js +255 -0
  35. package/dist/context/fileSummarizer.d.ts +56 -0
  36. package/dist/context/fileSummarizer.js +145 -0
  37. package/dist/context/fileTokenBudget.d.ts +53 -0
  38. package/dist/context/fileTokenBudget.js +127 -0
  39. package/dist/context/prompts/summarizationPrompt.d.ts +17 -0
  40. package/dist/context/prompts/summarizationPrompt.js +110 -0
  41. package/dist/context/stages/fileReadDeduplicator.d.ts +10 -0
  42. package/dist/context/stages/fileReadDeduplicator.js +66 -0
  43. package/dist/context/stages/slidingWindowTruncator.d.ts +11 -0
  44. package/dist/context/stages/slidingWindowTruncator.js +42 -0
  45. package/dist/context/stages/structuredSummarizer.d.ts +10 -0
  46. package/dist/context/stages/structuredSummarizer.js +49 -0
  47. package/dist/context/stages/toolOutputPruner.d.ts +10 -0
  48. package/dist/context/stages/toolOutputPruner.js +52 -0
  49. package/dist/context/summarizationEngine.d.ts +45 -0
  50. package/dist/context/summarizationEngine.js +110 -0
  51. package/dist/context/toolOutputLimits.d.ts +17 -0
  52. package/dist/context/toolOutputLimits.js +84 -0
  53. package/dist/context/toolPairRepair.d.ts +16 -0
  54. package/dist/context/toolPairRepair.js +66 -0
  55. package/dist/core/conversationMemoryManager.d.ts +5 -15
  56. package/dist/core/conversationMemoryManager.js +15 -75
  57. package/dist/core/modules/MessageBuilder.d.ts +1 -1
  58. package/dist/core/modules/MessageBuilder.js +2 -0
  59. package/dist/core/modules/TelemetryHandler.d.ts +2 -3
  60. package/dist/core/modules/TelemetryHandler.js +3 -3
  61. package/dist/core/modules/ToolsManager.d.ts +2 -2
  62. package/dist/core/redisConversationMemoryManager.d.ts +8 -14
  63. package/dist/core/redisConversationMemoryManager.js +69 -78
  64. package/dist/factories/providerFactory.d.ts +2 -2
  65. package/dist/files/fileReferenceRegistry.d.ts +276 -0
  66. package/dist/files/fileReferenceRegistry.js +1543 -0
  67. package/dist/files/fileTools.d.ts +423 -0
  68. package/dist/files/fileTools.js +449 -0
  69. package/dist/files/index.d.ts +14 -0
  70. package/dist/files/index.js +13 -0
  71. package/dist/files/streamingReader.d.ts +93 -0
  72. package/dist/files/streamingReader.js +321 -0
  73. package/dist/files/types.d.ts +23 -0
  74. package/dist/files/types.js +23 -0
  75. package/dist/image-gen/imageGenTools.d.ts +2 -2
  76. package/dist/image-gen/types.d.ts +12 -12
  77. package/dist/lib/agent/directTools.d.ts +7 -7
  78. package/dist/lib/config/conversationMemory.d.ts +2 -0
  79. package/dist/lib/config/conversationMemory.js +5 -5
  80. package/dist/lib/constants/contextWindows.d.ts +46 -0
  81. package/dist/lib/constants/contextWindows.js +157 -0
  82. package/dist/lib/context/budgetChecker.d.ts +18 -0
  83. package/dist/lib/context/budgetChecker.js +72 -0
  84. package/dist/lib/context/contextCompactor.d.ts +22 -0
  85. package/dist/lib/context/contextCompactor.js +107 -0
  86. package/dist/lib/context/effectiveHistory.d.ts +52 -0
  87. package/dist/lib/context/effectiveHistory.js +106 -0
  88. package/dist/lib/context/errorDetection.d.ts +14 -0
  89. package/dist/lib/context/errorDetection.js +125 -0
  90. package/dist/lib/context/fileSummarizationService.d.ts +54 -0
  91. package/dist/lib/context/fileSummarizationService.js +256 -0
  92. package/dist/lib/context/fileSummarizer.d.ts +56 -0
  93. package/dist/lib/context/fileSummarizer.js +146 -0
  94. package/dist/lib/context/fileTokenBudget.d.ts +53 -0
  95. package/dist/lib/context/fileTokenBudget.js +128 -0
  96. package/dist/lib/context/prompts/summarizationPrompt.d.ts +17 -0
  97. package/dist/lib/context/prompts/summarizationPrompt.js +111 -0
  98. package/dist/lib/context/stages/fileReadDeduplicator.d.ts +10 -0
  99. package/dist/lib/context/stages/fileReadDeduplicator.js +67 -0
  100. package/dist/lib/context/stages/slidingWindowTruncator.d.ts +11 -0
  101. package/dist/lib/context/stages/slidingWindowTruncator.js +43 -0
  102. package/dist/lib/context/stages/structuredSummarizer.d.ts +10 -0
  103. package/dist/lib/context/stages/structuredSummarizer.js +50 -0
  104. package/dist/lib/context/stages/toolOutputPruner.d.ts +10 -0
  105. package/dist/lib/context/stages/toolOutputPruner.js +53 -0
  106. package/dist/lib/context/summarizationEngine.d.ts +45 -0
  107. package/dist/lib/context/summarizationEngine.js +111 -0
  108. package/dist/lib/context/toolOutputLimits.d.ts +17 -0
  109. package/dist/lib/context/toolOutputLimits.js +85 -0
  110. package/dist/lib/context/toolPairRepair.d.ts +16 -0
  111. package/dist/lib/context/toolPairRepair.js +67 -0
  112. package/dist/lib/core/conversationMemoryManager.d.ts +5 -15
  113. package/dist/lib/core/conversationMemoryManager.js +15 -75
  114. package/dist/lib/core/modules/MessageBuilder.d.ts +1 -1
  115. package/dist/lib/core/modules/MessageBuilder.js +2 -0
  116. package/dist/lib/core/modules/TelemetryHandler.d.ts +2 -3
  117. package/dist/lib/core/modules/TelemetryHandler.js +3 -3
  118. package/dist/lib/core/modules/ToolsManager.d.ts +2 -2
  119. package/dist/lib/core/redisConversationMemoryManager.d.ts +8 -14
  120. package/dist/lib/core/redisConversationMemoryManager.js +69 -78
  121. package/dist/lib/factories/providerFactory.d.ts +2 -2
  122. package/dist/lib/files/fileReferenceRegistry.d.ts +276 -0
  123. package/dist/lib/files/fileReferenceRegistry.js +1544 -0
  124. package/dist/lib/files/fileTools.d.ts +423 -0
  125. package/dist/lib/files/fileTools.js +450 -0
  126. package/dist/lib/files/index.d.ts +14 -0
  127. package/dist/lib/files/index.js +14 -0
  128. package/dist/lib/files/streamingReader.d.ts +93 -0
  129. package/dist/lib/files/streamingReader.js +322 -0
  130. package/dist/lib/files/types.d.ts +23 -0
  131. package/dist/lib/files/types.js +24 -0
  132. package/dist/lib/image-gen/imageGenTools.d.ts +2 -2
  133. package/dist/lib/image-gen/types.d.ts +12 -12
  134. package/dist/lib/memory/mem0Initializer.d.ts +2 -2
  135. package/dist/lib/neurolink.d.ts +61 -2
  136. package/dist/lib/neurolink.js +619 -307
  137. package/dist/lib/processors/archive/ArchiveProcessor.d.ts +327 -0
  138. package/dist/lib/processors/archive/ArchiveProcessor.js +1309 -0
  139. package/dist/lib/processors/archive/index.d.ts +33 -0
  140. package/dist/lib/processors/archive/index.js +43 -0
  141. package/dist/lib/processors/base/types.d.ts +70 -64
  142. package/dist/lib/processors/base/types.js +6 -0
  143. package/dist/lib/processors/cli/fileProcessorCli.d.ts +8 -8
  144. package/dist/lib/processors/cli/fileProcessorCli.js +5 -5
  145. package/dist/lib/processors/config/mimeTypes.js +25 -0
  146. package/dist/lib/processors/config/sizeLimits.d.ts +52 -40
  147. package/dist/lib/processors/config/sizeLimits.js +56 -44
  148. package/dist/lib/processors/document/ExcelProcessor.d.ts +14 -0
  149. package/dist/lib/processors/document/ExcelProcessor.js +72 -1
  150. package/dist/lib/processors/document/PptxProcessor.d.ts +63 -0
  151. package/dist/lib/processors/document/PptxProcessor.js +158 -0
  152. package/dist/lib/processors/document/index.d.ts +1 -0
  153. package/dist/lib/processors/document/index.js +6 -0
  154. package/dist/lib/processors/errors/FileErrorCode.d.ts +2 -2
  155. package/dist/lib/processors/errors/errorHelpers.d.ts +2 -2
  156. package/dist/lib/processors/errors/errorSerializer.d.ts +4 -4
  157. package/dist/lib/processors/index.d.ts +8 -2
  158. package/dist/lib/processors/index.js +5 -2
  159. package/dist/lib/processors/integration/FileProcessorIntegration.d.ts +8 -8
  160. package/dist/lib/processors/integration/FileProcessorIntegration.js +7 -7
  161. package/dist/lib/processors/media/AudioProcessor.d.ts +328 -0
  162. package/dist/lib/processors/media/AudioProcessor.js +708 -0
  163. package/dist/lib/processors/media/VideoProcessor.d.ts +350 -0
  164. package/dist/lib/processors/media/VideoProcessor.js +992 -0
  165. package/dist/lib/processors/media/index.d.ts +27 -0
  166. package/dist/lib/processors/media/index.js +37 -0
  167. package/dist/lib/processors/registry/ProcessorRegistry.d.ts +19 -5
  168. package/dist/lib/processors/registry/ProcessorRegistry.js +103 -8
  169. package/dist/lib/processors/registry/index.d.ts +1 -1
  170. package/dist/lib/processors/registry/index.js +1 -1
  171. package/dist/lib/processors/registry/types.d.ts +2 -2
  172. package/dist/lib/providers/googleAiStudio.d.ts +34 -0
  173. package/dist/lib/providers/googleAiStudio.js +267 -397
  174. package/dist/lib/providers/googleVertex.d.ts +55 -1
  175. package/dist/lib/providers/googleVertex.js +452 -719
  176. package/dist/lib/providers/sagemaker/detection.d.ts +6 -6
  177. package/dist/lib/providers/sagemaker/diagnostics.d.ts +4 -4
  178. package/dist/lib/providers/sagemaker/parsers.d.ts +4 -4
  179. package/dist/lib/rag/chunkers/RecursiveChunker.js +2 -2
  180. package/dist/lib/rag/document/loaders.d.ts +6 -71
  181. package/dist/lib/rag/document/loaders.js +5 -5
  182. package/dist/lib/rag/graphRag/graphRAG.js +26 -9
  183. package/dist/lib/rag/metadata/MetadataExtractorFactory.d.ts +5 -55
  184. package/dist/lib/rag/metadata/metadataExtractor.js +6 -3
  185. package/dist/lib/rag/pipeline/RAGPipeline.d.ts +8 -126
  186. package/dist/lib/rag/pipeline/RAGPipeline.js +11 -11
  187. package/dist/lib/rag/pipeline/contextAssembly.d.ts +3 -42
  188. package/dist/lib/rag/pipeline/contextAssembly.js +6 -3
  189. package/dist/lib/rag/reranker/RerankerFactory.d.ts +5 -60
  190. package/dist/lib/rag/resilience/CircuitBreaker.d.ts +3 -33
  191. package/dist/lib/rag/resilience/RetryHandler.d.ts +2 -21
  192. package/dist/lib/rag/retrieval/hybridSearch.d.ts +3 -41
  193. package/dist/lib/rag/retrieval/vectorQueryTool.d.ts +2 -13
  194. package/dist/lib/rag/retrieval/vectorQueryTool.js +4 -3
  195. package/dist/lib/rag/types.d.ts +3 -3
  196. package/dist/lib/sdk/toolRegistration.d.ts +2 -2
  197. package/dist/lib/server/middleware/cache.d.ts +2 -2
  198. package/dist/lib/server/middleware/rateLimit.d.ts +2 -2
  199. package/dist/lib/server/routes/mcpRoutes.js +277 -249
  200. package/dist/lib/server/routes/memoryRoutes.js +287 -281
  201. package/dist/lib/server/utils/validation.d.ts +10 -10
  202. package/dist/lib/session/globalSessionState.d.ts +2 -2
  203. package/dist/lib/telemetry/telemetryService.d.ts +2 -2
  204. package/dist/lib/types/common.d.ts +39 -0
  205. package/dist/lib/types/contextTypes.d.ts +255 -0
  206. package/dist/lib/types/contextTypes.js +0 -2
  207. package/dist/lib/types/conversation.d.ts +62 -0
  208. package/dist/lib/types/conversationMemoryInterface.d.ts +27 -0
  209. package/dist/lib/types/conversationMemoryInterface.js +7 -0
  210. package/dist/lib/types/fileReferenceTypes.d.ts +222 -0
  211. package/dist/lib/types/fileReferenceTypes.js +9 -0
  212. package/dist/lib/types/fileTypes.d.ts +26 -3
  213. package/dist/lib/types/generateTypes.d.ts +22 -1
  214. package/dist/lib/types/index.d.ts +4 -5
  215. package/dist/lib/types/index.js +8 -10
  216. package/dist/lib/types/modelTypes.d.ts +2 -2
  217. package/dist/lib/types/processorTypes.d.ts +597 -0
  218. package/dist/lib/types/processorTypes.js +91 -0
  219. package/dist/lib/types/ragTypes.d.ts +481 -0
  220. package/dist/lib/types/ragTypes.js +8 -0
  221. package/dist/lib/types/sdkTypes.d.ts +17 -18
  222. package/dist/lib/types/streamTypes.d.ts +11 -1
  223. package/dist/lib/utils/async/retry.d.ts +2 -2
  224. package/dist/lib/utils/async/withTimeout.js +3 -1
  225. package/dist/lib/utils/conversationMemory.d.ts +12 -6
  226. package/dist/lib/utils/conversationMemory.js +76 -36
  227. package/dist/lib/utils/fileDetector.d.ts +62 -0
  228. package/dist/lib/utils/fileDetector.js +1014 -14
  229. package/dist/lib/utils/json/safeParse.d.ts +2 -2
  230. package/dist/lib/utils/messageBuilder.js +806 -153
  231. package/dist/lib/utils/modelChoices.d.ts +2 -2
  232. package/dist/lib/utils/multimodalOptionsBuilder.d.ts +2 -1
  233. package/dist/lib/utils/multimodalOptionsBuilder.js +1 -0
  234. package/dist/lib/utils/rateLimiter.d.ts +2 -2
  235. package/dist/lib/utils/sanitizers/filename.d.ts +4 -4
  236. package/dist/lib/utils/sanitizers/svg.d.ts +2 -2
  237. package/dist/lib/utils/thinkingConfig.d.ts +6 -6
  238. package/dist/lib/utils/tokenEstimation.d.ts +68 -0
  239. package/dist/lib/utils/tokenEstimation.js +113 -0
  240. package/dist/lib/utils/tokenUtils.d.ts +4 -4
  241. package/dist/lib/utils/ttsProcessor.d.ts +2 -2
  242. package/dist/lib/workflow/config.d.ts +150 -150
  243. package/dist/memory/mem0Initializer.d.ts +2 -2
  244. package/dist/neurolink.d.ts +61 -2
  245. package/dist/neurolink.js +619 -307
  246. package/dist/processors/archive/ArchiveProcessor.d.ts +327 -0
  247. package/dist/processors/archive/ArchiveProcessor.js +1308 -0
  248. package/dist/processors/archive/index.d.ts +33 -0
  249. package/dist/processors/archive/index.js +42 -0
  250. package/dist/processors/base/types.d.ts +70 -64
  251. package/dist/processors/base/types.js +6 -0
  252. package/dist/processors/cli/fileProcessorCli.d.ts +8 -8
  253. package/dist/processors/cli/fileProcessorCli.js +5 -5
  254. package/dist/processors/config/mimeTypes.js +25 -0
  255. package/dist/processors/config/sizeLimits.d.ts +52 -40
  256. package/dist/processors/config/sizeLimits.js +56 -44
  257. package/dist/processors/document/ExcelProcessor.d.ts +14 -0
  258. package/dist/processors/document/ExcelProcessor.js +72 -1
  259. package/dist/processors/document/PptxProcessor.d.ts +63 -0
  260. package/dist/processors/document/PptxProcessor.js +157 -0
  261. package/dist/processors/document/index.d.ts +1 -0
  262. package/dist/processors/document/index.js +6 -0
  263. package/dist/processors/errors/FileErrorCode.d.ts +2 -2
  264. package/dist/processors/errors/errorHelpers.d.ts +2 -2
  265. package/dist/processors/errors/errorSerializer.d.ts +4 -4
  266. package/dist/processors/index.d.ts +8 -2
  267. package/dist/processors/index.js +5 -2
  268. package/dist/processors/integration/FileProcessorIntegration.d.ts +8 -8
  269. package/dist/processors/integration/FileProcessorIntegration.js +7 -7
  270. package/dist/processors/media/AudioProcessor.d.ts +328 -0
  271. package/dist/processors/media/AudioProcessor.js +707 -0
  272. package/dist/processors/media/VideoProcessor.d.ts +350 -0
  273. package/dist/processors/media/VideoProcessor.js +991 -0
  274. package/dist/processors/media/ffprobe-static.d.ts +4 -0
  275. package/dist/processors/media/index.d.ts +27 -0
  276. package/dist/processors/media/index.js +36 -0
  277. package/dist/processors/registry/ProcessorRegistry.d.ts +19 -5
  278. package/dist/processors/registry/ProcessorRegistry.js +103 -8
  279. package/dist/processors/registry/index.d.ts +1 -1
  280. package/dist/processors/registry/index.js +1 -1
  281. package/dist/processors/registry/types.d.ts +2 -2
  282. package/dist/providers/googleAiStudio.d.ts +34 -0
  283. package/dist/providers/googleAiStudio.js +267 -397
  284. package/dist/providers/googleVertex.d.ts +55 -1
  285. package/dist/providers/googleVertex.js +452 -719
  286. package/dist/providers/sagemaker/detection.d.ts +6 -6
  287. package/dist/providers/sagemaker/diagnostics.d.ts +4 -4
  288. package/dist/providers/sagemaker/parsers.d.ts +4 -4
  289. package/dist/rag/chunkers/RecursiveChunker.js +2 -2
  290. package/dist/rag/document/loaders.d.ts +6 -71
  291. package/dist/rag/document/loaders.js +5 -5
  292. package/dist/rag/graphRag/graphRAG.js +26 -9
  293. package/dist/rag/metadata/MetadataExtractorFactory.d.ts +5 -55
  294. package/dist/rag/metadata/metadataExtractor.js +6 -3
  295. package/dist/rag/pipeline/RAGPipeline.d.ts +8 -126
  296. package/dist/rag/pipeline/RAGPipeline.js +11 -11
  297. package/dist/rag/pipeline/contextAssembly.d.ts +3 -42
  298. package/dist/rag/pipeline/contextAssembly.js +6 -3
  299. package/dist/rag/reranker/RerankerFactory.d.ts +5 -60
  300. package/dist/rag/resilience/CircuitBreaker.d.ts +3 -33
  301. package/dist/rag/resilience/RetryHandler.d.ts +2 -21
  302. package/dist/rag/retrieval/hybridSearch.d.ts +3 -41
  303. package/dist/rag/retrieval/vectorQueryTool.d.ts +2 -13
  304. package/dist/rag/retrieval/vectorQueryTool.js +4 -3
  305. package/dist/rag/types.d.ts +3 -3
  306. package/dist/sdk/toolRegistration.d.ts +2 -2
  307. package/dist/server/middleware/cache.d.ts +2 -2
  308. package/dist/server/middleware/rateLimit.d.ts +2 -2
  309. package/dist/server/routes/mcpRoutes.js +277 -249
  310. package/dist/server/routes/memoryRoutes.js +287 -281
  311. package/dist/server/utils/validation.d.ts +4 -4
  312. package/dist/session/globalSessionState.d.ts +2 -2
  313. package/dist/telemetry/telemetryService.d.ts +2 -2
  314. package/dist/types/common.d.ts +39 -0
  315. package/dist/types/contextTypes.d.ts +255 -0
  316. package/dist/types/contextTypes.js +0 -2
  317. package/dist/types/conversation.d.ts +62 -0
  318. package/dist/types/conversationMemoryInterface.d.ts +27 -0
  319. package/dist/types/conversationMemoryInterface.js +6 -0
  320. package/dist/types/fileReferenceTypes.d.ts +222 -0
  321. package/dist/types/fileReferenceTypes.js +8 -0
  322. package/dist/types/fileTypes.d.ts +26 -3
  323. package/dist/types/generateTypes.d.ts +22 -1
  324. package/dist/types/index.d.ts +4 -5
  325. package/dist/types/index.js +8 -10
  326. package/dist/types/processorTypes.d.ts +597 -0
  327. package/dist/types/processorTypes.js +90 -0
  328. package/dist/types/ragTypes.d.ts +481 -0
  329. package/dist/types/ragTypes.js +7 -0
  330. package/dist/types/sdkTypes.d.ts +17 -18
  331. package/dist/types/streamTypes.d.ts +11 -1
  332. package/dist/utils/async/retry.d.ts +2 -2
  333. package/dist/utils/async/withTimeout.js +3 -1
  334. package/dist/utils/conversationMemory.d.ts +12 -6
  335. package/dist/utils/conversationMemory.js +76 -36
  336. package/dist/utils/fileDetector.d.ts +62 -0
  337. package/dist/utils/fileDetector.js +1014 -14
  338. package/dist/utils/json/safeParse.d.ts +2 -2
  339. package/dist/utils/messageBuilder.js +806 -153
  340. package/dist/utils/modelChoices.d.ts +2 -2
  341. package/dist/utils/multimodalOptionsBuilder.d.ts +2 -1
  342. package/dist/utils/multimodalOptionsBuilder.js +1 -0
  343. package/dist/utils/rateLimiter.d.ts +2 -2
  344. package/dist/utils/sanitizers/filename.d.ts +4 -4
  345. package/dist/utils/sanitizers/svg.d.ts +2 -2
  346. package/dist/utils/thinkingConfig.d.ts +6 -6
  347. package/dist/utils/tokenEstimation.d.ts +68 -0
  348. package/dist/utils/tokenEstimation.js +112 -0
  349. package/dist/utils/tokenUtils.d.ts +4 -4
  350. package/dist/utils/ttsProcessor.d.ts +2 -2
  351. package/dist/workflow/config.d.ts +104 -104
  352. package/package.json +18 -6
  353. package/dist/lib/utils/conversationMemoryUtils.d.ts +0 -25
  354. package/dist/lib/utils/conversationMemoryUtils.js +0 -138
  355. package/dist/utils/conversationMemoryUtils.d.ts +0 -25
  356. package/dist/utils/conversationMemoryUtils.js +0 -137
@@ -6,11 +6,11 @@ import { AIProviderName } from "../constants/enums.js";
6
6
  /**
7
7
  * Model choice for CLI prompts (inquirer format)
8
8
  */
9
- export interface ModelChoice {
9
+ export type ModelChoice = {
10
10
  name: string;
11
11
  value: string;
12
12
  description?: string;
13
- }
13
+ };
14
14
  /**
15
15
  * Default models per provider (first choice/recommended)
16
16
  */
@@ -46,7 +46,7 @@ export declare function buildMultimodalOptions(options: StreamOptions, providerN
46
46
  text: string;
47
47
  images: (string | Buffer<ArrayBufferLike> | import("../index.js").ImageWithAltText)[] | undefined;
48
48
  content: import("../index.js").Content[] | undefined;
49
- files: (string | Buffer<ArrayBufferLike>)[] | undefined;
49
+ files: (string | Buffer<ArrayBufferLike> | import("../index.js").FileWithMetadata)[] | undefined;
50
50
  csvFiles: (string | Buffer<ArrayBufferLike>)[] | undefined;
51
51
  pdfFiles: (string | Buffer<ArrayBufferLike>)[] | undefined;
52
52
  };
@@ -64,4 +64,5 @@ export declare function buildMultimodalOptions(options: StreamOptions, providerN
64
64
  enableAnalytics: boolean | undefined;
65
65
  enableEvaluation: boolean | undefined;
66
66
  context: import("../index.js").UnknownRecord | undefined;
67
+ fileRegistry: unknown;
67
68
  };
@@ -60,5 +60,6 @@ export function buildMultimodalOptions(options, providerName, modelName) {
60
60
  enableAnalytics: options.enableAnalytics,
61
61
  enableEvaluation: options.enableEvaluation,
62
62
  context: options.context,
63
+ fileRegistry: options.fileRegistry,
63
64
  };
64
65
  }
@@ -9,7 +9,7 @@
9
9
  /**
10
10
  * Configuration options for the rate limiter
11
11
  */
12
- export interface RateLimiterConfig {
12
+ export type RateLimiterConfig = {
13
13
  /** Maximum tokens (downloads) allowed per interval */
14
14
  maxTokens: number;
15
15
  /** Refill interval in milliseconds */
@@ -20,7 +20,7 @@ export interface RateLimiterConfig {
20
20
  maxQueueSize: number;
21
21
  /** Timeout for queued requests in milliseconds */
22
22
  queueTimeoutMs: number;
23
- }
23
+ };
24
24
  /**
25
25
  * Token Bucket Rate Limiter
26
26
  *
@@ -12,7 +12,7 @@
12
12
  /**
13
13
  * Options for filename sanitization.
14
14
  */
15
- export interface SanitizeFileNameOptions {
15
+ export type SanitizeFileNameOptions = {
16
16
  /** Maximum length for the filename (default: 255) */
17
17
  maxLength?: number;
18
18
  /** Replacement character for invalid chars (default: '_') */
@@ -21,16 +21,16 @@ export interface SanitizeFileNameOptions {
21
21
  blockDangerousExtensions?: boolean;
22
22
  /** Whether to allow hidden files starting with dot (default: false) */
23
23
  allowHiddenFiles?: boolean;
24
- }
24
+ };
25
25
  /**
26
26
  * Options for display name sanitization.
27
27
  */
28
- export interface SanitizeDisplayNameOptions {
28
+ export type SanitizeDisplayNameOptions = {
29
29
  /** Maximum length for the name (default: 100) */
30
30
  maxLength?: number;
31
31
  /** Whether to allow unicode characters (default: true) */
32
32
  allowUnicode?: boolean;
33
- }
33
+ };
34
34
  /**
35
35
  * Sanitize a filename for safe filesystem storage.
36
36
  * Removes characters that are invalid on various operating systems.
@@ -19,14 +19,14 @@
19
19
  /**
20
20
  * Result of SVG sanitization including metadata about removed content
21
21
  */
22
- export interface SvgSanitizationResult {
22
+ export type SvgSanitizationResult = {
23
23
  /** Sanitized SVG content */
24
24
  content: string;
25
25
  /** Items that were removed during sanitization */
26
26
  removedItems: string[];
27
27
  /** Whether any content was modified */
28
28
  wasModified: boolean;
29
- }
29
+ };
30
30
  /**
31
31
  * Sanitize SVG content by removing dangerous elements and attributes.
32
32
  * Uses OWASP-compliant allowlist approach with regex-based parsing.
@@ -11,32 +11,32 @@ export type ThinkingLevel = "minimal" | "low" | "medium" | "high";
11
11
  /**
12
12
  * ThinkingConfig interface matching the SDK's expected structure
13
13
  */
14
- export interface ThinkingConfig {
14
+ export type ThinkingConfig = {
15
15
  enabled?: boolean;
16
16
  type?: "enabled" | "disabled";
17
17
  /** Token budget for thinking (Anthropic models: 5000-100000) */
18
18
  budgetTokens?: number;
19
19
  /** Thinking level for Gemini 3 models */
20
20
  thinkingLevel?: ThinkingLevel;
21
- }
21
+ };
22
22
  /**
23
23
  * Options for creating a thinkingConfig from CLI-style options
24
24
  */
25
- export interface CreateThinkingConfigOptions {
25
+ export type CreateThinkingConfigOptions = {
26
26
  /** Enable thinking mode */
27
27
  thinking?: boolean;
28
28
  /** Token budget for thinking (defaults to 10000) */
29
29
  thinkingBudget?: number;
30
30
  /** Thinking level for Gemini 3 models */
31
31
  thinkingLevel?: ThinkingLevel;
32
- }
32
+ };
33
33
  /**
34
34
  * Native SDK thinkingConfig structure for Gemini native SDK.
35
35
  */
36
- export interface NativeThinkingConfig {
36
+ export type NativeThinkingConfig = {
37
37
  includeThoughts: boolean;
38
38
  thinkingLevel: ThinkingLevel;
39
- }
39
+ };
40
40
  /**
41
41
  * Default token budget for thinking operations
42
42
  */
@@ -0,0 +1,68 @@
1
+ /**
2
+ * Token Estimation Utilities
3
+ *
4
+ * Provides character-based token estimation with per-provider adjustment
5
+ * multipliers. Uses the same approach as Continue (gpt-tokenizer baseline
6
+ * + provider multipliers) but without requiring a tokenizer dependency.
7
+ *
8
+ * Multiplier sources: Continue project's getAdjustedTokenCount.ts
9
+ * - Anthropic: 1.23x (Anthropic tokenizer produces ~23% more tokens)
10
+ * - Google (AI Studio / Vertex): 1.18x
11
+ * - Mistral/Codestral: 1.26x
12
+ * - OpenAI/GPT: 1.0x (baseline)
13
+ */
14
+ import type { ChatMessage } from "../types/conversation.js";
15
+ /** Characters per token for English text */
16
+ export declare const CHARS_PER_TOKEN = 4;
17
+ /** Characters per token for code */
18
+ export declare const CODE_CHARS_PER_TOKEN = 3;
19
+ /** Safety margin to avoid underestimation */
20
+ export declare const TOKEN_SAFETY_MARGIN = 1.15;
21
+ /** Message framing overhead in tokens (role + delimiters) */
22
+ export declare const TOKENS_PER_MESSAGE = 4;
23
+ /** Conversation-level overhead in tokens */
24
+ export declare const TOKENS_PER_CONVERSATION = 24;
25
+ /** Image token estimate (flat) */
26
+ export declare const IMAGE_TOKEN_ESTIMATE = 1024;
27
+ /**
28
+ * Get the token multiplier for a given provider.
29
+ */
30
+ export declare function getProviderMultiplier(provider?: string): number;
31
+ /**
32
+ * Estimate token count for a string.
33
+ *
34
+ * @param text - Input text
35
+ * @param provider - Optional provider for multiplier adjustment
36
+ * @param isCode - Whether the text is code (uses CODE_CHARS_PER_TOKEN)
37
+ * @returns Estimated token count
38
+ */
39
+ export declare function estimateTokens(text: string, provider?: string, isCode?: boolean): number;
40
+ /**
41
+ * Estimate token count for a single ChatMessage.
42
+ * Includes message framing overhead.
43
+ */
44
+ export declare function estimateMessageTokens(message: ChatMessage | {
45
+ role: string;
46
+ content: string;
47
+ }, provider?: string): number;
48
+ /**
49
+ * Estimate total token count for an array of messages.
50
+ * Includes conversation-level overhead.
51
+ */
52
+ export declare function estimateMessagesTokens(messages: Array<ChatMessage | {
53
+ role: string;
54
+ content: string;
55
+ }>, provider?: string): number;
56
+ /**
57
+ * Truncate text to fit within a token budget.
58
+ * Tries to cut at sentence or word boundaries.
59
+ *
60
+ * @param text - Input text
61
+ * @param maxTokens - Maximum tokens allowed
62
+ * @param provider - Optional provider for multiplier
63
+ * @returns Truncated text with "..." suffix if truncated
64
+ */
65
+ export declare function truncateToTokenBudget(text: string, maxTokens: number, provider?: string): {
66
+ text: string;
67
+ truncated: boolean;
68
+ };
@@ -0,0 +1,112 @@
1
+ /**
2
+ * Token Estimation Utilities
3
+ *
4
+ * Provides character-based token estimation with per-provider adjustment
5
+ * multipliers. Uses the same approach as Continue (gpt-tokenizer baseline
6
+ * + provider multipliers) but without requiring a tokenizer dependency.
7
+ *
8
+ * Multiplier sources: Continue project's getAdjustedTokenCount.ts
9
+ * - Anthropic: 1.23x (Anthropic tokenizer produces ~23% more tokens)
10
+ * - Google (AI Studio / Vertex): 1.18x
11
+ * - Mistral/Codestral: 1.26x
12
+ * - OpenAI/GPT: 1.0x (baseline)
13
+ */
14
+ /** Characters per token for English text */
15
+ export const CHARS_PER_TOKEN = 4;
16
+ /** Characters per token for code */
17
+ export const CODE_CHARS_PER_TOKEN = 3;
18
+ /** Safety margin to avoid underestimation */
19
+ export const TOKEN_SAFETY_MARGIN = 1.15;
20
+ /** Message framing overhead in tokens (role + delimiters) */
21
+ export const TOKENS_PER_MESSAGE = 4;
22
+ /** Conversation-level overhead in tokens */
23
+ export const TOKENS_PER_CONVERSATION = 24;
24
+ /** Image token estimate (flat) */
25
+ export const IMAGE_TOKEN_ESTIMATE = 1_024;
26
+ /**
27
+ * Per-provider token multipliers.
28
+ * Applied on top of the base GPT-style character estimate.
29
+ */
30
+ const PROVIDER_MULTIPLIERS = {
31
+ anthropic: 1.23,
32
+ "google-ai": 1.18,
33
+ vertex: 1.18,
34
+ mistral: 1.26,
35
+ openai: 1.0,
36
+ azure: 1.0,
37
+ bedrock: 1.23, // Bedrock is mostly Anthropic models
38
+ ollama: 1.0,
39
+ litellm: 1.0,
40
+ huggingface: 1.0,
41
+ sagemaker: 1.0,
42
+ };
43
+ /**
44
+ * Get the token multiplier for a given provider.
45
+ */
46
+ export function getProviderMultiplier(provider) {
47
+ if (!provider) {
48
+ return 1.0;
49
+ }
50
+ return PROVIDER_MULTIPLIERS[provider] ?? 1.0;
51
+ }
52
+ /**
53
+ * Estimate token count for a string.
54
+ *
55
+ * @param text - Input text
56
+ * @param provider - Optional provider for multiplier adjustment
57
+ * @param isCode - Whether the text is code (uses CODE_CHARS_PER_TOKEN)
58
+ * @returns Estimated token count
59
+ */
60
+ export function estimateTokens(text, provider, isCode) {
61
+ if (!text || text.length === 0) {
62
+ return 0;
63
+ }
64
+ const charsPerToken = isCode ? CODE_CHARS_PER_TOKEN : CHARS_PER_TOKEN;
65
+ const baseTokens = Math.ceil(text.length / charsPerToken);
66
+ const multiplier = getProviderMultiplier(provider);
67
+ return Math.ceil(baseTokens * multiplier * TOKEN_SAFETY_MARGIN);
68
+ }
69
+ /**
70
+ * Estimate token count for a single ChatMessage.
71
+ * Includes message framing overhead.
72
+ */
73
+ export function estimateMessageTokens(message, provider) {
74
+ const contentTokens = estimateTokens(message.content || "", provider);
75
+ return contentTokens + TOKENS_PER_MESSAGE;
76
+ }
77
+ /**
78
+ * Estimate total token count for an array of messages.
79
+ * Includes conversation-level overhead.
80
+ */
81
+ export function estimateMessagesTokens(messages, provider) {
82
+ if (!messages || messages.length === 0) {
83
+ return 0;
84
+ }
85
+ const messageTokens = messages.reduce((sum, msg) => sum + estimateMessageTokens(msg, provider), 0);
86
+ return messageTokens + TOKENS_PER_CONVERSATION;
87
+ }
88
+ /**
89
+ * Truncate text to fit within a token budget.
90
+ * Tries to cut at sentence or word boundaries.
91
+ *
92
+ * @param text - Input text
93
+ * @param maxTokens - Maximum tokens allowed
94
+ * @param provider - Optional provider for multiplier
95
+ * @returns Truncated text with "..." suffix if truncated
96
+ */
97
+ export function truncateToTokenBudget(text, maxTokens, provider) {
98
+ if (estimateTokens(text, provider) <= maxTokens) {
99
+ return { text, truncated: false };
100
+ }
101
+ const multiplier = getProviderMultiplier(provider);
102
+ const safetyFactor = multiplier * TOKEN_SAFETY_MARGIN;
103
+ const maxChars = Math.floor((maxTokens / safetyFactor) * CHARS_PER_TOKEN);
104
+ if (maxChars <= 0) {
105
+ return { text: "", truncated: true };
106
+ }
107
+ // Try to cut at sentence boundary
108
+ const candidate = text.slice(0, maxChars);
109
+ const lastSentence = Math.max(candidate.lastIndexOf(". "), candidate.lastIndexOf("! "), candidate.lastIndexOf("? "), candidate.lastIndexOf("\n"));
110
+ const cutPoint = lastSentence > maxChars * 0.5 ? lastSentence + 1 : maxChars;
111
+ return { text: text.slice(0, cutPoint) + "...", truncated: true };
112
+ }
@@ -12,7 +12,7 @@ import type { TokenUsage } from "../types/analytics.js";
12
12
  * Raw usage object that may come from various AI providers
13
13
  * Supports multiple naming conventions and nested structures
14
14
  */
15
- export interface RawUsageObject {
15
+ export type RawUsageObject = {
16
16
  input?: number;
17
17
  output?: number;
18
18
  total?: number;
@@ -30,11 +30,11 @@ export interface RawUsageObject {
30
30
  reasoning_tokens?: number;
31
31
  thinkingTokens?: number;
32
32
  usage?: RawUsageObject;
33
- }
33
+ };
34
34
  /**
35
35
  * Options for token extraction behavior
36
36
  */
37
- export interface TokenExtractionOptions {
37
+ export type TokenExtractionOptions = {
38
38
  /**
39
39
  * Whether to calculate cache savings percentage
40
40
  * @default true
@@ -46,7 +46,7 @@ export interface TokenExtractionOptions {
46
46
  * - "undefined": Return undefined for missing optional fields (default)
47
47
  */
48
48
  missingOptionalBehavior?: "zero" | "undefined";
49
- }
49
+ };
50
50
  /**
51
51
  * Extract input token count from various provider formats
52
52
  * Priority: input > inputTokens > promptTokens
@@ -67,7 +67,7 @@ export declare class TTSError extends NeuroLinkError {
67
67
  * }
68
68
  * ```
69
69
  */
70
- export interface TTSHandler {
70
+ export type TTSHandler = {
71
71
  /**
72
72
  * Generate audio from text using provider-specific TTS API
73
73
  *
@@ -101,7 +101,7 @@ export interface TTSHandler {
101
101
  * @default 3000 if not specified
102
102
  */
103
103
  maxTextLength?: number;
104
- }
104
+ };
105
105
  /**
106
106
  * TTS processor class for orchestrating text-to-speech operations
107
107
  *