@juspay/neurolink 9.32.0 → 9.33.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (475) hide show
  1. package/CHANGELOG.md +12 -0
  2. package/dist/auth/anthropicOAuth.js +1 -1
  3. package/dist/cli/commands/proxy.js +18 -5
  4. package/dist/client/aiSdkAdapter.js +1 -1
  5. package/dist/client/index.js +137 -501
  6. package/dist/core/factory.js +0 -1
  7. package/dist/core/redisConversationMemoryManager.js +1 -1
  8. package/dist/features/ppt/slideGenerator.js +0 -1
  9. package/dist/features/ppt/utils.js +0 -1
  10. package/dist/lib/neurolink.d.ts +10 -0
  11. package/dist/lib/neurolink.js +41 -7
  12. package/dist/lib/server/routes/claudeProxyRoutes.js +45 -9
  13. package/dist/lib/types/generateTypes.d.ts +16 -0
  14. package/dist/lib/types/streamTypes.d.ts +15 -0
  15. package/dist/mcp/elicitationProtocol.js +1 -1
  16. package/dist/mcp/servers/agent/directToolsServer.js +0 -1
  17. package/dist/neurolink.d.ts +10 -0
  18. package/dist/neurolink.js +41 -7
  19. package/dist/providers/azureOpenai.js +1 -1
  20. package/dist/providers/huggingFace.js +0 -1
  21. package/dist/providers/openaiCompatible.js +0 -1
  22. package/dist/sdk/toolRegistration.js +0 -1
  23. package/dist/server/openapi/generator.js +1 -1
  24. package/dist/server/routes/claudeProxyRoutes.js +45 -9
  25. package/dist/types/configTypes.js +0 -5
  26. package/dist/types/generateTypes.d.ts +16 -0
  27. package/dist/types/modelTypes.js +0 -1
  28. package/dist/types/streamTypes.d.ts +15 -0
  29. package/dist/types/tools.js +0 -1
  30. package/dist/types/typeAliases.js +0 -1
  31. package/dist/types/utilities.js +1 -1
  32. package/dist/types/workflowTypes.js +0 -1
  33. package/dist/utils/providerRetry.js +0 -1
  34. package/dist/utils/providerUtils.js +0 -1
  35. package/package.json +2 -2
  36. package/dist/client/adapters/providerImageAdapter.js +0 -588
  37. package/dist/client/adapters/tts/googleTTSHandler.js +0 -344
  38. package/dist/client/adapters/video/directorPipeline.js +0 -516
  39. package/dist/client/adapters/video/ffmpegAdapter.js +0 -206
  40. package/dist/client/adapters/video/frameExtractor.js +0 -143
  41. package/dist/client/adapters/video/vertexVideoHandler.js +0 -763
  42. package/dist/client/adapters/video/videoAnalyzer.js +0 -238
  43. package/dist/client/adapters/video/videoMerger.js +0 -171
  44. package/dist/client/agent/directTools.js +0 -840
  45. package/dist/client/auth/AuthProviderFactory.js +0 -111
  46. package/dist/client/auth/AuthProviderRegistry.js +0 -190
  47. package/dist/client/auth/RequestContext.js +0 -78
  48. package/dist/client/auth/accountPool.js +0 -178
  49. package/dist/client/auth/anthropicOAuth.js +0 -974
  50. package/dist/client/auth/authContext.js +0 -314
  51. package/dist/client/auth/errors.js +0 -39
  52. package/dist/client/auth/index.js +0 -61
  53. package/dist/client/auth/middleware/AuthMiddleware.js +0 -519
  54. package/dist/client/auth/middleware/rateLimitByUser.js +0 -554
  55. package/dist/client/auth/providers/BaseAuthProvider.js +0 -723
  56. package/dist/client/auth/providers/CognitoProvider.js +0 -304
  57. package/dist/client/auth/providers/KeycloakProvider.js +0 -393
  58. package/dist/client/auth/providers/auth0.js +0 -274
  59. package/dist/client/auth/providers/betterAuth.js +0 -182
  60. package/dist/client/auth/providers/clerk.js +0 -317
  61. package/dist/client/auth/providers/custom.js +0 -112
  62. package/dist/client/auth/providers/firebase.js +0 -226
  63. package/dist/client/auth/providers/jwt.js +0 -212
  64. package/dist/client/auth/providers/oauth2.js +0 -303
  65. package/dist/client/auth/providers/supabase.js +0 -259
  66. package/dist/client/auth/providers/workos.js +0 -284
  67. package/dist/client/auth/serverBridge.js +0 -25
  68. package/dist/client/auth/sessionManager.js +0 -437
  69. package/dist/client/auth/tokenStore.js +0 -799
  70. package/dist/client/client/aiSdkAdapter.js +0 -487
  71. package/dist/client/client/auth.js +0 -473
  72. package/dist/client/client/errors.js +0 -552
  73. package/dist/client/client/httpClient.js +0 -837
  74. package/dist/client/client/index.js +0 -172
  75. package/dist/client/client/interceptors.js +0 -601
  76. package/dist/client/client/sseClient.js +0 -545
  77. package/dist/client/client/streamingClient.js +0 -917
  78. package/dist/client/client/wsClient.js +0 -369
  79. package/dist/client/config/configManager.js +0 -303
  80. package/dist/client/config/conversationMemory.js +0 -86
  81. package/dist/client/config/taskClassificationConfig.js +0 -148
  82. package/dist/client/constants/contextWindows.js +0 -295
  83. package/dist/client/constants/enums.js +0 -853
  84. package/dist/client/constants/index.js +0 -207
  85. package/dist/client/constants/performance.js +0 -389
  86. package/dist/client/constants/retry.js +0 -266
  87. package/dist/client/constants/timeouts.js +0 -182
  88. package/dist/client/constants/tokens.js +0 -380
  89. package/dist/client/constants/videoErrors.js +0 -46
  90. package/dist/client/context/budgetChecker.js +0 -98
  91. package/dist/client/context/contextCompactor.js +0 -205
  92. package/dist/client/context/emergencyTruncation.js +0 -88
  93. package/dist/client/context/errorDetection.js +0 -171
  94. package/dist/client/context/errors.js +0 -21
  95. package/dist/client/context/fileTokenBudget.js +0 -127
  96. package/dist/client/context/prompts/summarizationPrompt.js +0 -117
  97. package/dist/client/context/stages/fileReadDeduplicator.js +0 -66
  98. package/dist/client/context/stages/slidingWindowTruncator.js +0 -190
  99. package/dist/client/context/stages/structuredSummarizer.js +0 -99
  100. package/dist/client/context/stages/toolOutputPruner.js +0 -52
  101. package/dist/client/context/summarizationEngine.js +0 -136
  102. package/dist/client/context/toolOutputLimits.js +0 -78
  103. package/dist/client/context/toolPairRepair.js +0 -66
  104. package/dist/client/core/analytics.js +0 -88
  105. package/dist/client/core/baseProvider.js +0 -1385
  106. package/dist/client/core/constants.js +0 -140
  107. package/dist/client/core/conversationMemoryFactory.js +0 -141
  108. package/dist/client/core/conversationMemoryInitializer.js +0 -128
  109. package/dist/client/core/conversationMemoryManager.js +0 -344
  110. package/dist/client/core/dynamicModels.js +0 -358
  111. package/dist/client/core/evaluation.js +0 -309
  112. package/dist/client/core/evaluationProviders.js +0 -248
  113. package/dist/client/core/factory.js +0 -412
  114. package/dist/client/core/infrastructure/baseError.js +0 -22
  115. package/dist/client/core/infrastructure/baseFactory.js +0 -54
  116. package/dist/client/core/infrastructure/baseRegistry.js +0 -53
  117. package/dist/client/core/infrastructure/index.js +0 -5
  118. package/dist/client/core/infrastructure/retry.js +0 -20
  119. package/dist/client/core/infrastructure/typedEventEmitter.js +0 -23
  120. package/dist/client/core/modelConfiguration.js +0 -851
  121. package/dist/client/core/modules/GenerationHandler.js +0 -588
  122. package/dist/client/core/modules/MessageBuilder.js +0 -273
  123. package/dist/client/core/modules/StreamHandler.js +0 -185
  124. package/dist/client/core/modules/TelemetryHandler.js +0 -203
  125. package/dist/client/core/modules/ToolsManager.js +0 -499
  126. package/dist/client/core/modules/Utilities.js +0 -331
  127. package/dist/client/core/redisConversationMemoryManager.js +0 -1435
  128. package/dist/client/core/streamAnalytics.js +0 -131
  129. package/dist/client/evaluation/contextBuilder.js +0 -134
  130. package/dist/client/evaluation/index.js +0 -61
  131. package/dist/client/evaluation/prompts.js +0 -73
  132. package/dist/client/evaluation/ragasEvaluator.js +0 -110
  133. package/dist/client/evaluation/retryManager.js +0 -78
  134. package/dist/client/evaluation/scoring.js +0 -61
  135. package/dist/client/factories/providerFactory.js +0 -166
  136. package/dist/client/factories/providerRegistry.js +0 -166
  137. package/dist/client/features/ppt/constants.js +0 -896
  138. package/dist/client/features/ppt/contentPlanner.js +0 -529
  139. package/dist/client/features/ppt/presentationOrchestrator.js +0 -236
  140. package/dist/client/features/ppt/slideGenerator.js +0 -532
  141. package/dist/client/features/ppt/slideRenderers.js +0 -2383
  142. package/dist/client/features/ppt/slideTypeInference.js +0 -405
  143. package/dist/client/features/ppt/types.js +0 -13
  144. package/dist/client/features/ppt/utils.js +0 -443
  145. package/dist/client/files/fileReferenceRegistry.js +0 -1543
  146. package/dist/client/files/fileTools.js +0 -450
  147. package/dist/client/files/streamingReader.js +0 -321
  148. package/dist/client/files/types.js +0 -23
  149. package/dist/client/hitl/hitlErrors.js +0 -54
  150. package/dist/client/hitl/hitlManager.js +0 -460
  151. package/dist/client/mcp/agentExposure.js +0 -356
  152. package/dist/client/mcp/auth/index.js +0 -11
  153. package/dist/client/mcp/auth/oauthClientProvider.js +0 -325
  154. package/dist/client/mcp/auth/tokenStorage.js +0 -134
  155. package/dist/client/mcp/batching/index.js +0 -10
  156. package/dist/client/mcp/batching/requestBatcher.js +0 -441
  157. package/dist/client/mcp/caching/index.js +0 -10
  158. package/dist/client/mcp/caching/toolCache.js +0 -433
  159. package/dist/client/mcp/elicitation/elicitationManager.js +0 -376
  160. package/dist/client/mcp/elicitation/index.js +0 -11
  161. package/dist/client/mcp/elicitation/types.js +0 -10
  162. package/dist/client/mcp/elicitationProtocol.js +0 -375
  163. package/dist/client/mcp/enhancedToolDiscovery.js +0 -481
  164. package/dist/client/mcp/externalServerManager.js +0 -1478
  165. package/dist/client/mcp/factory.js +0 -161
  166. package/dist/client/mcp/flexibleToolValidator.js +0 -161
  167. package/dist/client/mcp/httpRateLimiter.js +0 -391
  168. package/dist/client/mcp/httpRetryHandler.js +0 -178
  169. package/dist/client/mcp/index.js +0 -74
  170. package/dist/client/mcp/mcpCircuitBreaker.js +0 -427
  171. package/dist/client/mcp/mcpClientFactory.js +0 -708
  172. package/dist/client/mcp/mcpRegistryClient.js +0 -488
  173. package/dist/client/mcp/mcpServerBase.js +0 -373
  174. package/dist/client/mcp/multiServerManager.js +0 -579
  175. package/dist/client/mcp/registry.js +0 -158
  176. package/dist/client/mcp/routing/index.js +0 -10
  177. package/dist/client/mcp/routing/toolRouter.js +0 -416
  178. package/dist/client/mcp/serverCapabilities.js +0 -502
  179. package/dist/client/mcp/servers/agent/directToolsServer.js +0 -150
  180. package/dist/client/mcp/toolAnnotations.js +0 -239
  181. package/dist/client/mcp/toolConverter.js +0 -258
  182. package/dist/client/mcp/toolDiscoveryService.js +0 -798
  183. package/dist/client/mcp/toolIntegration.js +0 -334
  184. package/dist/client/mcp/toolRegistry.js +0 -729
  185. package/dist/client/memory/hippocampusInitializer.js +0 -19
  186. package/dist/client/memory/memoryRetrievalTools.js +0 -166
  187. package/dist/client/middleware/builtin/analytics.js +0 -132
  188. package/dist/client/middleware/builtin/autoEvaluation.js +0 -203
  189. package/dist/client/middleware/builtin/guardrails.js +0 -109
  190. package/dist/client/middleware/builtin/lifecycle.js +0 -168
  191. package/dist/client/middleware/factory.js +0 -327
  192. package/dist/client/middleware/registry.js +0 -295
  193. package/dist/client/middleware/utils/guardrailsUtils.js +0 -396
  194. package/dist/client/models/anthropicModels.js +0 -527
  195. package/dist/client/neurolink.js +0 -8233
  196. package/dist/client/observability/exporterRegistry.js +0 -413
  197. package/dist/client/observability/exporters/arizeExporter.js +0 -138
  198. package/dist/client/observability/exporters/baseExporter.js +0 -190
  199. package/dist/client/observability/exporters/braintrustExporter.js +0 -154
  200. package/dist/client/observability/exporters/datadogExporter.js +0 -196
  201. package/dist/client/observability/exporters/laminarExporter.js +0 -302
  202. package/dist/client/observability/exporters/langfuseExporter.js +0 -209
  203. package/dist/client/observability/exporters/langsmithExporter.js +0 -143
  204. package/dist/client/observability/exporters/otelExporter.js +0 -164
  205. package/dist/client/observability/exporters/posthogExporter.js +0 -287
  206. package/dist/client/observability/exporters/sentryExporter.js +0 -165
  207. package/dist/client/observability/index.js +0 -31
  208. package/dist/client/observability/metricsAggregator.js +0 -556
  209. package/dist/client/observability/otelBridge.js +0 -131
  210. package/dist/client/observability/retryPolicy.js +0 -383
  211. package/dist/client/observability/sampling/samplers.js +0 -216
  212. package/dist/client/observability/spanProcessor.js +0 -303
  213. package/dist/client/observability/tokenTracker.js +0 -413
  214. package/dist/client/observability/types/exporterTypes.js +0 -5
  215. package/dist/client/observability/types/index.js +0 -4
  216. package/dist/client/observability/types/spanTypes.js +0 -92
  217. package/dist/client/observability/utils/safeMetadata.js +0 -25
  218. package/dist/client/observability/utils/spanSerializer.js +0 -292
  219. package/dist/client/processors/archive/ArchiveProcessor.js +0 -1308
  220. package/dist/client/processors/base/BaseFileProcessor.js +0 -614
  221. package/dist/client/processors/base/types.js +0 -82
  222. package/dist/client/processors/config/fileTypes.js +0 -520
  223. package/dist/client/processors/config/index.js +0 -92
  224. package/dist/client/processors/config/languageMap.js +0 -410
  225. package/dist/client/processors/config/mimeTypes.js +0 -363
  226. package/dist/client/processors/config/sizeLimits.js +0 -258
  227. package/dist/client/processors/document/ExcelProcessor.js +0 -590
  228. package/dist/client/processors/document/OpenDocumentProcessor.js +0 -212
  229. package/dist/client/processors/document/PptxProcessor.js +0 -157
  230. package/dist/client/processors/document/RtfProcessor.js +0 -361
  231. package/dist/client/processors/document/WordProcessor.js +0 -353
  232. package/dist/client/processors/errors/FileErrorCode.js +0 -255
  233. package/dist/client/processors/errors/errorHelpers.js +0 -386
  234. package/dist/client/processors/errors/errorSerializer.js +0 -507
  235. package/dist/client/processors/errors/index.js +0 -49
  236. package/dist/client/processors/markup/SvgProcessor.js +0 -240
  237. package/dist/client/processors/media/AudioProcessor.js +0 -707
  238. package/dist/client/processors/media/VideoProcessor.js +0 -1045
  239. package/dist/client/providers/amazonBedrock.js +0 -1512
  240. package/dist/client/providers/amazonSagemaker.js +0 -162
  241. package/dist/client/providers/anthropic.js +0 -831
  242. package/dist/client/providers/azureOpenai.js +0 -143
  243. package/dist/client/providers/googleAiStudio.js +0 -1200
  244. package/dist/client/providers/googleNativeGemini3.js +0 -543
  245. package/dist/client/providers/googleVertex.js +0 -2936
  246. package/dist/client/providers/huggingFace.js +0 -315
  247. package/dist/client/providers/litellm.js +0 -488
  248. package/dist/client/providers/mistral.js +0 -157
  249. package/dist/client/providers/ollama.js +0 -1579
  250. package/dist/client/providers/openAI.js +0 -627
  251. package/dist/client/providers/openRouter.js +0 -543
  252. package/dist/client/providers/openaiCompatible.js +0 -290
  253. package/dist/client/providers/providerTypeUtils.js +0 -46
  254. package/dist/client/providers/sagemaker/adaptive-semaphore.js +0 -215
  255. package/dist/client/providers/sagemaker/client.js +0 -472
  256. package/dist/client/providers/sagemaker/config.js +0 -317
  257. package/dist/client/providers/sagemaker/detection.js +0 -606
  258. package/dist/client/providers/sagemaker/error-constants.js +0 -227
  259. package/dist/client/providers/sagemaker/errors.js +0 -299
  260. package/dist/client/providers/sagemaker/language-model.js +0 -775
  261. package/dist/client/providers/sagemaker/parsers.js +0 -634
  262. package/dist/client/providers/sagemaker/streaming.js +0 -331
  263. package/dist/client/providers/sagemaker/structured-parser.js +0 -625
  264. package/dist/client/proxy/accountQuota.js +0 -162
  265. package/dist/client/proxy/claudeFormat.js +0 -595
  266. package/dist/client/proxy/modelRouter.js +0 -29
  267. package/dist/client/proxy/oauthFetch.js +0 -367
  268. package/dist/client/proxy/proxyFetch.js +0 -586
  269. package/dist/client/proxy/requestLogger.js +0 -207
  270. package/dist/client/proxy/tokenRefresh.js +0 -124
  271. package/dist/client/proxy/usageStats.js +0 -74
  272. package/dist/client/proxy/utils/noProxyUtils.js +0 -149
  273. package/dist/client/rag/ChunkerFactory.js +0 -320
  274. package/dist/client/rag/ChunkerRegistry.js +0 -421
  275. package/dist/client/rag/chunkers/BaseChunker.js +0 -143
  276. package/dist/client/rag/chunkers/CharacterChunker.js +0 -28
  277. package/dist/client/rag/chunkers/HTMLChunker.js +0 -38
  278. package/dist/client/rag/chunkers/JSONChunker.js +0 -68
  279. package/dist/client/rag/chunkers/LaTeXChunker.js +0 -63
  280. package/dist/client/rag/chunkers/MarkdownChunker.js +0 -306
  281. package/dist/client/rag/chunkers/RecursiveChunker.js +0 -139
  282. package/dist/client/rag/chunkers/SemanticMarkdownChunker.js +0 -138
  283. package/dist/client/rag/chunkers/SentenceChunker.js +0 -66
  284. package/dist/client/rag/chunkers/TokenChunker.js +0 -61
  285. package/dist/client/rag/chunkers/index.js +0 -15
  286. package/dist/client/rag/chunking/characterChunker.js +0 -142
  287. package/dist/client/rag/chunking/chunkerRegistry.js +0 -194
  288. package/dist/client/rag/chunking/htmlChunker.js +0 -247
  289. package/dist/client/rag/chunking/index.js +0 -17
  290. package/dist/client/rag/chunking/jsonChunker.js +0 -281
  291. package/dist/client/rag/chunking/latexChunker.js +0 -251
  292. package/dist/client/rag/chunking/markdownChunker.js +0 -373
  293. package/dist/client/rag/chunking/recursiveChunker.js +0 -148
  294. package/dist/client/rag/chunking/semanticChunker.js +0 -306
  295. package/dist/client/rag/chunking/sentenceChunker.js +0 -230
  296. package/dist/client/rag/chunking/tokenChunker.js +0 -183
  297. package/dist/client/rag/document/MDocument.js +0 -392
  298. package/dist/client/rag/document/index.js +0 -5
  299. package/dist/client/rag/document/loaders.js +0 -500
  300. package/dist/client/rag/errors/RAGError.js +0 -274
  301. package/dist/client/rag/errors/index.js +0 -6
  302. package/dist/client/rag/graphRag/graphRAG.js +0 -401
  303. package/dist/client/rag/graphRag/index.js +0 -4
  304. package/dist/client/rag/index.js +0 -141
  305. package/dist/client/rag/metadata/MetadataExtractorFactory.js +0 -418
  306. package/dist/client/rag/metadata/MetadataExtractorRegistry.js +0 -362
  307. package/dist/client/rag/metadata/index.js +0 -9
  308. package/dist/client/rag/metadata/metadataExtractor.js +0 -280
  309. package/dist/client/rag/pipeline/RAGPipeline.js +0 -436
  310. package/dist/client/rag/pipeline/contextAssembly.js +0 -341
  311. package/dist/client/rag/pipeline/index.js +0 -5
  312. package/dist/client/rag/ragIntegration.js +0 -321
  313. package/dist/client/rag/reranker/RerankerFactory.js +0 -430
  314. package/dist/client/rag/reranker/RerankerRegistry.js +0 -402
  315. package/dist/client/rag/reranker/index.js +0 -9
  316. package/dist/client/rag/reranker/reranker.js +0 -277
  317. package/dist/client/rag/resilience/CircuitBreaker.js +0 -431
  318. package/dist/client/rag/resilience/RetryHandler.js +0 -304
  319. package/dist/client/rag/resilience/index.js +0 -7
  320. package/dist/client/rag/retrieval/hybridSearch.js +0 -335
  321. package/dist/client/rag/retrieval/index.js +0 -5
  322. package/dist/client/rag/retrieval/vectorQueryTool.js +0 -307
  323. package/dist/client/rag/types.js +0 -8
  324. package/dist/client/sdk/toolRegistration.js +0 -377
  325. package/dist/client/server/abstract/baseServerAdapter.js +0 -575
  326. package/dist/client/server/adapters/expressAdapter.js +0 -486
  327. package/dist/client/server/adapters/fastifyAdapter.js +0 -472
  328. package/dist/client/server/adapters/honoAdapter.js +0 -632
  329. package/dist/client/server/adapters/koaAdapter.js +0 -510
  330. package/dist/client/server/errors.js +0 -486
  331. package/dist/client/server/factory/serverAdapterFactory.js +0 -160
  332. package/dist/client/server/index.js +0 -108
  333. package/dist/client/server/middleware/abortSignal.js +0 -111
  334. package/dist/client/server/middleware/auth.js +0 -388
  335. package/dist/client/server/middleware/cache.js +0 -359
  336. package/dist/client/server/middleware/common.js +0 -281
  337. package/dist/client/server/middleware/deprecation.js +0 -190
  338. package/dist/client/server/middleware/mcpBodyAttachment.js +0 -63
  339. package/dist/client/server/middleware/rateLimit.js +0 -227
  340. package/dist/client/server/middleware/validation.js +0 -388
  341. package/dist/client/server/openapi/generator.js +0 -398
  342. package/dist/client/server/openapi/index.js +0 -36
  343. package/dist/client/server/openapi/schemas.js +0 -695
  344. package/dist/client/server/openapi/templates.js +0 -374
  345. package/dist/client/server/routes/agentRoutes.js +0 -189
  346. package/dist/client/server/routes/claudeProxyRoutes.js +0 -1600
  347. package/dist/client/server/routes/healthRoutes.js +0 -187
  348. package/dist/client/server/routes/index.js +0 -57
  349. package/dist/client/server/routes/mcpRoutes.js +0 -342
  350. package/dist/client/server/routes/memoryRoutes.js +0 -350
  351. package/dist/client/server/routes/openApiRoutes.js +0 -126
  352. package/dist/client/server/routes/toolRoutes.js +0 -199
  353. package/dist/client/server/streaming/dataStream.js +0 -486
  354. package/dist/client/server/streaming/index.js +0 -11
  355. package/dist/client/server/types.js +0 -67
  356. package/dist/client/server/utils/redaction.js +0 -334
  357. package/dist/client/server/utils/validation.js +0 -243
  358. package/dist/client/server/websocket/WebSocketHandler.js +0 -383
  359. package/dist/client/server/websocket/index.js +0 -4
  360. package/dist/client/services/server/ai/observability/instrumentation.js +0 -808
  361. package/dist/client/telemetry/attributes.js +0 -100
  362. package/dist/client/telemetry/index.js +0 -26
  363. package/dist/client/telemetry/telemetryService.js +0 -308
  364. package/dist/client/telemetry/tracers.js +0 -17
  365. package/dist/client/telemetry/withSpan.js +0 -34
  366. package/dist/client/types/actionTypes.js +0 -6
  367. package/dist/client/types/analytics.js +0 -5
  368. package/dist/client/types/authTypes.js +0 -9
  369. package/dist/client/types/circuitBreakerErrors.js +0 -34
  370. package/dist/client/types/cli.js +0 -21
  371. package/dist/client/types/clientTypes.js +0 -10
  372. package/dist/client/types/common.js +0 -51
  373. package/dist/client/types/configTypes.js +0 -49
  374. package/dist/client/types/content.js +0 -19
  375. package/dist/client/types/contextTypes.js +0 -400
  376. package/dist/client/types/conversation.js +0 -47
  377. package/dist/client/types/conversationMemoryInterface.js +0 -6
  378. package/dist/client/types/domainTypes.js +0 -5
  379. package/dist/client/types/errors.js +0 -167
  380. package/dist/client/types/evaluation.js +0 -5
  381. package/dist/client/types/evaluationProviders.js +0 -5
  382. package/dist/client/types/evaluationTypes.js +0 -1
  383. package/dist/client/types/externalMcp.js +0 -6
  384. package/dist/client/types/fileReferenceTypes.js +0 -8
  385. package/dist/client/types/fileTypes.js +0 -4
  386. package/dist/client/types/generateTypes.js +0 -1
  387. package/dist/client/types/guardrails.js +0 -1
  388. package/dist/client/types/hitlTypes.js +0 -8
  389. package/dist/client/types/index.js +0 -57
  390. package/dist/client/types/mcpTypes.js +0 -5
  391. package/dist/client/types/middlewareTypes.js +0 -1
  392. package/dist/client/types/modelTypes.js +0 -30
  393. package/dist/client/types/multimodal.js +0 -135
  394. package/dist/client/types/observability.js +0 -6
  395. package/dist/client/types/pptTypes.js +0 -82
  396. package/dist/client/types/providers.js +0 -111
  397. package/dist/client/types/proxyTypes.js +0 -16
  398. package/dist/client/types/ragTypes.js +0 -7
  399. package/dist/client/types/sdkTypes.js +0 -8
  400. package/dist/client/types/serviceTypes.js +0 -5
  401. package/dist/client/types/streamTypes.js +0 -1
  402. package/dist/client/types/subscriptionTypes.js +0 -9
  403. package/dist/client/types/taskClassificationTypes.js +0 -5
  404. package/dist/client/types/tools.js +0 -24
  405. package/dist/client/types/ttsTypes.js +0 -57
  406. package/dist/client/types/typeAliases.js +0 -48
  407. package/dist/client/types/utilities.js +0 -4
  408. package/dist/client/types/workflowTypes.js +0 -30
  409. package/dist/client/utils/async/withTimeout.js +0 -98
  410. package/dist/client/utils/asyncMutex.js +0 -60
  411. package/dist/client/utils/conversationMemory.js +0 -431
  412. package/dist/client/utils/csvProcessor.js +0 -846
  413. package/dist/client/utils/errorHandling.js +0 -936
  414. package/dist/client/utils/evaluationUtils.js +0 -131
  415. package/dist/client/utils/factoryProcessing.js +0 -589
  416. package/dist/client/utils/fileDetector.js +0 -2161
  417. package/dist/client/utils/imageCache.js +0 -376
  418. package/dist/client/utils/imageProcessor.js +0 -704
  419. package/dist/client/utils/logger.js +0 -491
  420. package/dist/client/utils/mcpDefaults.js +0 -134
  421. package/dist/client/utils/messageBuilder.js +0 -1653
  422. package/dist/client/utils/modelAliasResolver.js +0 -54
  423. package/dist/client/utils/modelDetection.js +0 -80
  424. package/dist/client/utils/modelRouter.js +0 -292
  425. package/dist/client/utils/multimodalOptionsBuilder.js +0 -65
  426. package/dist/client/utils/observabilityHelpers.js +0 -47
  427. package/dist/client/utils/parameterValidation.js +0 -966
  428. package/dist/client/utils/pdfProcessor.js +0 -410
  429. package/dist/client/utils/performance.js +0 -222
  430. package/dist/client/utils/pricing.js +0 -340
  431. package/dist/client/utils/promptRedaction.js +0 -62
  432. package/dist/client/utils/providerConfig.js +0 -1009
  433. package/dist/client/utils/providerHealth.js +0 -1237
  434. package/dist/client/utils/providerRetry.js +0 -112
  435. package/dist/client/utils/providerUtils.js +0 -434
  436. package/dist/client/utils/rateLimiter.js +0 -200
  437. package/dist/client/utils/redis.js +0 -368
  438. package/dist/client/utils/retryHandler.js +0 -269
  439. package/dist/client/utils/retryability.js +0 -22
  440. package/dist/client/utils/sanitizers/svg.js +0 -481
  441. package/dist/client/utils/schemaConversion.js +0 -255
  442. package/dist/client/utils/taskClassificationUtils.js +0 -149
  443. package/dist/client/utils/taskClassifier.js +0 -94
  444. package/dist/client/utils/thinkingConfig.js +0 -104
  445. package/dist/client/utils/timeout.js +0 -359
  446. package/dist/client/utils/tokenEstimation.js +0 -142
  447. package/dist/client/utils/tokenLimits.js +0 -125
  448. package/dist/client/utils/tokenUtils.js +0 -239
  449. package/dist/client/utils/toolUtils.js +0 -75
  450. package/dist/client/utils/transformationUtils.js +0 -554
  451. package/dist/client/utils/ttsProcessor.js +0 -286
  452. package/dist/client/utils/typeUtils.js +0 -97
  453. package/dist/client/utils/videoAnalysisProcessor.js +0 -67
  454. package/dist/client/workflow/config.js +0 -398
  455. package/dist/client/workflow/core/ensembleExecutor.js +0 -407
  456. package/dist/client/workflow/core/judgeScorer.js +0 -544
  457. package/dist/client/workflow/core/responseConditioner.js +0 -225
  458. package/dist/client/workflow/core/types/conditionerTypes.js +0 -7
  459. package/dist/client/workflow/core/types/ensembleTypes.js +0 -7
  460. package/dist/client/workflow/core/types/index.js +0 -7
  461. package/dist/client/workflow/core/types/judgeTypes.js +0 -7
  462. package/dist/client/workflow/core/types/layerTypes.js +0 -7
  463. package/dist/client/workflow/core/types/registryTypes.js +0 -7
  464. package/dist/client/workflow/core/workflowRegistry.js +0 -304
  465. package/dist/client/workflow/core/workflowRunner.js +0 -586
  466. package/dist/client/workflow/index.js +0 -50
  467. package/dist/client/workflow/types.js +0 -9
  468. package/dist/client/workflow/utils/types/index.js +0 -7
  469. package/dist/client/workflow/utils/workflowMetrics.js +0 -311
  470. package/dist/client/workflow/utils/workflowValidation.js +0 -420
  471. package/dist/client/workflow/workflows/adaptiveWorkflow.js +0 -366
  472. package/dist/client/workflow/workflows/consensusWorkflow.js +0 -192
  473. package/dist/client/workflow/workflows/fallbackWorkflow.js +0 -225
  474. package/dist/client/workflow/workflows/multiJudgeWorkflow.js +0 -351
  475. /package/dist/client/{client/reactHooks.js → reactHooks.js} +0 -0
@@ -1,1385 +0,0 @@
1
- import { context, SpanKind, SpanStatusCode, trace } from "@opentelemetry/api";
2
- import { generateText } from "ai";
3
- import { directAgentTools } from "../agent/directTools.js";
4
- import { IMAGE_GENERATION_MODELS } from "../core/constants.js";
5
- import { MiddlewareFactory } from "../middleware/factory.js";
6
- import { SpanStatus, SpanType } from "../observability/types/spanTypes.js";
7
- import { SpanSerializer } from "../observability/utils/spanSerializer.js";
8
- import { ATTR, tracers } from "../telemetry/index.js";
9
- import { calculateCost } from "../utils/pricing.js";
10
- import { isAbortError } from "../utils/errorHandling.js";
11
- import { logger } from "../utils/logger.js";
12
- import { composeAbortSignals, createTimeoutController, TimeoutError, } from "../utils/timeout.js";
13
- import { shouldDisableBuiltinTools } from "../utils/toolUtils.js";
14
- import { getKeyCount, getKeysAsString } from "../utils/transformationUtils.js";
15
- import { TTSProcessor } from "../utils/ttsProcessor.js";
16
- import { executeVideoAnalysis, hasVideoFrames, } from "../utils/videoAnalysisProcessor.js";
17
- import { GenerationHandler } from "./modules/GenerationHandler.js";
18
- // Import modules for composition
19
- import { MessageBuilder } from "./modules/MessageBuilder.js";
20
- import { StreamHandler } from "./modules/StreamHandler.js";
21
- import { TelemetryHandler } from "./modules/TelemetryHandler.js";
22
- import { ToolsManager } from "./modules/ToolsManager.js";
23
- import { Utilities } from "./modules/Utilities.js";
24
- /**
25
- * Abstract base class for all AI providers
26
- * Tools are integrated as first-class citizens - always available by default
27
- */
28
- export class BaseProvider {
29
- modelName;
30
- providerName;
31
- defaultTimeout = 30000; // 30 seconds
32
- middlewareOptions; // TODO: Implement global level middlewares that can be used
33
- // Tools are conditionally included based on centralized configuration
34
- directTools = shouldDisableBuiltinTools()
35
- ? {}
36
- : directAgentTools;
37
- mcpTools; // MCP tools loaded dynamically when available
38
- customTools; // Custom tools from registerTool()
39
- toolExecutor; // Tool executor from setupToolExecutor
40
- sessionId;
41
- userId;
42
- neurolink; // Reference to actual NeuroLink instance for MCP tools
43
- /** @internal Trace context propagated from NeuroLink SDK for span hierarchy */
44
- _traceContext = null;
45
- setTraceContext(ctx) {
46
- this._traceContext = ctx;
47
- }
48
- // Composition modules - Single Responsibility Principle
49
- messageBuilder;
50
- streamHandler;
51
- generationHandler;
52
- telemetryHandler;
53
- utilities;
54
- toolsManager;
55
- constructor(modelName, providerName, neurolink, middleware) {
56
- this.modelName = modelName || this.getDefaultModel();
57
- this.providerName = providerName || this.getProviderName();
58
- this.neurolink = neurolink;
59
- this.middlewareOptions = middleware;
60
- // Initialize composition modules
61
- this.messageBuilder = new MessageBuilder(this.providerName, this.modelName);
62
- this.streamHandler = new StreamHandler(this.providerName, this.modelName);
63
- this.telemetryHandler = new TelemetryHandler(this.providerName, this.modelName, this.neurolink);
64
- this.generationHandler = new GenerationHandler(this.providerName, this.modelName, () => this.supportsTools(), (options, type) => this.telemetryHandler.getTelemetryConfig(options, type), (toolCalls, toolResults, options, timestamp) => this.handleToolExecutionStorage(toolCalls, toolResults, options, timestamp));
65
- this.utilities = new Utilities(this.providerName, this.modelName, this.defaultTimeout, this.middlewareOptions);
66
- this.toolsManager = new ToolsManager(this.providerName, this.directTools, this.neurolink, {
67
- isZodSchema: (schema) => this.isZodSchema(schema),
68
- convertToolResult: (result) => this.convertToolResult(result),
69
- createPermissiveZodSchema: () => this.createPermissiveZodSchema(),
70
- fixSchemaForOpenAIStrictMode: (schema) => this.fixSchemaForOpenAIStrictMode(schema),
71
- });
72
- }
73
- /**
74
- * Check if this provider supports tool/function calling
75
- * Override in subclasses to disable tools for specific providers or models
76
- * @returns true by default, providers can override to return false
77
- */
78
- supportsTools() {
79
- return true;
80
- }
81
- // ===================
82
- // PUBLIC API METHODS
83
- // ===================
84
- /**
85
- * Primary streaming method - implements AIProvider interface
86
- * When tools are involved, falls back to generate() with synthetic streaming
87
- */
88
- async stream(optionsOrPrompt, analysisSchema) {
89
- let options = this.normalizeStreamOptions(optionsOrPrompt);
90
- // Observability: create metrics span for provider.stream
91
- const metricsSpan = SpanSerializer.createSpan(SpanType.MODEL_GENERATION, "provider.stream", {
92
- "ai.provider": this.providerName || "unknown",
93
- "ai.model": this.modelName || options.model || "unknown",
94
- "ai.temperature": options.temperature,
95
- "ai.max_tokens": options.maxTokens,
96
- }, this._traceContext?.parentSpanId, this._traceContext?.traceId);
97
- let metricsSpanRecorded = false;
98
- // OTEL span for provider-level stream tracing
99
- const otelStreamSpan = tracers.provider.startSpan("neurolink.provider.stream", {
100
- kind: SpanKind.CLIENT,
101
- attributes: {
102
- [ATTR.GEN_AI_SYSTEM]: this.providerName || "unknown",
103
- [ATTR.GEN_AI_MODEL]: this.modelName || options.model || "unknown",
104
- [ATTR.GEN_AI_OPERATION]: "stream",
105
- [ATTR.NL_PROVIDER]: this.providerName || "unknown",
106
- },
107
- });
108
- try {
109
- logger.info(`Starting stream`, {
110
- provider: this.providerName,
111
- hasTools: !options.disableTools && this.supportsTools(),
112
- disableTools: !!options.disableTools,
113
- supportsTools: this.supportsTools(),
114
- inputLength: options.input?.text?.length || 0,
115
- maxTokens: options.maxTokens,
116
- temperature: options.temperature,
117
- timestamp: Date.now(),
118
- });
119
- // ===== EARLY MULTIMODAL DETECTION =====
120
- const hasFileInput = !!options.input?.files?.length || !!options.input?.videoFiles?.length;
121
- if (hasFileInput) {
122
- // ===== VIDEO ANALYSIS DETECTION =====
123
- // Check if video frames are present and handle with fake streaming
124
- const messages = await this.buildMessagesForStream(options);
125
- if (hasVideoFrames(messages)) {
126
- logger.info(`Video frames detected in stream, using fake streaming for video analysis`, {
127
- provider: this.providerName,
128
- model: this.modelName,
129
- });
130
- return await this.executeFakeStreaming(options, analysisSchema);
131
- }
132
- }
133
- // CRITICAL: Image generation models don't support real streaming
134
- // Force fake streaming for image models to ensure image output is yielded
135
- const isImageModel = IMAGE_GENERATION_MODELS.some((m) => this.modelName.includes(m));
136
- if (isImageModel) {
137
- logger.info(`Image model detected, forcing fake streaming`, {
138
- provider: this.providerName,
139
- model: this.modelName,
140
- reason: "Image generation requires fake streaming to yield image output",
141
- });
142
- // Skip real streaming, go directly to fake streaming
143
- return await this.executeFakeStreaming(options, analysisSchema);
144
- }
145
- // Central tool merge: Pre-merge base tools (MCP/built-in) with user-provided
146
- // tools (e.g. RAG tools) into options.tools. This way, every provider's
147
- // executeStream() can simply use options.tools (or getAllTools() + options.tools)
148
- // and get the complete tool set without needing per-provider merge logic.
149
- if (!options.disableTools && this.supportsTools()) {
150
- const mergedTools = await this.getToolsForStream(options);
151
- options = { ...options, tools: mergedTools };
152
- }
153
- else {
154
- options = { ...options, tools: {} };
155
- }
156
- // CRITICAL FIX: Always prefer real streaming over fake streaming
157
- // Try real streaming first, use fake streaming only as fallback
158
- try {
159
- logger.debug(`Attempting real streaming`, {
160
- provider: this.providerName,
161
- timestamp: Date.now(),
162
- });
163
- const realStreamResult = await this.executeStream(options, analysisSchema);
164
- logger.info(`Real streaming succeeded`, {
165
- provider: this.providerName,
166
- timestamp: Date.now(),
167
- });
168
- // If real streaming succeeds, return it (with tools support via Vercel AI SDK)
169
- return realStreamResult;
170
- }
171
- catch (realStreamError) {
172
- logger.warn(`Real streaming failed for ${this.providerName}, falling back to fake streaming:`, {
173
- error: realStreamError instanceof Error
174
- ? realStreamError.message
175
- : String(realStreamError),
176
- timestamp: Date.now(),
177
- });
178
- // Fallback to fake streaming only if real streaming fails AND tools are enabled
179
- if (!options.disableTools && this.supportsTools()) {
180
- return await this.executeFakeStreaming(options, analysisSchema);
181
- }
182
- else {
183
- // If real streaming failed and no tools are enabled, re-throw the original error
184
- logger.error(`Real streaming failed for ${this.providerName}:`, realStreamError);
185
- throw this.handleProviderError(realStreamError);
186
- }
187
- }
188
- }
189
- catch (error) {
190
- // Observability: record failed stream span
191
- metricsSpanRecorded = true;
192
- const _endedStreamSpan = SpanSerializer.endSpan(metricsSpan, SpanStatus.ERROR, error instanceof Error ? error.message : String(error));
193
- // Note: Do NOT record to getMetricsAggregator() here — neurolink.ts
194
- // stream:complete listener handles authoritative metrics to avoid double-counting.
195
- otelStreamSpan.setStatus({
196
- code: SpanStatusCode.ERROR,
197
- message: error instanceof Error ? error.message : String(error),
198
- });
199
- otelStreamSpan.end();
200
- throw error;
201
- }
202
- finally {
203
- // Observability: record successful stream span (only if not already ended via error path)
204
- if (!metricsSpanRecorded) {
205
- const _endedStreamSpan = SpanSerializer.endSpan(metricsSpan, SpanStatus.OK);
206
- // Note: Do NOT record to getMetricsAggregator() here — neurolink.ts
207
- // stream:complete listener handles authoritative metrics to avoid double-counting.
208
- }
209
- // End OTEL span on success (only if not already ended via error path)
210
- if (otelStreamSpan.isRecording()) {
211
- otelStreamSpan.setStatus({ code: SpanStatusCode.OK });
212
- otelStreamSpan.end();
213
- }
214
- }
215
- }
216
- /**
217
- * Execute fake streaming - extracted method for reusability
218
- */
219
- async executeFakeStreaming(options, analysisSchema) {
220
- try {
221
- logger.info(`Starting fake streaming with tools`, {
222
- provider: this.providerName,
223
- supportsTools: this.supportsTools(),
224
- timestamp: Date.now(),
225
- });
226
- // Convert stream options to text generation options
227
- const textOptions = {
228
- prompt: options.input?.text || "",
229
- input: options.input,
230
- systemPrompt: options.systemPrompt,
231
- temperature: options.temperature,
232
- maxTokens: options.maxTokens,
233
- tools: options.tools, // 🔧 FIX: Pass user-provided tools (including RAG tools) to generation pipeline
234
- disableTools: !!options.disableTools,
235
- maxSteps: options.maxSteps || 5,
236
- provider: options.provider,
237
- model: options.model,
238
- region: options.region, // Pass region for Vertex AI
239
- // 🔧 FIX: Include analytics and evaluation options from stream options
240
- enableAnalytics: options.enableAnalytics,
241
- enableEvaluation: options.enableEvaluation,
242
- evaluationDomain: options.evaluationDomain,
243
- toolUsageContext: options.toolUsageContext,
244
- context: options.context,
245
- csvOptions: options.csvOptions,
246
- // Forward abort, tool filtering, and timeout options to prevent
247
- // silent bypass when falling back from real streaming to fake streaming
248
- abortSignal: options.abortSignal,
249
- toolFilter: options.toolFilter,
250
- excludeTools: options.excludeTools,
251
- skipToolPromptInjection: options.skipToolPromptInjection,
252
- timeout: options.timeout,
253
- };
254
- logger.debug(`Calling generate for fake streaming`, {
255
- provider: this.providerName,
256
- maxSteps: textOptions.maxSteps,
257
- disableTools: textOptions.disableTools,
258
- timestamp: Date.now(),
259
- });
260
- const result = await this.generate(textOptions, analysisSchema);
261
- logger.info(`Generate completed for fake streaming`, {
262
- provider: this.providerName,
263
- hasContent: !!result?.content,
264
- contentLength: result?.content?.length || 0,
265
- toolsUsed: result?.toolsUsed?.length || 0,
266
- hasImageOutput: !!result?.imageOutput,
267
- timestamp: Date.now(),
268
- });
269
- // Create a synthetic stream from the generate result that simulates progressive delivery
270
- return {
271
- stream: (async function* () {
272
- if (result?.content) {
273
- // Split content into words for more natural streaming
274
- const words = result.content.split(/(\s+)/); // Keep whitespace
275
- let buffer = "";
276
- for (let i = 0; i < words.length; i++) {
277
- buffer += words[i];
278
- // Yield chunks of roughly 5-10 words or at punctuation
279
- const shouldYield = i === words.length - 1 || // Last word
280
- buffer.length > 50 || // Buffer getting long
281
- /[.!?;,]\s*$/.test(buffer); // End of sentence/clause
282
- if (shouldYield && buffer.trim()) {
283
- yield { content: buffer };
284
- buffer = "";
285
- // Small delay to simulate streaming (1-10ms)
286
- await new Promise((resolve) => {
287
- setTimeout(resolve, Math.random() * 9 + 1);
288
- });
289
- }
290
- }
291
- // Yield all remaining content
292
- if (buffer.trim()) {
293
- yield { content: buffer };
294
- }
295
- }
296
- // 🔧 CRITICAL FIX: Yield image output if present
297
- if (result?.imageOutput) {
298
- yield {
299
- type: "image",
300
- imageOutput: result.imageOutput,
301
- };
302
- }
303
- })(),
304
- usage: result?.usage,
305
- provider: result?.provider,
306
- model: result?.model,
307
- toolCalls: result?.toolCalls?.map((call) => ({
308
- toolName: call.toolName,
309
- parameters: call.args,
310
- id: call.toolCallId,
311
- })),
312
- toolResults: result?.toolResults
313
- ? result.toolResults.map((tr) => ({
314
- toolName: tr.toolName || "unknown",
315
- status: (tr.status === "error"
316
- ? "failure"
317
- : "success"),
318
- result: tr.result,
319
- error: tr.error,
320
- }))
321
- : undefined,
322
- // 🔧 FIX: Include analytics and evaluation from generate result
323
- analytics: result?.analytics,
324
- evaluation: result?.evaluation,
325
- };
326
- }
327
- catch (error) {
328
- logger.error(`Fake streaming fallback failed for ${this.providerName}:`, error);
329
- throw this.handleProviderError(error);
330
- }
331
- }
332
- /**
333
- * Apply per-call tool filtering (whitelist/blacklist) to a tools record.
334
- * If toolFilter is set, only tools whose names are in the list are kept.
335
- * If excludeTools is set, matching tools are removed. excludeTools is applied after toolFilter.
336
- */
337
- applyToolFiltering(tools, options) {
338
- if ((!options.toolFilter || options.toolFilter.length === 0) &&
339
- (!options.excludeTools || options.excludeTools.length === 0)) {
340
- return tools;
341
- }
342
- const beforeCount = Object.keys(tools).length;
343
- let filtered = { ...tools };
344
- if (options.toolFilter && options.toolFilter.length > 0) {
345
- const allowSet = new Set(options.toolFilter);
346
- const result = {};
347
- for (const [name, tool] of Object.entries(filtered)) {
348
- if (allowSet.has(name)) {
349
- result[name] = tool;
350
- }
351
- }
352
- filtered = result;
353
- }
354
- if (options.excludeTools && options.excludeTools.length > 0) {
355
- const denySet = new Set(options.excludeTools);
356
- for (const name of Object.keys(filtered)) {
357
- if (denySet.has(name)) {
358
- delete filtered[name];
359
- }
360
- }
361
- }
362
- const afterCount = Object.keys(filtered).length;
363
- if (beforeCount !== afterCount) {
364
- logger.debug(`Tool filtering applied`, {
365
- provider: this.providerName,
366
- beforeCount,
367
- afterCount,
368
- toolFilter: options.toolFilter,
369
- excludeTools: options.excludeTools,
370
- });
371
- }
372
- return filtered;
373
- }
374
- /**
375
- * Prepare generation context including tools and model
376
- */
377
- async prepareGenerationContext(options) {
378
- const shouldUseTools = !options.disableTools && this.supportsTools();
379
- const baseTools = shouldUseTools ? await this.getAllTools() : {};
380
- let tools = shouldUseTools
381
- ? {
382
- ...baseTools,
383
- ...(options.tools || {}),
384
- }
385
- : {};
386
- // Apply per-call tool filtering (whitelist/blacklist)
387
- tools = this.applyToolFiltering(tools, options);
388
- logger.debug(`Final tools prepared for AI`, {
389
- provider: this.providerName,
390
- directTools: getKeyCount(baseTools),
391
- directToolNames: getKeysAsString(baseTools),
392
- externalTools: getKeyCount(options.tools || {}),
393
- externalToolNames: getKeysAsString(options.tools || {}),
394
- totalTools: getKeyCount(tools),
395
- totalToolNames: getKeysAsString(tools),
396
- shouldUseTools,
397
- timestamp: Date.now(),
398
- });
399
- const model = await this.getAISDKModelWithMiddleware(options);
400
- return { tools, model };
401
- }
402
- /**
403
- * Get merged tools for streaming: combines base tools (MCP/built-in) with
404
- * user-provided tools (e.g., RAG tools passed via options.tools).
405
- *
406
- * This is the canonical tool-merge pattern for executeStream() implementations.
407
- * All providers should call this instead of getAllTools() directly.
408
- */
409
- async getToolsForStream(options) {
410
- const shouldUseTools = !options.disableTools && this.supportsTools();
411
- if (!shouldUseTools) {
412
- return {};
413
- }
414
- const baseTools = await this.getAllTools();
415
- const externalTools = (options.tools || {});
416
- let merged = { ...baseTools, ...externalTools };
417
- // Apply per-call tool filtering (whitelist/blacklist)
418
- merged = this.applyToolFiltering(merged, options);
419
- logger.debug(`Tools prepared for streaming`, {
420
- provider: this.providerName,
421
- baseToolCount: Object.keys(baseTools).length,
422
- externalToolCount: Object.keys(externalTools).length,
423
- totalToolCount: Object.keys(merged).length,
424
- });
425
- return merged;
426
- }
427
- /**
428
- * Build messages array for generation - delegated to MessageBuilder
429
- */
430
- async buildMessages(options) {
431
- return this.messageBuilder.buildMessages(options);
432
- }
433
- /**
434
- * Build messages array for streaming operations - delegated to MessageBuilder
435
- * This is a protected helper method that providers can use to build messages
436
- * with automatic multimodal detection, eliminating code duplication
437
- *
438
- * @param options - Stream options or text generation options
439
- * @returns Promise resolving to ModelMessage array ready for AI SDK
440
- */
441
- async buildMessagesForStream(options) {
442
- return this.messageBuilder.buildMessagesForStream(options);
443
- }
444
- /**
445
- * Execute the generation with AI SDK - delegated to GenerationHandler
446
- */
447
- async executeGeneration(model, messages, tools, options) {
448
- return this.generationHandler.executeGeneration(model, messages, tools, options);
449
- }
450
- /**
451
- * Log generation completion information - delegated to GenerationHandler
452
- */
453
- logGenerationComplete(generateResult) {
454
- this.generationHandler.logGenerationComplete(generateResult);
455
- }
456
- /**
457
- * Record performance metrics - delegated to TelemetryHandler
458
- */
459
- async recordPerformanceMetrics(usage, responseTime) {
460
- await this.telemetryHandler.recordPerformanceMetrics(usage, responseTime);
461
- }
462
- /**
463
- * Extract tool information from generation result - delegated to GenerationHandler
464
- */
465
- extractToolInformation(generateResult) {
466
- return this.generationHandler.extractToolInformation(generateResult);
467
- }
468
- /**
469
- * Format the enhanced result - delegated to GenerationHandler
470
- */
471
- formatEnhancedResult(generateResult, tools, toolsUsed, toolExecutions, options) {
472
- return this.generationHandler.formatEnhancedResult(generateResult, tools, toolsUsed, toolExecutions, options);
473
- }
474
- /**
475
- * Analyze AI response structure and log detailed debugging information - delegated to GenerationHandler
476
- */
477
- analyzeAIResponse(result) {
478
- this.generationHandler.analyzeAIResponse(result);
479
- }
480
- /**
481
- * Text generation method - implements AIProvider interface
482
- * Tools are always available unless explicitly disabled
483
- *
484
- * Supports Text-to-Speech (TTS) audio generation in two modes:
485
- * 1. Direct synthesis (default): TTS synthesizes the input text without AI generation
486
- * 2. AI response synthesis: TTS synthesizes the AI-generated response after generation
487
- *
488
- * When TTS is enabled with useAiResponse=false (default), the method returns early with
489
- * only the audio result, skipping AI generation entirely for optimal performance.
490
- *
491
- * When TTS is enabled with useAiResponse=true, the method performs full AI generation
492
- * and then synthesizes the AI response to audio.
493
- *
494
- * @param optionsOrPrompt - Generation options or prompt string
495
- * @param _analysisSchema - Optional analysis schema (not used)
496
- * @returns Enhanced result with optional audio field containing TTSResult
497
- *
498
- * IMPLEMENTATION NOTE: Uses streamText() under the hood and accumulates results
499
- * for consistency and better performance
500
- */
501
- async generate(optionsOrPrompt, _analysisSchema) {
502
- const options = this.normalizeTextOptions(optionsOrPrompt);
503
- this.validateOptions(options);
504
- const startTime = Date.now();
505
- // Observability: create metrics span for provider.generate
506
- const metricsSpan = SpanSerializer.createSpan(SpanType.MODEL_GENERATION, "provider.generate", {
507
- "ai.provider": this.providerName || "unknown",
508
- "ai.model": this.modelName || options.model || "unknown",
509
- "ai.temperature": options.temperature,
510
- "ai.max_tokens": options.maxTokens,
511
- }, this._traceContext?.parentSpanId, this._traceContext?.traceId);
512
- // OTEL span for provider-level generate tracing
513
- // Use startActiveSpan pattern via context.with() so child spans become descendants
514
- const otelSpan = tracers.provider.startSpan("neurolink.provider.generate", {
515
- kind: SpanKind.CLIENT,
516
- attributes: {
517
- [ATTR.GEN_AI_SYSTEM]: this.providerName || "unknown",
518
- [ATTR.GEN_AI_MODEL]: this.modelName || options.model || "unknown",
519
- [ATTR.GEN_AI_OPERATION]: "generate",
520
- [ATTR.NL_PROVIDER]: this.providerName || "unknown",
521
- },
522
- });
523
- // Set this span as the active context so child spans (GenerationHandler, etc.) become descendants
524
- const activeCtx = trace.setSpan(context.active(), otelSpan);
525
- let otelSpanEnded = false;
526
- return await context.with(activeCtx, async () => {
527
- try {
528
- // ===== VIDEO GENERATION MODE =====
529
- // Generate video from image + prompt using Veo 3.1
530
- if (options.output?.mode === "video") {
531
- return await this.handleVideoGeneration(options, startTime);
532
- }
533
- // ===== IMAGE GENERATION MODE =====
534
- // Route to executeImageGeneration for image generation models
535
- const isImageModel = IMAGE_GENERATION_MODELS.some((m) => this.modelName.includes(m));
536
- if (isImageModel) {
537
- logger.info(`Image generation model detected, routing to executeImageGeneration`, {
538
- provider: this.providerName,
539
- model: this.modelName,
540
- });
541
- const imageResult = await this.executeImageGeneration(options);
542
- return await this.enhanceResult(imageResult, options, startTime);
543
- }
544
- // ===== TTS MODE 1: Direct Input Synthesis (useAiResponse=false) =====
545
- // Synthesize input text directly without AI generation
546
- // This is optimal for simple read-aloud scenarios
547
- if (options.tts?.enabled && !options.tts?.useAiResponse) {
548
- const textToSynthesize = options.prompt ?? options.input?.text ?? "";
549
- // Build base result structure - common to both paths
550
- const baseResult = {
551
- content: textToSynthesize,
552
- provider: options.provider ?? this.providerName,
553
- model: this.modelName,
554
- usage: { input: 0, output: 0, total: 0 },
555
- };
556
- try {
557
- const ttsResult = await TTSProcessor.synthesize(textToSynthesize, options.provider ?? this.providerName, options.tts);
558
- baseResult.audio = ttsResult;
559
- }
560
- catch (ttsError) {
561
- logger.error(`TTS synthesis failed in Mode 1 (direct input synthesis):`, ttsError);
562
- // baseResult remains without audio - graceful degradation
563
- }
564
- // Call enhanceResult for consistency - enables analytics/evaluation for TTS-only requests
565
- return await this.enhanceResult(baseResult, options, startTime);
566
- }
567
- // ===== Normal AI Generation Flow =====
568
- const { tools, model } = await this.prepareGenerationContext(options);
569
- const messages = await this.buildMessages(options);
570
- // ===== VIDEO ANALYSIS FROM MESSAGES CONTENT =====
571
- // Check if video files are present in messages content array
572
- // If video analysis is needed, perform it via Gemini, then pass through Claude for formatting
573
- if (hasVideoFrames(messages)) {
574
- const videoAnalysisResult = await executeVideoAnalysis(messages, {
575
- provider: options.provider,
576
- providerName: this.providerName,
577
- region: options.region,
578
- // Don't pass the main conversation model — video analysis uses
579
- // Google's Gemini API (generateContent) which only supports Gemini models.
580
- // Let videoAnalysisProcessor use its own default (gemini-2.5-flash).
581
- });
582
- // Extract user's original text from messages (excluding image parts)
583
- const userTextParts = messages
584
- .filter((m) => m.role === "user")
585
- .flatMap((m) => Array.isArray(m.content)
586
- ? m.content
587
- .filter((p) => p.type === "text")
588
- .map((p) => p.text)
589
- : [typeof m.content === "string" ? m.content : ""])
590
- .filter(Boolean);
591
- const userText = userTextParts.join("\n").trim();
592
- // Pass Gemini's analysis through Claude for structured JSON formatting
593
- // The system prompt (from Curator) includes JSON_REPORT_PROMPT_SUFFIX
594
- // which instructs Claude to output {"summary": "...", "details": "..."}
595
- let formattedContent = videoAnalysisResult;
596
- let usage = { input: 0, output: 0, total: 0 };
597
- if (options.systemPrompt) {
598
- try {
599
- const formattingPrompt = userText
600
- ? `The user asked: "${userText}"\n\nHere is the video/image analysis result from the visual analysis system:\n\n${videoAnalysisResult}\n\nBased on this analysis, provide your response.`
601
- : `Here is a video/image analysis result from the visual analysis system:\n\n${videoAnalysisResult}\n\nBased on this analysis, provide your response.`;
602
- logger.debug("[VideoAnalysis] Formatting via Claude", {
603
- userTextLength: userText.length,
604
- analysisLength: videoAnalysisResult.length,
605
- });
606
- const formattedResult = await generateText({
607
- model,
608
- system: options.systemPrompt,
609
- messages: [
610
- { role: "user", content: formattingPrompt },
611
- ],
612
- maxOutputTokens: options.maxTokens || 8192,
613
- temperature: 0.3,
614
- abortSignal: options.abortSignal,
615
- experimental_telemetry: this.telemetryHandler?.getTelemetryConfig(options, "generate"),
616
- });
617
- formattedContent = formattedResult.text;
618
- usage = {
619
- input: formattedResult.usage?.inputTokens || 0,
620
- output: formattedResult.usage?.outputTokens || 0,
621
- total: (formattedResult.usage?.inputTokens || 0) +
622
- (formattedResult.usage?.outputTokens || 0),
623
- };
624
- logger.debug("[VideoAnalysis] Claude formatting complete", {
625
- formattedLength: formattedContent.length,
626
- usage,
627
- });
628
- }
629
- catch (error) {
630
- logger.warn("[VideoAnalysis] Claude formatting failed, using raw Gemini output", {
631
- error: error instanceof Error ? error.message : String(error),
632
- });
633
- // formattedContent remains as raw videoAnalysisResult (graceful degradation)
634
- }
635
- }
636
- const videoResult = {
637
- content: formattedContent,
638
- provider: options.provider ?? this.providerName,
639
- model: this.modelName,
640
- usage,
641
- };
642
- return await this.enhanceResult(videoResult, options, startTime);
643
- }
644
- // Compose timeout signal with user-provided abort signal (mirrors stream path)
645
- const timeoutController = createTimeoutController(options.timeout, this.providerName, "generate");
646
- const composedSignal = composeAbortSignals(options.abortSignal, timeoutController?.controller.signal);
647
- const composedOptions = composedSignal
648
- ? { ...options, abortSignal: composedSignal }
649
- : options;
650
- let generateResult;
651
- try {
652
- generateResult = await this.executeGeneration(model, messages, tools, composedOptions);
653
- }
654
- finally {
655
- timeoutController?.cleanup();
656
- }
657
- this.analyzeAIResponse(generateResult);
658
- this.logGenerationComplete(generateResult);
659
- const responseTime = Date.now() - startTime;
660
- await this.recordPerformanceMetrics(generateResult.usage, responseTime);
661
- const { toolsUsed, toolExecutions } = this.extractToolInformation(generateResult);
662
- let enhancedResult = this.formatEnhancedResult(generateResult, tools, toolsUsed, toolExecutions, options);
663
- // ===== TTS MODE 2: AI Response Synthesis (useAiResponse=true) =====
664
- // Synthesize AI-generated response after generation completes
665
- if (options.tts?.enabled && options.tts?.useAiResponse) {
666
- const aiResponse = enhancedResult.content;
667
- const provider = options.provider ?? this.providerName;
668
- // Validate AI response and provider before synthesis
669
- if (aiResponse && provider) {
670
- try {
671
- const ttsResult = await TTSProcessor.synthesize(aiResponse, provider, options.tts);
672
- // Add audio to enhanced result (TTSProcessor already includes latency in metadata)
673
- enhancedResult = {
674
- ...enhancedResult,
675
- audio: ttsResult,
676
- };
677
- }
678
- catch (ttsError) {
679
- // Log TTS error but continue with text-only result
680
- logger.error(`TTS synthesis failed in Mode 2 (AI response synthesis):`, ttsError);
681
- // enhancedResult remains unchanged (no audio field added)
682
- }
683
- }
684
- else {
685
- logger.warn(`TTS synthesis skipped despite being enabled`, {
686
- provider: this.providerName,
687
- hasAiResponse: !!aiResponse,
688
- aiResponseLength: aiResponse?.length ?? 0,
689
- hasProvider: !!provider,
690
- ttsConfig: {
691
- enabled: options.tts?.enabled,
692
- useAiResponse: options.tts?.useAiResponse,
693
- },
694
- reason: !aiResponse
695
- ? "AI response is empty or undefined"
696
- : "Provider is missing",
697
- });
698
- }
699
- }
700
- // Observability: record successful generate span with token/cost data
701
- let enrichedGenerateSpan = { ...metricsSpan };
702
- if (enhancedResult?.usage) {
703
- enrichedGenerateSpan = SpanSerializer.enrichWithTokenUsage(enrichedGenerateSpan, {
704
- promptTokens: enhancedResult.usage.input || 0,
705
- completionTokens: enhancedResult.usage.output || 0,
706
- totalTokens: enhancedResult.usage.total || 0,
707
- });
708
- const cost = calculateCost(this.providerName, this.modelName, {
709
- input: enhancedResult.usage.input || 0,
710
- output: enhancedResult.usage.output || 0,
711
- total: enhancedResult.usage.total || 0,
712
- });
713
- if (cost && cost > 0) {
714
- enrichedGenerateSpan = SpanSerializer.enrichWithCost(enrichedGenerateSpan, {
715
- totalCost: cost,
716
- });
717
- }
718
- }
719
- const _endedGenerateSpan = SpanSerializer.endSpan(enrichedGenerateSpan, SpanStatus.OK);
720
- // Note: Do NOT record to getMetricsAggregator() here — the neurolink.ts
721
- // generation:end listener creates an authoritative span with richer context
722
- // (provider name, model, input/output) and records to both aggregators.
723
- // Recording here would double-count cost and token metrics.
724
- return await this.enhanceResult(enhancedResult, options, startTime);
725
- }
726
- catch (error) {
727
- // Observability: record failed generate span
728
- const _endedGenerateSpan = SpanSerializer.endSpan(metricsSpan, SpanStatus.ERROR, error instanceof Error ? error.message : String(error));
729
- // Note: Do NOT record to getMetricsAggregator() here — neurolink.ts
730
- // handles authoritative metrics recording to avoid double-counting.
731
- otelSpan.setStatus({
732
- code: SpanStatusCode.ERROR,
733
- message: error instanceof Error ? error.message : String(error),
734
- });
735
- otelSpan.end();
736
- otelSpanEnded = true;
737
- // Abort errors are expected when a generation is cancelled — log at info, not error
738
- if (isAbortError(error)) {
739
- logger.info(`Generate aborted for ${this.providerName}`, {
740
- error: error instanceof Error ? error.message : String(error),
741
- });
742
- }
743
- else {
744
- logger.error(`Generate failed for ${this.providerName}:`, error);
745
- }
746
- throw this.handleProviderError(error);
747
- }
748
- finally {
749
- if (!otelSpanEnded) {
750
- otelSpan.setStatus({ code: SpanStatusCode.OK });
751
- otelSpan.end();
752
- }
753
- }
754
- }); // end context.with
755
- }
756
- /**
757
- * Alias for generate method - implements AIProvider interface
758
- */
759
- async gen(optionsOrPrompt, analysisSchema) {
760
- return this.generate(optionsOrPrompt, analysisSchema);
761
- }
762
- /**
763
- * BACKWARD COMPATIBILITY: Legacy generateText method
764
- * Converts EnhancedGenerateResult to TextGenerationResult format
765
- * Ensures existing scripts using createAIProvider().generateText() continue to work
766
- */
767
- async generateText(options) {
768
- // Validate required parameters for backward compatibility - support both prompt and input.text
769
- const promptText = options.prompt || options.input?.text;
770
- if (!promptText ||
771
- typeof promptText !== "string" ||
772
- promptText.trim() === "") {
773
- throw new Error("GenerateText options must include prompt or input.text as a non-empty string");
774
- }
775
- // Call the main generate method
776
- const result = await this.generate(options);
777
- if (!result) {
778
- throw new Error("Generation failed: No result returned");
779
- }
780
- // Convert EnhancedGenerateResult to TextGenerationResult format
781
- return {
782
- content: result.content || "",
783
- provider: result.provider || this.providerName,
784
- model: result.model || this.modelName,
785
- usage: result.usage || {
786
- input: 0,
787
- output: 0,
788
- total: 0,
789
- },
790
- responseTime: 0, // BaseProvider doesn't track response time directly
791
- toolsUsed: result.toolsUsed || [],
792
- enhancedWithTools: !!(result.toolsUsed && result.toolsUsed.length > 0),
793
- analytics: result.analytics,
794
- evaluation: result.evaluation,
795
- audio: result.audio,
796
- };
797
- }
798
- /**
799
- * Generate embeddings for text
800
- *
801
- * This is a default implementation that throws an error.
802
- * Providers that support embeddings (OpenAI, Google Vertex, Amazon Bedrock)
803
- * should override this method with their specific implementation.
804
- *
805
- * @param text - The text to embed
806
- * @param _modelName - Optional embedding model name (provider-specific)
807
- * @returns Promise resolving to the embedding vector (array of numbers)
808
- * @throws Error if the provider does not support embeddings
809
- *
810
- * @example
811
- * ```typescript
812
- * const provider = await ProviderFactory.createProvider('openai', 'text-embedding-3-small');
813
- * const embedding = await provider.embed('Hello world');
814
- * console.log(embedding); // [0.123, -0.456, ...]
815
- * ```
816
- */
817
- async embed(text, _modelName) {
818
- logger.warn(`embed() called on ${this.providerName} which does not have a native implementation`, {
819
- textLength: text.length,
820
- });
821
- throw new Error(`Embedding generation is not supported by the ${this.providerName} provider. ` +
822
- `Supported providers: openai, vertex/google, bedrock. ` +
823
- `Use an embedding model like text-embedding-3-small (OpenAI), text-embedding-004 (Vertex), ` +
824
- `or amazon.titan-embed-text-v2:0 (Bedrock).`);
825
- }
826
- /**
827
- * Generate embeddings for multiple texts in a single batch
828
- *
829
- * This is a default implementation that throws an error.
830
- * Providers that support embeddings should override this method.
831
- * The AI SDK's embedMany automatically handles chunking for models with batch limits.
832
- *
833
- * @param texts - The texts to embed
834
- * @param _modelName - Optional embedding model name (provider-specific)
835
- * @returns Promise resolving to an array of embedding vectors
836
- * @throws Error if the provider does not support embeddings
837
- */
838
- async embedMany(texts, _modelName) {
839
- logger.warn(`embedMany() called on ${this.providerName} which does not have a native implementation`, {
840
- count: texts.length,
841
- });
842
- throw new Error(`Batch embedding generation is not supported by the ${this.providerName} provider. ` +
843
- `Supported providers: openai, googleAiStudio, vertex/google, bedrock. ` +
844
- `Use an embedding model like text-embedding-3-small (OpenAI), gemini-embedding-001 (Google AI), ` +
845
- `text-embedding-004 (Vertex), or amazon.titan-embed-text-v2:0 (Bedrock).`);
846
- }
847
- /**
848
- * Get the default embedding model for this provider
849
- *
850
- * Override in subclasses to provide provider-specific defaults.
851
- * Returns undefined for providers that don't support embeddings.
852
- *
853
- * @returns The default embedding model name, or undefined if not supported
854
- */
855
- getDefaultEmbeddingModel() {
856
- // Default implementation returns undefined - providers override this
857
- return undefined;
858
- }
859
- /**
860
- * Get AI SDK model with middleware applied
861
- * This method wraps the base model with any configured middleware
862
- * TODO: Implement global level middlewares that can be used
863
- */
864
- async getAISDKModelWithMiddleware(options = {}) {
865
- // Get the base model
866
- const baseModel = await this.getAISDKModel();
867
- logger.debug(`Retrieved base model for ${this.providerName}`, {
868
- provider: this.providerName,
869
- model: this.modelName,
870
- hasMiddlewareConfig: !!this.middlewareOptions,
871
- timestamp: Date.now(),
872
- });
873
- // Check if middleware should be applied
874
- const middlewareOptions = this.extractMiddlewareOptions(options);
875
- logger.debug(`Middleware extraction result`, {
876
- provider: this.providerName,
877
- model: this.modelName,
878
- middlewareOptions,
879
- });
880
- if (!middlewareOptions) {
881
- return baseModel;
882
- }
883
- try {
884
- logger.debug(`Applying middleware to ${this.providerName} model`, {
885
- provider: this.providerName,
886
- model: this.modelName,
887
- middlewareOptions,
888
- });
889
- // Create a new factory instance with the specified options
890
- const factory = new MiddlewareFactory(middlewareOptions);
891
- // Create middleware context
892
- const context = factory.createContext(this.providerName, this.modelName, options, {
893
- sessionId: this.sessionId,
894
- userId: this.userId,
895
- });
896
- // Apply middleware to the model
897
- const wrappedModel = factory.applyMiddleware(baseModel, context, middlewareOptions);
898
- logger.debug(`Applied middleware to ${this.providerName} model`, {
899
- provider: this.providerName,
900
- model: this.modelName,
901
- hasMiddleware: true,
902
- });
903
- return wrappedModel;
904
- }
905
- catch (error) {
906
- logger.warn(`Failed to apply middleware to ${this.providerName}, using base model`, {
907
- error: error instanceof Error ? error.message : String(error),
908
- });
909
- // Return base model on middleware failure to maintain functionality
910
- return baseModel;
911
- }
912
- }
913
- /**
914
- * Extract middleware options - delegated to Utilities
915
- */
916
- extractMiddlewareOptions(options) {
917
- return this.utilities.extractMiddlewareOptions(options);
918
- }
919
- // ===================
920
- // TOOL MANAGEMENT
921
- // ===================
922
- /**
923
- * Check if a schema is a Zod schema - delegated to Utilities
924
- */
925
- isZodSchema(schema) {
926
- return this.utilities.isZodSchema(schema);
927
- }
928
- /**
929
- * Convert tool execution result - delegated to Utilities
930
- */
931
- async convertToolResult(result) {
932
- return this.utilities.convertToolResult(result);
933
- }
934
- /**
935
- * Fix JSON Schema for OpenAI strict mode - delegated to Utilities
936
- */
937
- fixSchemaForOpenAIStrictMode(schema) {
938
- return this.utilities.fixSchemaForOpenAIStrictMode(schema);
939
- }
940
- /**
941
- * Get all available tools - delegated to ToolsManager
942
- */
943
- async getAllTools() {
944
- return this.toolsManager.getAllTools();
945
- }
946
- /**
947
- * Calculate actual cost - delegated to TelemetryHandler
948
- */
949
- async calculateActualCost(usage) {
950
- return this.telemetryHandler.calculateActualCost(usage);
951
- }
952
- /**
953
- * Create a permissive Zod schema - delegated to Utilities
954
- */
955
- createPermissiveZodSchema() {
956
- return this.utilities.createPermissiveZodSchema();
957
- }
958
- /**
959
- * Set session context for MCP tools - delegated to ToolsManager
960
- */
961
- setSessionContext(sessionId, userId) {
962
- this.sessionId = sessionId;
963
- this.userId = userId;
964
- this.toolsManager.setSessionContext(sessionId, userId);
965
- }
966
- /**
967
- * Handle provider errors with abort passthrough.
968
- * AbortErrors are never wrapped — they must propagate with their
969
- * original identity so that isAbortError() can detect them in
970
- * retry/fallback loops (directProviderGeneration, performMCPGenerationRetries).
971
- */
972
- handleProviderError(error) {
973
- if (isAbortError(error)) {
974
- // Preserve AbortError identity — never wrap in provider-specific formatting
975
- return error instanceof Error
976
- ? error
977
- : new DOMException("The operation was aborted", "AbortError");
978
- }
979
- return this.formatProviderError(error);
980
- }
981
- /**
982
- * Image generation method. Providers that support it should override this.
983
- * By default, it throws an error indicating that the functionality is not supported.
984
- * @param _options The generation options.
985
- * @returns A promise that resolves to the generation result.
986
- */
987
- async executeImageGeneration(_options) {
988
- throw new Error(`Image generation is not supported by the ${this.providerName} provider or the selected model.`);
989
- }
990
- // ===================
991
- // CONSOLIDATED PROVIDER METHODS - MOVED FROM INDIVIDUAL PROVIDERS
992
- // ===================
993
- /**
994
- * Execute operation with timeout and proper cleanup
995
- * Consolidates identical timeout handling from 8/10 providers
996
- */
997
- async executeWithTimeout(operation, options) {
998
- const timeout = this.getTimeout(options);
999
- const timeoutController = createTimeoutController(timeout, this.providerName, options.operationType || "generate");
1000
- try {
1001
- if (timeoutController) {
1002
- return await Promise.race([
1003
- operation(),
1004
- new Promise((_, reject) => {
1005
- timeoutController.controller.signal.addEventListener("abort", () => {
1006
- reject(new TimeoutError(`${this.providerName} operation timed out`, timeoutController.timeoutMs, this.providerName, options.operationType ||
1007
- "generate"));
1008
- });
1009
- }),
1010
- ]);
1011
- }
1012
- else {
1013
- return await operation();
1014
- }
1015
- }
1016
- finally {
1017
- timeoutController?.cleanup();
1018
- }
1019
- }
1020
- /**
1021
- * Validate stream options - delegated to StreamHandler
1022
- */
1023
- validateStreamOptions(options) {
1024
- this.streamHandler.validateStreamOptions(options);
1025
- }
1026
- /**
1027
- * Create text stream transformation - delegated to StreamHandler
1028
- */
1029
- createTextStream(result) {
1030
- return this.streamHandler.createTextStream(result);
1031
- }
1032
- /**
1033
- * Create standardized stream result - delegated to StreamHandler
1034
- */
1035
- createStreamResult(stream, additionalProps = {}) {
1036
- return this.streamHandler.createStreamResult(stream, additionalProps);
1037
- }
1038
- /**
1039
- * Create stream analytics - delegated to StreamHandler
1040
- */
1041
- async createStreamAnalytics(result, startTime, options) {
1042
- return this.streamHandler.createStreamAnalytics(result, startTime, options);
1043
- }
1044
- /**
1045
- * Handle common error patterns - delegated to Utilities
1046
- */
1047
- handleCommonErrors(error) {
1048
- return this.utilities.handleCommonErrors(error);
1049
- }
1050
- /**
1051
- * Set up tool executor - delegated to ToolsManager
1052
- * @param sdk - The NeuroLinkSDK instance for tool execution
1053
- * @param functionTag - Function name for logging
1054
- */
1055
- setupToolExecutor(sdk, functionTag) {
1056
- this.toolsManager.setupToolExecutor(sdk, functionTag);
1057
- }
1058
- // ===================
1059
- // TEMPLATE METHODS - COMMON FUNCTIONALITY
1060
- // ===================
1061
- /**
1062
- * Normalize text generation options - delegated to Utilities
1063
- */
1064
- normalizeTextOptions(optionsOrPrompt) {
1065
- return this.utilities.normalizeTextOptions(optionsOrPrompt);
1066
- }
1067
- /**
1068
- * Normalize stream options - delegated to Utilities
1069
- */
1070
- normalizeStreamOptions(optionsOrPrompt) {
1071
- return this.utilities.normalizeStreamOptions(optionsOrPrompt);
1072
- }
1073
- async enhanceResult(result, options, startTime) {
1074
- const responseTime = Date.now() - startTime;
1075
- // CRITICAL FIX: Store imageOutput separately to ensure it's preserved
1076
- const imageOutput = result.imageOutput;
1077
- let enhancedResult = { ...result };
1078
- if (options.enableAnalytics) {
1079
- try {
1080
- const analytics = await this.createAnalytics(result, responseTime, options);
1081
- // Preserve ALL fields including imageOutput when adding analytics
1082
- enhancedResult = { ...enhancedResult, analytics, imageOutput };
1083
- }
1084
- catch (error) {
1085
- logger.warn(`Analytics creation failed for ${this.providerName}:`, error);
1086
- }
1087
- }
1088
- if (options.enableEvaluation) {
1089
- try {
1090
- const evaluation = await this.createEvaluation(result, options);
1091
- // Preserve ALL fields including imageOutput when adding evaluation
1092
- enhancedResult = { ...enhancedResult, evaluation, imageOutput };
1093
- }
1094
- catch (error) {
1095
- logger.warn(`Evaluation creation failed for ${this.providerName}:`, error);
1096
- }
1097
- }
1098
- // CRITICAL FIX: Always restore imageOutput if it existed in the original result
1099
- if (imageOutput) {
1100
- enhancedResult.imageOutput = imageOutput;
1101
- }
1102
- return enhancedResult;
1103
- }
1104
- /**
1105
- * Handle video generation mode
1106
- *
1107
- * Generates video from input image + text prompt using Vertex AI Veo 3.1.
1108
- *
1109
- * @param options - Text generation options with video configuration
1110
- * @param startTime - Generation start timestamp for metrics
1111
- * @returns Enhanced result with video data
1112
- *
1113
- * @example
1114
- * ```typescript
1115
- * const result = await provider.generate({
1116
- * input: { text: "Product showcase", images: [imageBuffer] },
1117
- * output: { mode: "video", video: { resolution: "1080p" } }
1118
- * });
1119
- * // result.video contains the generated video
1120
- * ```
1121
- */
1122
- async handleVideoGeneration(options, startTime) {
1123
- // Dynamic imports to avoid loading video dependencies unless needed
1124
- const { generateVideoWithVertex, VideoError, VIDEO_ERROR_CODES } = await import("../adapters/video/vertexVideoHandler.js");
1125
- const { validateVideoGenerationInput, validateImageForVideo, validateDirectorModeInput, } = await import("../utils/parameterValidation.js");
1126
- const { ErrorFactory } = await import("../utils/errorHandling.js");
1127
- // Build GenerateOptions for validation
1128
- const generateOptions = {
1129
- input: options.input || { text: options.prompt || "" },
1130
- output: options.output,
1131
- provider: options.provider,
1132
- model: options.model,
1133
- };
1134
- // ===== DIRECTOR MODE =====
1135
- // Route to Director pipeline when segments are provided
1136
- if (generateOptions.input?.segments &&
1137
- Array.isArray(generateOptions.input.segments) &&
1138
- generateOptions.input.segments.length > 0) {
1139
- // Type narrowing: segments is guaranteed to exist here
1140
- const segments = generateOptions.input.segments;
1141
- const directorValidation = validateDirectorModeInput(generateOptions);
1142
- if (!directorValidation.isValid) {
1143
- throw ErrorFactory.invalidParameters("director-mode", new Error(directorValidation.errors
1144
- .map((e) => e.message)
1145
- .join("; ")), { errors: directorValidation.errors });
1146
- }
1147
- if (directorValidation.warnings.length > 0) {
1148
- for (const warning of directorValidation.warnings) {
1149
- logger.warn(`Director Mode warning: ${warning}`);
1150
- }
1151
- }
1152
- const { executeDirectorPipeline, DIRECTOR_PIPELINE_TIMEOUT_MS } = await import("../adapters/video/directorPipeline.js");
1153
- // Use caller's timeout if provided, otherwise use default Director timeout
1154
- const directorTimeout = options.timeout ?? DIRECTOR_PIPELINE_TIMEOUT_MS;
1155
- const videoResult = await this.executeWithTimeout(() => executeDirectorPipeline(segments, generateOptions.output?.video ?? {}, generateOptions.output?.director ?? {}, options.region), { timeout: directorTimeout, operationType: "generate" });
1156
- // Build content summary with metadata
1157
- const joinedPrompts = generateOptions.input.segments
1158
- .map((s) => s.prompt)
1159
- .join(" → ");
1160
- const segmentCount = videoResult.metadata?.segmentCount ??
1161
- generateOptions.input.segments.length;
1162
- const transitionCount = videoResult.metadata?.transitionCount ?? Math.max(0, segmentCount - 1);
1163
- const totalDuration = videoResult.metadata?.duration ?? 0;
1164
- const contentSummary = `${joinedPrompts} — duration: ${totalDuration}s, segments: ${segmentCount}, transitions: ${transitionCount}`;
1165
- const baseResult = {
1166
- content: contentSummary,
1167
- provider: "vertex",
1168
- model: options.model || "veo-3.1-generate-001",
1169
- usage: { input: 0, output: 0, total: 0 },
1170
- video: videoResult,
1171
- };
1172
- return await this.enhanceResult(baseResult, options, startTime);
1173
- }
1174
- // ===== STANDARD SINGLE-CLIP VIDEO GENERATION =====
1175
- // Validate video generation input
1176
- const validation = validateVideoGenerationInput(generateOptions);
1177
- if (!validation.isValid) {
1178
- throw ErrorFactory.invalidParameters("video-generation", new Error(validation.errors.map((e) => e.message).join("; ")), { errors: validation.errors });
1179
- }
1180
- // Log warnings if any
1181
- if (validation.warnings.length > 0) {
1182
- for (const warning of validation.warnings) {
1183
- logger.warn(`Video generation warning: ${warning}`);
1184
- }
1185
- }
1186
- // Extract image from input
1187
- const imageInput = options.input?.images?.[0];
1188
- if (!imageInput) {
1189
- throw new VideoError({
1190
- code: VIDEO_ERROR_CODES.INVALID_INPUT,
1191
- message: "Video generation requires an input image. Provide via input.images array.",
1192
- retriable: false,
1193
- context: { field: "input.images" },
1194
- });
1195
- }
1196
- // Timeout for image IO operations (15 seconds)
1197
- const IMAGE_IO_TIMEOUT_MS = 15000;
1198
- // Load image buffer if path/URL
1199
- let imageBuffer;
1200
- if (typeof imageInput === "string") {
1201
- if (imageInput.startsWith("http://") ||
1202
- imageInput.startsWith("https://")) {
1203
- // URL - fetch the image with timeout
1204
- logger.debug("Fetching image from URL for video generation", {
1205
- url: imageInput.substring(0, 100),
1206
- });
1207
- let response;
1208
- try {
1209
- response = await this.executeWithTimeout(() => fetch(imageInput), {
1210
- timeout: IMAGE_IO_TIMEOUT_MS,
1211
- operationType: "generate", // Part of video generation flow
1212
- });
1213
- }
1214
- catch (error) {
1215
- throw new VideoError({
1216
- code: VIDEO_ERROR_CODES.INVALID_INPUT,
1217
- message: `Failed to fetch image from URL: ${error instanceof Error ? error.message : "Request timed out"}`,
1218
- retriable: true,
1219
- context: { url: imageInput, timeout: IMAGE_IO_TIMEOUT_MS },
1220
- originalError: error instanceof Error ? error : undefined,
1221
- });
1222
- }
1223
- if (!response.ok) {
1224
- throw new VideoError({
1225
- code: VIDEO_ERROR_CODES.INVALID_INPUT,
1226
- message: `Failed to fetch image from URL: ${response.status} ${response.statusText}`,
1227
- retriable: response.status >= 500,
1228
- context: { url: imageInput, status: response.status },
1229
- });
1230
- }
1231
- imageBuffer = Buffer.from(await response.arrayBuffer());
1232
- }
1233
- else {
1234
- // File path - read from disk with timeout
1235
- logger.debug("Reading image from path for video generation", {
1236
- path: imageInput,
1237
- });
1238
- const fs = await import("node:fs/promises");
1239
- try {
1240
- imageBuffer = await this.executeWithTimeout(() => fs.readFile(imageInput), { timeout: IMAGE_IO_TIMEOUT_MS, operationType: "generate" });
1241
- }
1242
- catch (error) {
1243
- throw new VideoError({
1244
- code: VIDEO_ERROR_CODES.INVALID_INPUT,
1245
- message: `Failed to read image file: ${error instanceof Error ? error.message : String(error)}`,
1246
- retriable: false,
1247
- context: { path: imageInput, timeout: IMAGE_IO_TIMEOUT_MS },
1248
- originalError: error instanceof Error ? error : undefined,
1249
- });
1250
- }
1251
- }
1252
- }
1253
- else if (Buffer.isBuffer(imageInput)) {
1254
- imageBuffer = imageInput;
1255
- }
1256
- else if (typeof imageInput === "object" && "data" in imageInput) {
1257
- // ImageWithAltText type
1258
- const imgData = imageInput.data;
1259
- if (typeof imgData === "string") {
1260
- imageBuffer = Buffer.from(imgData, "base64");
1261
- }
1262
- else if (Buffer.isBuffer(imgData)) {
1263
- imageBuffer = imgData;
1264
- }
1265
- else {
1266
- throw new VideoError({
1267
- code: VIDEO_ERROR_CODES.INVALID_INPUT,
1268
- message: "ImageWithAltText.data must be a base64 string or Buffer.",
1269
- retriable: false,
1270
- context: { field: "input.images[0].data", type: typeof imgData },
1271
- });
1272
- }
1273
- }
1274
- else {
1275
- throw new VideoError({
1276
- code: VIDEO_ERROR_CODES.INVALID_INPUT,
1277
- message: "Invalid image input type. Provide Buffer, path string, URL, or ImageWithAltText.",
1278
- retriable: false,
1279
- context: { field: "input.images[0]", type: typeof imageInput },
1280
- });
1281
- }
1282
- // Validate image format and size (for Buffer inputs)
1283
- const imageValidation = validateImageForVideo(imageBuffer);
1284
- if (imageValidation) {
1285
- throw ErrorFactory.invalidParameters("video-generation", new Error(imageValidation.message), {
1286
- field: "input.images[0]",
1287
- validation: imageValidation,
1288
- });
1289
- }
1290
- // Get prompt text
1291
- const prompt = options.prompt || options.input?.text || "";
1292
- logger.info("Starting video generation", {
1293
- provider: "vertex",
1294
- model: options.model || "veo-3.1-generate-001",
1295
- promptLength: prompt.length,
1296
- imageSize: imageBuffer.length,
1297
- resolution: options.output?.video?.resolution || "720p",
1298
- duration: options.output?.video?.length || 6,
1299
- });
1300
- // Generate video using Vertex handler (no processor abstraction)
1301
- const videoResult = await generateVideoWithVertex(imageBuffer, prompt, options.output?.video, options.region);
1302
- logger.info("Video generation complete", {
1303
- videoSize: videoResult.data.length,
1304
- duration: videoResult.metadata?.duration,
1305
- processingTime: videoResult.metadata?.processingTime,
1306
- });
1307
- // Build result
1308
- const baseResult = {
1309
- content: prompt, // Echo the prompt as content
1310
- provider: "vertex",
1311
- model: options.model || "veo-3.1-generate-001",
1312
- usage: { input: 0, output: 0, total: 0 },
1313
- video: videoResult,
1314
- };
1315
- return await this.enhanceResult(baseResult, options, startTime);
1316
- }
1317
- /**
1318
- * Create analytics - delegated to TelemetryHandler
1319
- */
1320
- async createAnalytics(result, responseTime, options) {
1321
- return this.telemetryHandler.createAnalytics(result, responseTime, options.context);
1322
- }
1323
- /**
1324
- * Create evaluation - delegated to TelemetryHandler
1325
- */
1326
- async createEvaluation(result, options) {
1327
- return this.telemetryHandler.createEvaluation(result, options);
1328
- }
1329
- /**
1330
- * Validate text generation options - delegated to Utilities
1331
- */
1332
- validateOptions(options) {
1333
- this.utilities.validateOptions(options);
1334
- }
1335
- /**
1336
- * Get provider information - delegated to Utilities
1337
- */
1338
- getProviderInfo() {
1339
- return this.utilities.getProviderInfo();
1340
- }
1341
- /**
1342
- * Get timeout value in milliseconds - delegated to Utilities
1343
- */
1344
- getTimeout(options) {
1345
- return this.utilities.getTimeout(options);
1346
- }
1347
- /**
1348
- * Check if tool executions should be stored and handle storage
1349
- */
1350
- async handleToolExecutionStorage(toolCalls, toolResults, options, currentTime) {
1351
- return this.telemetryHandler.handleToolExecutionStorage(toolCalls, toolResults, options, currentTime);
1352
- }
1353
- /**
1354
- * Utility method to chunk large prompts into smaller pieces
1355
- * @param prompt The prompt to chunk
1356
- * @param maxChunkSize Maximum size per chunk (default: 900,000 characters)
1357
- * @param overlap Overlap between chunks to maintain context (default: 100 characters)
1358
- * @returns Array of prompt chunks
1359
- */
1360
- static chunkPrompt(prompt, maxChunkSize = 900000, overlap = 100) {
1361
- if (prompt.length <= maxChunkSize) {
1362
- return [prompt];
1363
- }
1364
- const chunks = [];
1365
- let start = 0;
1366
- while (start < prompt.length) {
1367
- const end = Math.min(start + maxChunkSize, prompt.length);
1368
- chunks.push(prompt.slice(start, end));
1369
- // Break if we've reached the end
1370
- if (end >= prompt.length) {
1371
- break;
1372
- }
1373
- // Move start forward, accounting for overlap
1374
- const nextStart = end - overlap;
1375
- // Ensure we make progress (avoid infinite loops)
1376
- if (nextStart <= start) {
1377
- start = end;
1378
- }
1379
- else {
1380
- start = Math.max(nextStart, 0);
1381
- }
1382
- }
1383
- return chunks;
1384
- }
1385
- }