@juspay/neurolink 9.32.0 → 9.32.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (467) hide show
  1. package/CHANGELOG.md +6 -0
  2. package/dist/auth/anthropicOAuth.js +1 -1
  3. package/dist/cli/commands/proxy.js +18 -5
  4. package/dist/client/aiSdkAdapter.js +1 -1
  5. package/dist/client/index.js +137 -501
  6. package/dist/core/factory.js +0 -1
  7. package/dist/core/redisConversationMemoryManager.js +1 -1
  8. package/dist/features/ppt/slideGenerator.js +0 -1
  9. package/dist/features/ppt/utils.js +0 -1
  10. package/dist/lib/server/routes/claudeProxyRoutes.js +45 -9
  11. package/dist/mcp/elicitationProtocol.js +1 -1
  12. package/dist/mcp/servers/agent/directToolsServer.js +0 -1
  13. package/dist/providers/azureOpenai.js +1 -1
  14. package/dist/providers/huggingFace.js +0 -1
  15. package/dist/providers/openaiCompatible.js +0 -1
  16. package/dist/sdk/toolRegistration.js +0 -1
  17. package/dist/server/openapi/generator.js +1 -1
  18. package/dist/server/routes/claudeProxyRoutes.js +45 -9
  19. package/dist/types/configTypes.js +0 -5
  20. package/dist/types/modelTypes.js +0 -1
  21. package/dist/types/tools.js +0 -1
  22. package/dist/types/typeAliases.js +0 -1
  23. package/dist/types/utilities.js +1 -1
  24. package/dist/types/workflowTypes.js +0 -1
  25. package/dist/utils/providerRetry.js +0 -1
  26. package/dist/utils/providerUtils.js +0 -1
  27. package/package.json +2 -2
  28. package/dist/client/adapters/providerImageAdapter.js +0 -588
  29. package/dist/client/adapters/tts/googleTTSHandler.js +0 -344
  30. package/dist/client/adapters/video/directorPipeline.js +0 -516
  31. package/dist/client/adapters/video/ffmpegAdapter.js +0 -206
  32. package/dist/client/adapters/video/frameExtractor.js +0 -143
  33. package/dist/client/adapters/video/vertexVideoHandler.js +0 -763
  34. package/dist/client/adapters/video/videoAnalyzer.js +0 -238
  35. package/dist/client/adapters/video/videoMerger.js +0 -171
  36. package/dist/client/agent/directTools.js +0 -840
  37. package/dist/client/auth/AuthProviderFactory.js +0 -111
  38. package/dist/client/auth/AuthProviderRegistry.js +0 -190
  39. package/dist/client/auth/RequestContext.js +0 -78
  40. package/dist/client/auth/accountPool.js +0 -178
  41. package/dist/client/auth/anthropicOAuth.js +0 -974
  42. package/dist/client/auth/authContext.js +0 -314
  43. package/dist/client/auth/errors.js +0 -39
  44. package/dist/client/auth/index.js +0 -61
  45. package/dist/client/auth/middleware/AuthMiddleware.js +0 -519
  46. package/dist/client/auth/middleware/rateLimitByUser.js +0 -554
  47. package/dist/client/auth/providers/BaseAuthProvider.js +0 -723
  48. package/dist/client/auth/providers/CognitoProvider.js +0 -304
  49. package/dist/client/auth/providers/KeycloakProvider.js +0 -393
  50. package/dist/client/auth/providers/auth0.js +0 -274
  51. package/dist/client/auth/providers/betterAuth.js +0 -182
  52. package/dist/client/auth/providers/clerk.js +0 -317
  53. package/dist/client/auth/providers/custom.js +0 -112
  54. package/dist/client/auth/providers/firebase.js +0 -226
  55. package/dist/client/auth/providers/jwt.js +0 -212
  56. package/dist/client/auth/providers/oauth2.js +0 -303
  57. package/dist/client/auth/providers/supabase.js +0 -259
  58. package/dist/client/auth/providers/workos.js +0 -284
  59. package/dist/client/auth/serverBridge.js +0 -25
  60. package/dist/client/auth/sessionManager.js +0 -437
  61. package/dist/client/auth/tokenStore.js +0 -799
  62. package/dist/client/client/aiSdkAdapter.js +0 -487
  63. package/dist/client/client/auth.js +0 -473
  64. package/dist/client/client/errors.js +0 -552
  65. package/dist/client/client/httpClient.js +0 -837
  66. package/dist/client/client/index.js +0 -172
  67. package/dist/client/client/interceptors.js +0 -601
  68. package/dist/client/client/sseClient.js +0 -545
  69. package/dist/client/client/streamingClient.js +0 -917
  70. package/dist/client/client/wsClient.js +0 -369
  71. package/dist/client/config/configManager.js +0 -303
  72. package/dist/client/config/conversationMemory.js +0 -86
  73. package/dist/client/config/taskClassificationConfig.js +0 -148
  74. package/dist/client/constants/contextWindows.js +0 -295
  75. package/dist/client/constants/enums.js +0 -853
  76. package/dist/client/constants/index.js +0 -207
  77. package/dist/client/constants/performance.js +0 -389
  78. package/dist/client/constants/retry.js +0 -266
  79. package/dist/client/constants/timeouts.js +0 -182
  80. package/dist/client/constants/tokens.js +0 -380
  81. package/dist/client/constants/videoErrors.js +0 -46
  82. package/dist/client/context/budgetChecker.js +0 -98
  83. package/dist/client/context/contextCompactor.js +0 -205
  84. package/dist/client/context/emergencyTruncation.js +0 -88
  85. package/dist/client/context/errorDetection.js +0 -171
  86. package/dist/client/context/errors.js +0 -21
  87. package/dist/client/context/fileTokenBudget.js +0 -127
  88. package/dist/client/context/prompts/summarizationPrompt.js +0 -117
  89. package/dist/client/context/stages/fileReadDeduplicator.js +0 -66
  90. package/dist/client/context/stages/slidingWindowTruncator.js +0 -190
  91. package/dist/client/context/stages/structuredSummarizer.js +0 -99
  92. package/dist/client/context/stages/toolOutputPruner.js +0 -52
  93. package/dist/client/context/summarizationEngine.js +0 -136
  94. package/dist/client/context/toolOutputLimits.js +0 -78
  95. package/dist/client/context/toolPairRepair.js +0 -66
  96. package/dist/client/core/analytics.js +0 -88
  97. package/dist/client/core/baseProvider.js +0 -1385
  98. package/dist/client/core/constants.js +0 -140
  99. package/dist/client/core/conversationMemoryFactory.js +0 -141
  100. package/dist/client/core/conversationMemoryInitializer.js +0 -128
  101. package/dist/client/core/conversationMemoryManager.js +0 -344
  102. package/dist/client/core/dynamicModels.js +0 -358
  103. package/dist/client/core/evaluation.js +0 -309
  104. package/dist/client/core/evaluationProviders.js +0 -248
  105. package/dist/client/core/factory.js +0 -412
  106. package/dist/client/core/infrastructure/baseError.js +0 -22
  107. package/dist/client/core/infrastructure/baseFactory.js +0 -54
  108. package/dist/client/core/infrastructure/baseRegistry.js +0 -53
  109. package/dist/client/core/infrastructure/index.js +0 -5
  110. package/dist/client/core/infrastructure/retry.js +0 -20
  111. package/dist/client/core/infrastructure/typedEventEmitter.js +0 -23
  112. package/dist/client/core/modelConfiguration.js +0 -851
  113. package/dist/client/core/modules/GenerationHandler.js +0 -588
  114. package/dist/client/core/modules/MessageBuilder.js +0 -273
  115. package/dist/client/core/modules/StreamHandler.js +0 -185
  116. package/dist/client/core/modules/TelemetryHandler.js +0 -203
  117. package/dist/client/core/modules/ToolsManager.js +0 -499
  118. package/dist/client/core/modules/Utilities.js +0 -331
  119. package/dist/client/core/redisConversationMemoryManager.js +0 -1435
  120. package/dist/client/core/streamAnalytics.js +0 -131
  121. package/dist/client/evaluation/contextBuilder.js +0 -134
  122. package/dist/client/evaluation/index.js +0 -61
  123. package/dist/client/evaluation/prompts.js +0 -73
  124. package/dist/client/evaluation/ragasEvaluator.js +0 -110
  125. package/dist/client/evaluation/retryManager.js +0 -78
  126. package/dist/client/evaluation/scoring.js +0 -61
  127. package/dist/client/factories/providerFactory.js +0 -166
  128. package/dist/client/factories/providerRegistry.js +0 -166
  129. package/dist/client/features/ppt/constants.js +0 -896
  130. package/dist/client/features/ppt/contentPlanner.js +0 -529
  131. package/dist/client/features/ppt/presentationOrchestrator.js +0 -236
  132. package/dist/client/features/ppt/slideGenerator.js +0 -532
  133. package/dist/client/features/ppt/slideRenderers.js +0 -2383
  134. package/dist/client/features/ppt/slideTypeInference.js +0 -405
  135. package/dist/client/features/ppt/types.js +0 -13
  136. package/dist/client/features/ppt/utils.js +0 -443
  137. package/dist/client/files/fileReferenceRegistry.js +0 -1543
  138. package/dist/client/files/fileTools.js +0 -450
  139. package/dist/client/files/streamingReader.js +0 -321
  140. package/dist/client/files/types.js +0 -23
  141. package/dist/client/hitl/hitlErrors.js +0 -54
  142. package/dist/client/hitl/hitlManager.js +0 -460
  143. package/dist/client/mcp/agentExposure.js +0 -356
  144. package/dist/client/mcp/auth/index.js +0 -11
  145. package/dist/client/mcp/auth/oauthClientProvider.js +0 -325
  146. package/dist/client/mcp/auth/tokenStorage.js +0 -134
  147. package/dist/client/mcp/batching/index.js +0 -10
  148. package/dist/client/mcp/batching/requestBatcher.js +0 -441
  149. package/dist/client/mcp/caching/index.js +0 -10
  150. package/dist/client/mcp/caching/toolCache.js +0 -433
  151. package/dist/client/mcp/elicitation/elicitationManager.js +0 -376
  152. package/dist/client/mcp/elicitation/index.js +0 -11
  153. package/dist/client/mcp/elicitation/types.js +0 -10
  154. package/dist/client/mcp/elicitationProtocol.js +0 -375
  155. package/dist/client/mcp/enhancedToolDiscovery.js +0 -481
  156. package/dist/client/mcp/externalServerManager.js +0 -1478
  157. package/dist/client/mcp/factory.js +0 -161
  158. package/dist/client/mcp/flexibleToolValidator.js +0 -161
  159. package/dist/client/mcp/httpRateLimiter.js +0 -391
  160. package/dist/client/mcp/httpRetryHandler.js +0 -178
  161. package/dist/client/mcp/index.js +0 -74
  162. package/dist/client/mcp/mcpCircuitBreaker.js +0 -427
  163. package/dist/client/mcp/mcpClientFactory.js +0 -708
  164. package/dist/client/mcp/mcpRegistryClient.js +0 -488
  165. package/dist/client/mcp/mcpServerBase.js +0 -373
  166. package/dist/client/mcp/multiServerManager.js +0 -579
  167. package/dist/client/mcp/registry.js +0 -158
  168. package/dist/client/mcp/routing/index.js +0 -10
  169. package/dist/client/mcp/routing/toolRouter.js +0 -416
  170. package/dist/client/mcp/serverCapabilities.js +0 -502
  171. package/dist/client/mcp/servers/agent/directToolsServer.js +0 -150
  172. package/dist/client/mcp/toolAnnotations.js +0 -239
  173. package/dist/client/mcp/toolConverter.js +0 -258
  174. package/dist/client/mcp/toolDiscoveryService.js +0 -798
  175. package/dist/client/mcp/toolIntegration.js +0 -334
  176. package/dist/client/mcp/toolRegistry.js +0 -729
  177. package/dist/client/memory/hippocampusInitializer.js +0 -19
  178. package/dist/client/memory/memoryRetrievalTools.js +0 -166
  179. package/dist/client/middleware/builtin/analytics.js +0 -132
  180. package/dist/client/middleware/builtin/autoEvaluation.js +0 -203
  181. package/dist/client/middleware/builtin/guardrails.js +0 -109
  182. package/dist/client/middleware/builtin/lifecycle.js +0 -168
  183. package/dist/client/middleware/factory.js +0 -327
  184. package/dist/client/middleware/registry.js +0 -295
  185. package/dist/client/middleware/utils/guardrailsUtils.js +0 -396
  186. package/dist/client/models/anthropicModels.js +0 -527
  187. package/dist/client/neurolink.js +0 -8233
  188. package/dist/client/observability/exporterRegistry.js +0 -413
  189. package/dist/client/observability/exporters/arizeExporter.js +0 -138
  190. package/dist/client/observability/exporters/baseExporter.js +0 -190
  191. package/dist/client/observability/exporters/braintrustExporter.js +0 -154
  192. package/dist/client/observability/exporters/datadogExporter.js +0 -196
  193. package/dist/client/observability/exporters/laminarExporter.js +0 -302
  194. package/dist/client/observability/exporters/langfuseExporter.js +0 -209
  195. package/dist/client/observability/exporters/langsmithExporter.js +0 -143
  196. package/dist/client/observability/exporters/otelExporter.js +0 -164
  197. package/dist/client/observability/exporters/posthogExporter.js +0 -287
  198. package/dist/client/observability/exporters/sentryExporter.js +0 -165
  199. package/dist/client/observability/index.js +0 -31
  200. package/dist/client/observability/metricsAggregator.js +0 -556
  201. package/dist/client/observability/otelBridge.js +0 -131
  202. package/dist/client/observability/retryPolicy.js +0 -383
  203. package/dist/client/observability/sampling/samplers.js +0 -216
  204. package/dist/client/observability/spanProcessor.js +0 -303
  205. package/dist/client/observability/tokenTracker.js +0 -413
  206. package/dist/client/observability/types/exporterTypes.js +0 -5
  207. package/dist/client/observability/types/index.js +0 -4
  208. package/dist/client/observability/types/spanTypes.js +0 -92
  209. package/dist/client/observability/utils/safeMetadata.js +0 -25
  210. package/dist/client/observability/utils/spanSerializer.js +0 -292
  211. package/dist/client/processors/archive/ArchiveProcessor.js +0 -1308
  212. package/dist/client/processors/base/BaseFileProcessor.js +0 -614
  213. package/dist/client/processors/base/types.js +0 -82
  214. package/dist/client/processors/config/fileTypes.js +0 -520
  215. package/dist/client/processors/config/index.js +0 -92
  216. package/dist/client/processors/config/languageMap.js +0 -410
  217. package/dist/client/processors/config/mimeTypes.js +0 -363
  218. package/dist/client/processors/config/sizeLimits.js +0 -258
  219. package/dist/client/processors/document/ExcelProcessor.js +0 -590
  220. package/dist/client/processors/document/OpenDocumentProcessor.js +0 -212
  221. package/dist/client/processors/document/PptxProcessor.js +0 -157
  222. package/dist/client/processors/document/RtfProcessor.js +0 -361
  223. package/dist/client/processors/document/WordProcessor.js +0 -353
  224. package/dist/client/processors/errors/FileErrorCode.js +0 -255
  225. package/dist/client/processors/errors/errorHelpers.js +0 -386
  226. package/dist/client/processors/errors/errorSerializer.js +0 -507
  227. package/dist/client/processors/errors/index.js +0 -49
  228. package/dist/client/processors/markup/SvgProcessor.js +0 -240
  229. package/dist/client/processors/media/AudioProcessor.js +0 -707
  230. package/dist/client/processors/media/VideoProcessor.js +0 -1045
  231. package/dist/client/providers/amazonBedrock.js +0 -1512
  232. package/dist/client/providers/amazonSagemaker.js +0 -162
  233. package/dist/client/providers/anthropic.js +0 -831
  234. package/dist/client/providers/azureOpenai.js +0 -143
  235. package/dist/client/providers/googleAiStudio.js +0 -1200
  236. package/dist/client/providers/googleNativeGemini3.js +0 -543
  237. package/dist/client/providers/googleVertex.js +0 -2936
  238. package/dist/client/providers/huggingFace.js +0 -315
  239. package/dist/client/providers/litellm.js +0 -488
  240. package/dist/client/providers/mistral.js +0 -157
  241. package/dist/client/providers/ollama.js +0 -1579
  242. package/dist/client/providers/openAI.js +0 -627
  243. package/dist/client/providers/openRouter.js +0 -543
  244. package/dist/client/providers/openaiCompatible.js +0 -290
  245. package/dist/client/providers/providerTypeUtils.js +0 -46
  246. package/dist/client/providers/sagemaker/adaptive-semaphore.js +0 -215
  247. package/dist/client/providers/sagemaker/client.js +0 -472
  248. package/dist/client/providers/sagemaker/config.js +0 -317
  249. package/dist/client/providers/sagemaker/detection.js +0 -606
  250. package/dist/client/providers/sagemaker/error-constants.js +0 -227
  251. package/dist/client/providers/sagemaker/errors.js +0 -299
  252. package/dist/client/providers/sagemaker/language-model.js +0 -775
  253. package/dist/client/providers/sagemaker/parsers.js +0 -634
  254. package/dist/client/providers/sagemaker/streaming.js +0 -331
  255. package/dist/client/providers/sagemaker/structured-parser.js +0 -625
  256. package/dist/client/proxy/accountQuota.js +0 -162
  257. package/dist/client/proxy/claudeFormat.js +0 -595
  258. package/dist/client/proxy/modelRouter.js +0 -29
  259. package/dist/client/proxy/oauthFetch.js +0 -367
  260. package/dist/client/proxy/proxyFetch.js +0 -586
  261. package/dist/client/proxy/requestLogger.js +0 -207
  262. package/dist/client/proxy/tokenRefresh.js +0 -124
  263. package/dist/client/proxy/usageStats.js +0 -74
  264. package/dist/client/proxy/utils/noProxyUtils.js +0 -149
  265. package/dist/client/rag/ChunkerFactory.js +0 -320
  266. package/dist/client/rag/ChunkerRegistry.js +0 -421
  267. package/dist/client/rag/chunkers/BaseChunker.js +0 -143
  268. package/dist/client/rag/chunkers/CharacterChunker.js +0 -28
  269. package/dist/client/rag/chunkers/HTMLChunker.js +0 -38
  270. package/dist/client/rag/chunkers/JSONChunker.js +0 -68
  271. package/dist/client/rag/chunkers/LaTeXChunker.js +0 -63
  272. package/dist/client/rag/chunkers/MarkdownChunker.js +0 -306
  273. package/dist/client/rag/chunkers/RecursiveChunker.js +0 -139
  274. package/dist/client/rag/chunkers/SemanticMarkdownChunker.js +0 -138
  275. package/dist/client/rag/chunkers/SentenceChunker.js +0 -66
  276. package/dist/client/rag/chunkers/TokenChunker.js +0 -61
  277. package/dist/client/rag/chunkers/index.js +0 -15
  278. package/dist/client/rag/chunking/characterChunker.js +0 -142
  279. package/dist/client/rag/chunking/chunkerRegistry.js +0 -194
  280. package/dist/client/rag/chunking/htmlChunker.js +0 -247
  281. package/dist/client/rag/chunking/index.js +0 -17
  282. package/dist/client/rag/chunking/jsonChunker.js +0 -281
  283. package/dist/client/rag/chunking/latexChunker.js +0 -251
  284. package/dist/client/rag/chunking/markdownChunker.js +0 -373
  285. package/dist/client/rag/chunking/recursiveChunker.js +0 -148
  286. package/dist/client/rag/chunking/semanticChunker.js +0 -306
  287. package/dist/client/rag/chunking/sentenceChunker.js +0 -230
  288. package/dist/client/rag/chunking/tokenChunker.js +0 -183
  289. package/dist/client/rag/document/MDocument.js +0 -392
  290. package/dist/client/rag/document/index.js +0 -5
  291. package/dist/client/rag/document/loaders.js +0 -500
  292. package/dist/client/rag/errors/RAGError.js +0 -274
  293. package/dist/client/rag/errors/index.js +0 -6
  294. package/dist/client/rag/graphRag/graphRAG.js +0 -401
  295. package/dist/client/rag/graphRag/index.js +0 -4
  296. package/dist/client/rag/index.js +0 -141
  297. package/dist/client/rag/metadata/MetadataExtractorFactory.js +0 -418
  298. package/dist/client/rag/metadata/MetadataExtractorRegistry.js +0 -362
  299. package/dist/client/rag/metadata/index.js +0 -9
  300. package/dist/client/rag/metadata/metadataExtractor.js +0 -280
  301. package/dist/client/rag/pipeline/RAGPipeline.js +0 -436
  302. package/dist/client/rag/pipeline/contextAssembly.js +0 -341
  303. package/dist/client/rag/pipeline/index.js +0 -5
  304. package/dist/client/rag/ragIntegration.js +0 -321
  305. package/dist/client/rag/reranker/RerankerFactory.js +0 -430
  306. package/dist/client/rag/reranker/RerankerRegistry.js +0 -402
  307. package/dist/client/rag/reranker/index.js +0 -9
  308. package/dist/client/rag/reranker/reranker.js +0 -277
  309. package/dist/client/rag/resilience/CircuitBreaker.js +0 -431
  310. package/dist/client/rag/resilience/RetryHandler.js +0 -304
  311. package/dist/client/rag/resilience/index.js +0 -7
  312. package/dist/client/rag/retrieval/hybridSearch.js +0 -335
  313. package/dist/client/rag/retrieval/index.js +0 -5
  314. package/dist/client/rag/retrieval/vectorQueryTool.js +0 -307
  315. package/dist/client/rag/types.js +0 -8
  316. package/dist/client/sdk/toolRegistration.js +0 -377
  317. package/dist/client/server/abstract/baseServerAdapter.js +0 -575
  318. package/dist/client/server/adapters/expressAdapter.js +0 -486
  319. package/dist/client/server/adapters/fastifyAdapter.js +0 -472
  320. package/dist/client/server/adapters/honoAdapter.js +0 -632
  321. package/dist/client/server/adapters/koaAdapter.js +0 -510
  322. package/dist/client/server/errors.js +0 -486
  323. package/dist/client/server/factory/serverAdapterFactory.js +0 -160
  324. package/dist/client/server/index.js +0 -108
  325. package/dist/client/server/middleware/abortSignal.js +0 -111
  326. package/dist/client/server/middleware/auth.js +0 -388
  327. package/dist/client/server/middleware/cache.js +0 -359
  328. package/dist/client/server/middleware/common.js +0 -281
  329. package/dist/client/server/middleware/deprecation.js +0 -190
  330. package/dist/client/server/middleware/mcpBodyAttachment.js +0 -63
  331. package/dist/client/server/middleware/rateLimit.js +0 -227
  332. package/dist/client/server/middleware/validation.js +0 -388
  333. package/dist/client/server/openapi/generator.js +0 -398
  334. package/dist/client/server/openapi/index.js +0 -36
  335. package/dist/client/server/openapi/schemas.js +0 -695
  336. package/dist/client/server/openapi/templates.js +0 -374
  337. package/dist/client/server/routes/agentRoutes.js +0 -189
  338. package/dist/client/server/routes/claudeProxyRoutes.js +0 -1600
  339. package/dist/client/server/routes/healthRoutes.js +0 -187
  340. package/dist/client/server/routes/index.js +0 -57
  341. package/dist/client/server/routes/mcpRoutes.js +0 -342
  342. package/dist/client/server/routes/memoryRoutes.js +0 -350
  343. package/dist/client/server/routes/openApiRoutes.js +0 -126
  344. package/dist/client/server/routes/toolRoutes.js +0 -199
  345. package/dist/client/server/streaming/dataStream.js +0 -486
  346. package/dist/client/server/streaming/index.js +0 -11
  347. package/dist/client/server/types.js +0 -67
  348. package/dist/client/server/utils/redaction.js +0 -334
  349. package/dist/client/server/utils/validation.js +0 -243
  350. package/dist/client/server/websocket/WebSocketHandler.js +0 -383
  351. package/dist/client/server/websocket/index.js +0 -4
  352. package/dist/client/services/server/ai/observability/instrumentation.js +0 -808
  353. package/dist/client/telemetry/attributes.js +0 -100
  354. package/dist/client/telemetry/index.js +0 -26
  355. package/dist/client/telemetry/telemetryService.js +0 -308
  356. package/dist/client/telemetry/tracers.js +0 -17
  357. package/dist/client/telemetry/withSpan.js +0 -34
  358. package/dist/client/types/actionTypes.js +0 -6
  359. package/dist/client/types/analytics.js +0 -5
  360. package/dist/client/types/authTypes.js +0 -9
  361. package/dist/client/types/circuitBreakerErrors.js +0 -34
  362. package/dist/client/types/cli.js +0 -21
  363. package/dist/client/types/clientTypes.js +0 -10
  364. package/dist/client/types/common.js +0 -51
  365. package/dist/client/types/configTypes.js +0 -49
  366. package/dist/client/types/content.js +0 -19
  367. package/dist/client/types/contextTypes.js +0 -400
  368. package/dist/client/types/conversation.js +0 -47
  369. package/dist/client/types/conversationMemoryInterface.js +0 -6
  370. package/dist/client/types/domainTypes.js +0 -5
  371. package/dist/client/types/errors.js +0 -167
  372. package/dist/client/types/evaluation.js +0 -5
  373. package/dist/client/types/evaluationProviders.js +0 -5
  374. package/dist/client/types/evaluationTypes.js +0 -1
  375. package/dist/client/types/externalMcp.js +0 -6
  376. package/dist/client/types/fileReferenceTypes.js +0 -8
  377. package/dist/client/types/fileTypes.js +0 -4
  378. package/dist/client/types/generateTypes.js +0 -1
  379. package/dist/client/types/guardrails.js +0 -1
  380. package/dist/client/types/hitlTypes.js +0 -8
  381. package/dist/client/types/index.js +0 -57
  382. package/dist/client/types/mcpTypes.js +0 -5
  383. package/dist/client/types/middlewareTypes.js +0 -1
  384. package/dist/client/types/modelTypes.js +0 -30
  385. package/dist/client/types/multimodal.js +0 -135
  386. package/dist/client/types/observability.js +0 -6
  387. package/dist/client/types/pptTypes.js +0 -82
  388. package/dist/client/types/providers.js +0 -111
  389. package/dist/client/types/proxyTypes.js +0 -16
  390. package/dist/client/types/ragTypes.js +0 -7
  391. package/dist/client/types/sdkTypes.js +0 -8
  392. package/dist/client/types/serviceTypes.js +0 -5
  393. package/dist/client/types/streamTypes.js +0 -1
  394. package/dist/client/types/subscriptionTypes.js +0 -9
  395. package/dist/client/types/taskClassificationTypes.js +0 -5
  396. package/dist/client/types/tools.js +0 -24
  397. package/dist/client/types/ttsTypes.js +0 -57
  398. package/dist/client/types/typeAliases.js +0 -48
  399. package/dist/client/types/utilities.js +0 -4
  400. package/dist/client/types/workflowTypes.js +0 -30
  401. package/dist/client/utils/async/withTimeout.js +0 -98
  402. package/dist/client/utils/asyncMutex.js +0 -60
  403. package/dist/client/utils/conversationMemory.js +0 -431
  404. package/dist/client/utils/csvProcessor.js +0 -846
  405. package/dist/client/utils/errorHandling.js +0 -936
  406. package/dist/client/utils/evaluationUtils.js +0 -131
  407. package/dist/client/utils/factoryProcessing.js +0 -589
  408. package/dist/client/utils/fileDetector.js +0 -2161
  409. package/dist/client/utils/imageCache.js +0 -376
  410. package/dist/client/utils/imageProcessor.js +0 -704
  411. package/dist/client/utils/logger.js +0 -491
  412. package/dist/client/utils/mcpDefaults.js +0 -134
  413. package/dist/client/utils/messageBuilder.js +0 -1653
  414. package/dist/client/utils/modelAliasResolver.js +0 -54
  415. package/dist/client/utils/modelDetection.js +0 -80
  416. package/dist/client/utils/modelRouter.js +0 -292
  417. package/dist/client/utils/multimodalOptionsBuilder.js +0 -65
  418. package/dist/client/utils/observabilityHelpers.js +0 -47
  419. package/dist/client/utils/parameterValidation.js +0 -966
  420. package/dist/client/utils/pdfProcessor.js +0 -410
  421. package/dist/client/utils/performance.js +0 -222
  422. package/dist/client/utils/pricing.js +0 -340
  423. package/dist/client/utils/promptRedaction.js +0 -62
  424. package/dist/client/utils/providerConfig.js +0 -1009
  425. package/dist/client/utils/providerHealth.js +0 -1237
  426. package/dist/client/utils/providerRetry.js +0 -112
  427. package/dist/client/utils/providerUtils.js +0 -434
  428. package/dist/client/utils/rateLimiter.js +0 -200
  429. package/dist/client/utils/redis.js +0 -368
  430. package/dist/client/utils/retryHandler.js +0 -269
  431. package/dist/client/utils/retryability.js +0 -22
  432. package/dist/client/utils/sanitizers/svg.js +0 -481
  433. package/dist/client/utils/schemaConversion.js +0 -255
  434. package/dist/client/utils/taskClassificationUtils.js +0 -149
  435. package/dist/client/utils/taskClassifier.js +0 -94
  436. package/dist/client/utils/thinkingConfig.js +0 -104
  437. package/dist/client/utils/timeout.js +0 -359
  438. package/dist/client/utils/tokenEstimation.js +0 -142
  439. package/dist/client/utils/tokenLimits.js +0 -125
  440. package/dist/client/utils/tokenUtils.js +0 -239
  441. package/dist/client/utils/toolUtils.js +0 -75
  442. package/dist/client/utils/transformationUtils.js +0 -554
  443. package/dist/client/utils/ttsProcessor.js +0 -286
  444. package/dist/client/utils/typeUtils.js +0 -97
  445. package/dist/client/utils/videoAnalysisProcessor.js +0 -67
  446. package/dist/client/workflow/config.js +0 -398
  447. package/dist/client/workflow/core/ensembleExecutor.js +0 -407
  448. package/dist/client/workflow/core/judgeScorer.js +0 -544
  449. package/dist/client/workflow/core/responseConditioner.js +0 -225
  450. package/dist/client/workflow/core/types/conditionerTypes.js +0 -7
  451. package/dist/client/workflow/core/types/ensembleTypes.js +0 -7
  452. package/dist/client/workflow/core/types/index.js +0 -7
  453. package/dist/client/workflow/core/types/judgeTypes.js +0 -7
  454. package/dist/client/workflow/core/types/layerTypes.js +0 -7
  455. package/dist/client/workflow/core/types/registryTypes.js +0 -7
  456. package/dist/client/workflow/core/workflowRegistry.js +0 -304
  457. package/dist/client/workflow/core/workflowRunner.js +0 -586
  458. package/dist/client/workflow/index.js +0 -50
  459. package/dist/client/workflow/types.js +0 -9
  460. package/dist/client/workflow/utils/types/index.js +0 -7
  461. package/dist/client/workflow/utils/workflowMetrics.js +0 -311
  462. package/dist/client/workflow/utils/workflowValidation.js +0 -420
  463. package/dist/client/workflow/workflows/adaptiveWorkflow.js +0 -366
  464. package/dist/client/workflow/workflows/consensusWorkflow.js +0 -192
  465. package/dist/client/workflow/workflows/fallbackWorkflow.js +0 -225
  466. package/dist/client/workflow/workflows/multiJudgeWorkflow.js +0 -351
  467. /package/dist/client/{client/reactHooks.js → reactHooks.js} +0 -0
@@ -1,1385 +0,0 @@
1
- import { context, SpanKind, SpanStatusCode, trace } from "@opentelemetry/api";
2
- import { generateText } from "ai";
3
- import { directAgentTools } from "../agent/directTools.js";
4
- import { IMAGE_GENERATION_MODELS } from "../core/constants.js";
5
- import { MiddlewareFactory } from "../middleware/factory.js";
6
- import { SpanStatus, SpanType } from "../observability/types/spanTypes.js";
7
- import { SpanSerializer } from "../observability/utils/spanSerializer.js";
8
- import { ATTR, tracers } from "../telemetry/index.js";
9
- import { calculateCost } from "../utils/pricing.js";
10
- import { isAbortError } from "../utils/errorHandling.js";
11
- import { logger } from "../utils/logger.js";
12
- import { composeAbortSignals, createTimeoutController, TimeoutError, } from "../utils/timeout.js";
13
- import { shouldDisableBuiltinTools } from "../utils/toolUtils.js";
14
- import { getKeyCount, getKeysAsString } from "../utils/transformationUtils.js";
15
- import { TTSProcessor } from "../utils/ttsProcessor.js";
16
- import { executeVideoAnalysis, hasVideoFrames, } from "../utils/videoAnalysisProcessor.js";
17
- import { GenerationHandler } from "./modules/GenerationHandler.js";
18
- // Import modules for composition
19
- import { MessageBuilder } from "./modules/MessageBuilder.js";
20
- import { StreamHandler } from "./modules/StreamHandler.js";
21
- import { TelemetryHandler } from "./modules/TelemetryHandler.js";
22
- import { ToolsManager } from "./modules/ToolsManager.js";
23
- import { Utilities } from "./modules/Utilities.js";
24
- /**
25
- * Abstract base class for all AI providers
26
- * Tools are integrated as first-class citizens - always available by default
27
- */
28
- export class BaseProvider {
29
- modelName;
30
- providerName;
31
- defaultTimeout = 30000; // 30 seconds
32
- middlewareOptions; // TODO: Implement global level middlewares that can be used
33
- // Tools are conditionally included based on centralized configuration
34
- directTools = shouldDisableBuiltinTools()
35
- ? {}
36
- : directAgentTools;
37
- mcpTools; // MCP tools loaded dynamically when available
38
- customTools; // Custom tools from registerTool()
39
- toolExecutor; // Tool executor from setupToolExecutor
40
- sessionId;
41
- userId;
42
- neurolink; // Reference to actual NeuroLink instance for MCP tools
43
- /** @internal Trace context propagated from NeuroLink SDK for span hierarchy */
44
- _traceContext = null;
45
- setTraceContext(ctx) {
46
- this._traceContext = ctx;
47
- }
48
- // Composition modules - Single Responsibility Principle
49
- messageBuilder;
50
- streamHandler;
51
- generationHandler;
52
- telemetryHandler;
53
- utilities;
54
- toolsManager;
55
- constructor(modelName, providerName, neurolink, middleware) {
56
- this.modelName = modelName || this.getDefaultModel();
57
- this.providerName = providerName || this.getProviderName();
58
- this.neurolink = neurolink;
59
- this.middlewareOptions = middleware;
60
- // Initialize composition modules
61
- this.messageBuilder = new MessageBuilder(this.providerName, this.modelName);
62
- this.streamHandler = new StreamHandler(this.providerName, this.modelName);
63
- this.telemetryHandler = new TelemetryHandler(this.providerName, this.modelName, this.neurolink);
64
- this.generationHandler = new GenerationHandler(this.providerName, this.modelName, () => this.supportsTools(), (options, type) => this.telemetryHandler.getTelemetryConfig(options, type), (toolCalls, toolResults, options, timestamp) => this.handleToolExecutionStorage(toolCalls, toolResults, options, timestamp));
65
- this.utilities = new Utilities(this.providerName, this.modelName, this.defaultTimeout, this.middlewareOptions);
66
- this.toolsManager = new ToolsManager(this.providerName, this.directTools, this.neurolink, {
67
- isZodSchema: (schema) => this.isZodSchema(schema),
68
- convertToolResult: (result) => this.convertToolResult(result),
69
- createPermissiveZodSchema: () => this.createPermissiveZodSchema(),
70
- fixSchemaForOpenAIStrictMode: (schema) => this.fixSchemaForOpenAIStrictMode(schema),
71
- });
72
- }
73
- /**
74
- * Check if this provider supports tool/function calling
75
- * Override in subclasses to disable tools for specific providers or models
76
- * @returns true by default, providers can override to return false
77
- */
78
- supportsTools() {
79
- return true;
80
- }
81
- // ===================
82
- // PUBLIC API METHODS
83
- // ===================
84
- /**
85
- * Primary streaming method - implements AIProvider interface
86
- * When tools are involved, falls back to generate() with synthetic streaming
87
- */
88
- async stream(optionsOrPrompt, analysisSchema) {
89
- let options = this.normalizeStreamOptions(optionsOrPrompt);
90
- // Observability: create metrics span for provider.stream
91
- const metricsSpan = SpanSerializer.createSpan(SpanType.MODEL_GENERATION, "provider.stream", {
92
- "ai.provider": this.providerName || "unknown",
93
- "ai.model": this.modelName || options.model || "unknown",
94
- "ai.temperature": options.temperature,
95
- "ai.max_tokens": options.maxTokens,
96
- }, this._traceContext?.parentSpanId, this._traceContext?.traceId);
97
- let metricsSpanRecorded = false;
98
- // OTEL span for provider-level stream tracing
99
- const otelStreamSpan = tracers.provider.startSpan("neurolink.provider.stream", {
100
- kind: SpanKind.CLIENT,
101
- attributes: {
102
- [ATTR.GEN_AI_SYSTEM]: this.providerName || "unknown",
103
- [ATTR.GEN_AI_MODEL]: this.modelName || options.model || "unknown",
104
- [ATTR.GEN_AI_OPERATION]: "stream",
105
- [ATTR.NL_PROVIDER]: this.providerName || "unknown",
106
- },
107
- });
108
- try {
109
- logger.info(`Starting stream`, {
110
- provider: this.providerName,
111
- hasTools: !options.disableTools && this.supportsTools(),
112
- disableTools: !!options.disableTools,
113
- supportsTools: this.supportsTools(),
114
- inputLength: options.input?.text?.length || 0,
115
- maxTokens: options.maxTokens,
116
- temperature: options.temperature,
117
- timestamp: Date.now(),
118
- });
119
- // ===== EARLY MULTIMODAL DETECTION =====
120
- const hasFileInput = !!options.input?.files?.length || !!options.input?.videoFiles?.length;
121
- if (hasFileInput) {
122
- // ===== VIDEO ANALYSIS DETECTION =====
123
- // Check if video frames are present and handle with fake streaming
124
- const messages = await this.buildMessagesForStream(options);
125
- if (hasVideoFrames(messages)) {
126
- logger.info(`Video frames detected in stream, using fake streaming for video analysis`, {
127
- provider: this.providerName,
128
- model: this.modelName,
129
- });
130
- return await this.executeFakeStreaming(options, analysisSchema);
131
- }
132
- }
133
- // CRITICAL: Image generation models don't support real streaming
134
- // Force fake streaming for image models to ensure image output is yielded
135
- const isImageModel = IMAGE_GENERATION_MODELS.some((m) => this.modelName.includes(m));
136
- if (isImageModel) {
137
- logger.info(`Image model detected, forcing fake streaming`, {
138
- provider: this.providerName,
139
- model: this.modelName,
140
- reason: "Image generation requires fake streaming to yield image output",
141
- });
142
- // Skip real streaming, go directly to fake streaming
143
- return await this.executeFakeStreaming(options, analysisSchema);
144
- }
145
- // Central tool merge: Pre-merge base tools (MCP/built-in) with user-provided
146
- // tools (e.g. RAG tools) into options.tools. This way, every provider's
147
- // executeStream() can simply use options.tools (or getAllTools() + options.tools)
148
- // and get the complete tool set without needing per-provider merge logic.
149
- if (!options.disableTools && this.supportsTools()) {
150
- const mergedTools = await this.getToolsForStream(options);
151
- options = { ...options, tools: mergedTools };
152
- }
153
- else {
154
- options = { ...options, tools: {} };
155
- }
156
- // CRITICAL FIX: Always prefer real streaming over fake streaming
157
- // Try real streaming first, use fake streaming only as fallback
158
- try {
159
- logger.debug(`Attempting real streaming`, {
160
- provider: this.providerName,
161
- timestamp: Date.now(),
162
- });
163
- const realStreamResult = await this.executeStream(options, analysisSchema);
164
- logger.info(`Real streaming succeeded`, {
165
- provider: this.providerName,
166
- timestamp: Date.now(),
167
- });
168
- // If real streaming succeeds, return it (with tools support via Vercel AI SDK)
169
- return realStreamResult;
170
- }
171
- catch (realStreamError) {
172
- logger.warn(`Real streaming failed for ${this.providerName}, falling back to fake streaming:`, {
173
- error: realStreamError instanceof Error
174
- ? realStreamError.message
175
- : String(realStreamError),
176
- timestamp: Date.now(),
177
- });
178
- // Fallback to fake streaming only if real streaming fails AND tools are enabled
179
- if (!options.disableTools && this.supportsTools()) {
180
- return await this.executeFakeStreaming(options, analysisSchema);
181
- }
182
- else {
183
- // If real streaming failed and no tools are enabled, re-throw the original error
184
- logger.error(`Real streaming failed for ${this.providerName}:`, realStreamError);
185
- throw this.handleProviderError(realStreamError);
186
- }
187
- }
188
- }
189
- catch (error) {
190
- // Observability: record failed stream span
191
- metricsSpanRecorded = true;
192
- const _endedStreamSpan = SpanSerializer.endSpan(metricsSpan, SpanStatus.ERROR, error instanceof Error ? error.message : String(error));
193
- // Note: Do NOT record to getMetricsAggregator() here — neurolink.ts
194
- // stream:complete listener handles authoritative metrics to avoid double-counting.
195
- otelStreamSpan.setStatus({
196
- code: SpanStatusCode.ERROR,
197
- message: error instanceof Error ? error.message : String(error),
198
- });
199
- otelStreamSpan.end();
200
- throw error;
201
- }
202
- finally {
203
- // Observability: record successful stream span (only if not already ended via error path)
204
- if (!metricsSpanRecorded) {
205
- const _endedStreamSpan = SpanSerializer.endSpan(metricsSpan, SpanStatus.OK);
206
- // Note: Do NOT record to getMetricsAggregator() here — neurolink.ts
207
- // stream:complete listener handles authoritative metrics to avoid double-counting.
208
- }
209
- // End OTEL span on success (only if not already ended via error path)
210
- if (otelStreamSpan.isRecording()) {
211
- otelStreamSpan.setStatus({ code: SpanStatusCode.OK });
212
- otelStreamSpan.end();
213
- }
214
- }
215
- }
216
- /**
217
- * Execute fake streaming - extracted method for reusability
218
- */
219
- async executeFakeStreaming(options, analysisSchema) {
220
- try {
221
- logger.info(`Starting fake streaming with tools`, {
222
- provider: this.providerName,
223
- supportsTools: this.supportsTools(),
224
- timestamp: Date.now(),
225
- });
226
- // Convert stream options to text generation options
227
- const textOptions = {
228
- prompt: options.input?.text || "",
229
- input: options.input,
230
- systemPrompt: options.systemPrompt,
231
- temperature: options.temperature,
232
- maxTokens: options.maxTokens,
233
- tools: options.tools, // 🔧 FIX: Pass user-provided tools (including RAG tools) to generation pipeline
234
- disableTools: !!options.disableTools,
235
- maxSteps: options.maxSteps || 5,
236
- provider: options.provider,
237
- model: options.model,
238
- region: options.region, // Pass region for Vertex AI
239
- // 🔧 FIX: Include analytics and evaluation options from stream options
240
- enableAnalytics: options.enableAnalytics,
241
- enableEvaluation: options.enableEvaluation,
242
- evaluationDomain: options.evaluationDomain,
243
- toolUsageContext: options.toolUsageContext,
244
- context: options.context,
245
- csvOptions: options.csvOptions,
246
- // Forward abort, tool filtering, and timeout options to prevent
247
- // silent bypass when falling back from real streaming to fake streaming
248
- abortSignal: options.abortSignal,
249
- toolFilter: options.toolFilter,
250
- excludeTools: options.excludeTools,
251
- skipToolPromptInjection: options.skipToolPromptInjection,
252
- timeout: options.timeout,
253
- };
254
- logger.debug(`Calling generate for fake streaming`, {
255
- provider: this.providerName,
256
- maxSteps: textOptions.maxSteps,
257
- disableTools: textOptions.disableTools,
258
- timestamp: Date.now(),
259
- });
260
- const result = await this.generate(textOptions, analysisSchema);
261
- logger.info(`Generate completed for fake streaming`, {
262
- provider: this.providerName,
263
- hasContent: !!result?.content,
264
- contentLength: result?.content?.length || 0,
265
- toolsUsed: result?.toolsUsed?.length || 0,
266
- hasImageOutput: !!result?.imageOutput,
267
- timestamp: Date.now(),
268
- });
269
- // Create a synthetic stream from the generate result that simulates progressive delivery
270
- return {
271
- stream: (async function* () {
272
- if (result?.content) {
273
- // Split content into words for more natural streaming
274
- const words = result.content.split(/(\s+)/); // Keep whitespace
275
- let buffer = "";
276
- for (let i = 0; i < words.length; i++) {
277
- buffer += words[i];
278
- // Yield chunks of roughly 5-10 words or at punctuation
279
- const shouldYield = i === words.length - 1 || // Last word
280
- buffer.length > 50 || // Buffer getting long
281
- /[.!?;,]\s*$/.test(buffer); // End of sentence/clause
282
- if (shouldYield && buffer.trim()) {
283
- yield { content: buffer };
284
- buffer = "";
285
- // Small delay to simulate streaming (1-10ms)
286
- await new Promise((resolve) => {
287
- setTimeout(resolve, Math.random() * 9 + 1);
288
- });
289
- }
290
- }
291
- // Yield all remaining content
292
- if (buffer.trim()) {
293
- yield { content: buffer };
294
- }
295
- }
296
- // 🔧 CRITICAL FIX: Yield image output if present
297
- if (result?.imageOutput) {
298
- yield {
299
- type: "image",
300
- imageOutput: result.imageOutput,
301
- };
302
- }
303
- })(),
304
- usage: result?.usage,
305
- provider: result?.provider,
306
- model: result?.model,
307
- toolCalls: result?.toolCalls?.map((call) => ({
308
- toolName: call.toolName,
309
- parameters: call.args,
310
- id: call.toolCallId,
311
- })),
312
- toolResults: result?.toolResults
313
- ? result.toolResults.map((tr) => ({
314
- toolName: tr.toolName || "unknown",
315
- status: (tr.status === "error"
316
- ? "failure"
317
- : "success"),
318
- result: tr.result,
319
- error: tr.error,
320
- }))
321
- : undefined,
322
- // 🔧 FIX: Include analytics and evaluation from generate result
323
- analytics: result?.analytics,
324
- evaluation: result?.evaluation,
325
- };
326
- }
327
- catch (error) {
328
- logger.error(`Fake streaming fallback failed for ${this.providerName}:`, error);
329
- throw this.handleProviderError(error);
330
- }
331
- }
332
- /**
333
- * Apply per-call tool filtering (whitelist/blacklist) to a tools record.
334
- * If toolFilter is set, only tools whose names are in the list are kept.
335
- * If excludeTools is set, matching tools are removed. excludeTools is applied after toolFilter.
336
- */
337
- applyToolFiltering(tools, options) {
338
- if ((!options.toolFilter || options.toolFilter.length === 0) &&
339
- (!options.excludeTools || options.excludeTools.length === 0)) {
340
- return tools;
341
- }
342
- const beforeCount = Object.keys(tools).length;
343
- let filtered = { ...tools };
344
- if (options.toolFilter && options.toolFilter.length > 0) {
345
- const allowSet = new Set(options.toolFilter);
346
- const result = {};
347
- for (const [name, tool] of Object.entries(filtered)) {
348
- if (allowSet.has(name)) {
349
- result[name] = tool;
350
- }
351
- }
352
- filtered = result;
353
- }
354
- if (options.excludeTools && options.excludeTools.length > 0) {
355
- const denySet = new Set(options.excludeTools);
356
- for (const name of Object.keys(filtered)) {
357
- if (denySet.has(name)) {
358
- delete filtered[name];
359
- }
360
- }
361
- }
362
- const afterCount = Object.keys(filtered).length;
363
- if (beforeCount !== afterCount) {
364
- logger.debug(`Tool filtering applied`, {
365
- provider: this.providerName,
366
- beforeCount,
367
- afterCount,
368
- toolFilter: options.toolFilter,
369
- excludeTools: options.excludeTools,
370
- });
371
- }
372
- return filtered;
373
- }
374
- /**
375
- * Prepare generation context including tools and model
376
- */
377
- async prepareGenerationContext(options) {
378
- const shouldUseTools = !options.disableTools && this.supportsTools();
379
- const baseTools = shouldUseTools ? await this.getAllTools() : {};
380
- let tools = shouldUseTools
381
- ? {
382
- ...baseTools,
383
- ...(options.tools || {}),
384
- }
385
- : {};
386
- // Apply per-call tool filtering (whitelist/blacklist)
387
- tools = this.applyToolFiltering(tools, options);
388
- logger.debug(`Final tools prepared for AI`, {
389
- provider: this.providerName,
390
- directTools: getKeyCount(baseTools),
391
- directToolNames: getKeysAsString(baseTools),
392
- externalTools: getKeyCount(options.tools || {}),
393
- externalToolNames: getKeysAsString(options.tools || {}),
394
- totalTools: getKeyCount(tools),
395
- totalToolNames: getKeysAsString(tools),
396
- shouldUseTools,
397
- timestamp: Date.now(),
398
- });
399
- const model = await this.getAISDKModelWithMiddleware(options);
400
- return { tools, model };
401
- }
402
- /**
403
- * Get merged tools for streaming: combines base tools (MCP/built-in) with
404
- * user-provided tools (e.g., RAG tools passed via options.tools).
405
- *
406
- * This is the canonical tool-merge pattern for executeStream() implementations.
407
- * All providers should call this instead of getAllTools() directly.
408
- */
409
- async getToolsForStream(options) {
410
- const shouldUseTools = !options.disableTools && this.supportsTools();
411
- if (!shouldUseTools) {
412
- return {};
413
- }
414
- const baseTools = await this.getAllTools();
415
- const externalTools = (options.tools || {});
416
- let merged = { ...baseTools, ...externalTools };
417
- // Apply per-call tool filtering (whitelist/blacklist)
418
- merged = this.applyToolFiltering(merged, options);
419
- logger.debug(`Tools prepared for streaming`, {
420
- provider: this.providerName,
421
- baseToolCount: Object.keys(baseTools).length,
422
- externalToolCount: Object.keys(externalTools).length,
423
- totalToolCount: Object.keys(merged).length,
424
- });
425
- return merged;
426
- }
427
- /**
428
- * Build messages array for generation - delegated to MessageBuilder
429
- */
430
- async buildMessages(options) {
431
- return this.messageBuilder.buildMessages(options);
432
- }
433
- /**
434
- * Build messages array for streaming operations - delegated to MessageBuilder
435
- * This is a protected helper method that providers can use to build messages
436
- * with automatic multimodal detection, eliminating code duplication
437
- *
438
- * @param options - Stream options or text generation options
439
- * @returns Promise resolving to ModelMessage array ready for AI SDK
440
- */
441
- async buildMessagesForStream(options) {
442
- return this.messageBuilder.buildMessagesForStream(options);
443
- }
444
- /**
445
- * Execute the generation with AI SDK - delegated to GenerationHandler
446
- */
447
- async executeGeneration(model, messages, tools, options) {
448
- return this.generationHandler.executeGeneration(model, messages, tools, options);
449
- }
450
- /**
451
- * Log generation completion information - delegated to GenerationHandler
452
- */
453
- logGenerationComplete(generateResult) {
454
- this.generationHandler.logGenerationComplete(generateResult);
455
- }
456
- /**
457
- * Record performance metrics - delegated to TelemetryHandler
458
- */
459
- async recordPerformanceMetrics(usage, responseTime) {
460
- await this.telemetryHandler.recordPerformanceMetrics(usage, responseTime);
461
- }
462
- /**
463
- * Extract tool information from generation result - delegated to GenerationHandler
464
- */
465
- extractToolInformation(generateResult) {
466
- return this.generationHandler.extractToolInformation(generateResult);
467
- }
468
- /**
469
- * Format the enhanced result - delegated to GenerationHandler
470
- */
471
- formatEnhancedResult(generateResult, tools, toolsUsed, toolExecutions, options) {
472
- return this.generationHandler.formatEnhancedResult(generateResult, tools, toolsUsed, toolExecutions, options);
473
- }
474
- /**
475
- * Analyze AI response structure and log detailed debugging information - delegated to GenerationHandler
476
- */
477
- analyzeAIResponse(result) {
478
- this.generationHandler.analyzeAIResponse(result);
479
- }
480
- /**
481
- * Text generation method - implements AIProvider interface
482
- * Tools are always available unless explicitly disabled
483
- *
484
- * Supports Text-to-Speech (TTS) audio generation in two modes:
485
- * 1. Direct synthesis (default): TTS synthesizes the input text without AI generation
486
- * 2. AI response synthesis: TTS synthesizes the AI-generated response after generation
487
- *
488
- * When TTS is enabled with useAiResponse=false (default), the method returns early with
489
- * only the audio result, skipping AI generation entirely for optimal performance.
490
- *
491
- * When TTS is enabled with useAiResponse=true, the method performs full AI generation
492
- * and then synthesizes the AI response to audio.
493
- *
494
- * @param optionsOrPrompt - Generation options or prompt string
495
- * @param _analysisSchema - Optional analysis schema (not used)
496
- * @returns Enhanced result with optional audio field containing TTSResult
497
- *
498
- * IMPLEMENTATION NOTE: Uses streamText() under the hood and accumulates results
499
- * for consistency and better performance
500
- */
501
- async generate(optionsOrPrompt, _analysisSchema) {
502
- const options = this.normalizeTextOptions(optionsOrPrompt);
503
- this.validateOptions(options);
504
- const startTime = Date.now();
505
- // Observability: create metrics span for provider.generate
506
- const metricsSpan = SpanSerializer.createSpan(SpanType.MODEL_GENERATION, "provider.generate", {
507
- "ai.provider": this.providerName || "unknown",
508
- "ai.model": this.modelName || options.model || "unknown",
509
- "ai.temperature": options.temperature,
510
- "ai.max_tokens": options.maxTokens,
511
- }, this._traceContext?.parentSpanId, this._traceContext?.traceId);
512
- // OTEL span for provider-level generate tracing
513
- // Use startActiveSpan pattern via context.with() so child spans become descendants
514
- const otelSpan = tracers.provider.startSpan("neurolink.provider.generate", {
515
- kind: SpanKind.CLIENT,
516
- attributes: {
517
- [ATTR.GEN_AI_SYSTEM]: this.providerName || "unknown",
518
- [ATTR.GEN_AI_MODEL]: this.modelName || options.model || "unknown",
519
- [ATTR.GEN_AI_OPERATION]: "generate",
520
- [ATTR.NL_PROVIDER]: this.providerName || "unknown",
521
- },
522
- });
523
- // Set this span as the active context so child spans (GenerationHandler, etc.) become descendants
524
- const activeCtx = trace.setSpan(context.active(), otelSpan);
525
- let otelSpanEnded = false;
526
- return await context.with(activeCtx, async () => {
527
- try {
528
- // ===== VIDEO GENERATION MODE =====
529
- // Generate video from image + prompt using Veo 3.1
530
- if (options.output?.mode === "video") {
531
- return await this.handleVideoGeneration(options, startTime);
532
- }
533
- // ===== IMAGE GENERATION MODE =====
534
- // Route to executeImageGeneration for image generation models
535
- const isImageModel = IMAGE_GENERATION_MODELS.some((m) => this.modelName.includes(m));
536
- if (isImageModel) {
537
- logger.info(`Image generation model detected, routing to executeImageGeneration`, {
538
- provider: this.providerName,
539
- model: this.modelName,
540
- });
541
- const imageResult = await this.executeImageGeneration(options);
542
- return await this.enhanceResult(imageResult, options, startTime);
543
- }
544
- // ===== TTS MODE 1: Direct Input Synthesis (useAiResponse=false) =====
545
- // Synthesize input text directly without AI generation
546
- // This is optimal for simple read-aloud scenarios
547
- if (options.tts?.enabled && !options.tts?.useAiResponse) {
548
- const textToSynthesize = options.prompt ?? options.input?.text ?? "";
549
- // Build base result structure - common to both paths
550
- const baseResult = {
551
- content: textToSynthesize,
552
- provider: options.provider ?? this.providerName,
553
- model: this.modelName,
554
- usage: { input: 0, output: 0, total: 0 },
555
- };
556
- try {
557
- const ttsResult = await TTSProcessor.synthesize(textToSynthesize, options.provider ?? this.providerName, options.tts);
558
- baseResult.audio = ttsResult;
559
- }
560
- catch (ttsError) {
561
- logger.error(`TTS synthesis failed in Mode 1 (direct input synthesis):`, ttsError);
562
- // baseResult remains without audio - graceful degradation
563
- }
564
- // Call enhanceResult for consistency - enables analytics/evaluation for TTS-only requests
565
- return await this.enhanceResult(baseResult, options, startTime);
566
- }
567
- // ===== Normal AI Generation Flow =====
568
- const { tools, model } = await this.prepareGenerationContext(options);
569
- const messages = await this.buildMessages(options);
570
- // ===== VIDEO ANALYSIS FROM MESSAGES CONTENT =====
571
- // Check if video files are present in messages content array
572
- // If video analysis is needed, perform it via Gemini, then pass through Claude for formatting
573
- if (hasVideoFrames(messages)) {
574
- const videoAnalysisResult = await executeVideoAnalysis(messages, {
575
- provider: options.provider,
576
- providerName: this.providerName,
577
- region: options.region,
578
- // Don't pass the main conversation model — video analysis uses
579
- // Google's Gemini API (generateContent) which only supports Gemini models.
580
- // Let videoAnalysisProcessor use its own default (gemini-2.5-flash).
581
- });
582
- // Extract user's original text from messages (excluding image parts)
583
- const userTextParts = messages
584
- .filter((m) => m.role === "user")
585
- .flatMap((m) => Array.isArray(m.content)
586
- ? m.content
587
- .filter((p) => p.type === "text")
588
- .map((p) => p.text)
589
- : [typeof m.content === "string" ? m.content : ""])
590
- .filter(Boolean);
591
- const userText = userTextParts.join("\n").trim();
592
- // Pass Gemini's analysis through Claude for structured JSON formatting
593
- // The system prompt (from Curator) includes JSON_REPORT_PROMPT_SUFFIX
594
- // which instructs Claude to output {"summary": "...", "details": "..."}
595
- let formattedContent = videoAnalysisResult;
596
- let usage = { input: 0, output: 0, total: 0 };
597
- if (options.systemPrompt) {
598
- try {
599
- const formattingPrompt = userText
600
- ? `The user asked: "${userText}"\n\nHere is the video/image analysis result from the visual analysis system:\n\n${videoAnalysisResult}\n\nBased on this analysis, provide your response.`
601
- : `Here is a video/image analysis result from the visual analysis system:\n\n${videoAnalysisResult}\n\nBased on this analysis, provide your response.`;
602
- logger.debug("[VideoAnalysis] Formatting via Claude", {
603
- userTextLength: userText.length,
604
- analysisLength: videoAnalysisResult.length,
605
- });
606
- const formattedResult = await generateText({
607
- model,
608
- system: options.systemPrompt,
609
- messages: [
610
- { role: "user", content: formattingPrompt },
611
- ],
612
- maxOutputTokens: options.maxTokens || 8192,
613
- temperature: 0.3,
614
- abortSignal: options.abortSignal,
615
- experimental_telemetry: this.telemetryHandler?.getTelemetryConfig(options, "generate"),
616
- });
617
- formattedContent = formattedResult.text;
618
- usage = {
619
- input: formattedResult.usage?.inputTokens || 0,
620
- output: formattedResult.usage?.outputTokens || 0,
621
- total: (formattedResult.usage?.inputTokens || 0) +
622
- (formattedResult.usage?.outputTokens || 0),
623
- };
624
- logger.debug("[VideoAnalysis] Claude formatting complete", {
625
- formattedLength: formattedContent.length,
626
- usage,
627
- });
628
- }
629
- catch (error) {
630
- logger.warn("[VideoAnalysis] Claude formatting failed, using raw Gemini output", {
631
- error: error instanceof Error ? error.message : String(error),
632
- });
633
- // formattedContent remains as raw videoAnalysisResult (graceful degradation)
634
- }
635
- }
636
- const videoResult = {
637
- content: formattedContent,
638
- provider: options.provider ?? this.providerName,
639
- model: this.modelName,
640
- usage,
641
- };
642
- return await this.enhanceResult(videoResult, options, startTime);
643
- }
644
- // Compose timeout signal with user-provided abort signal (mirrors stream path)
645
- const timeoutController = createTimeoutController(options.timeout, this.providerName, "generate");
646
- const composedSignal = composeAbortSignals(options.abortSignal, timeoutController?.controller.signal);
647
- const composedOptions = composedSignal
648
- ? { ...options, abortSignal: composedSignal }
649
- : options;
650
- let generateResult;
651
- try {
652
- generateResult = await this.executeGeneration(model, messages, tools, composedOptions);
653
- }
654
- finally {
655
- timeoutController?.cleanup();
656
- }
657
- this.analyzeAIResponse(generateResult);
658
- this.logGenerationComplete(generateResult);
659
- const responseTime = Date.now() - startTime;
660
- await this.recordPerformanceMetrics(generateResult.usage, responseTime);
661
- const { toolsUsed, toolExecutions } = this.extractToolInformation(generateResult);
662
- let enhancedResult = this.formatEnhancedResult(generateResult, tools, toolsUsed, toolExecutions, options);
663
- // ===== TTS MODE 2: AI Response Synthesis (useAiResponse=true) =====
664
- // Synthesize AI-generated response after generation completes
665
- if (options.tts?.enabled && options.tts?.useAiResponse) {
666
- const aiResponse = enhancedResult.content;
667
- const provider = options.provider ?? this.providerName;
668
- // Validate AI response and provider before synthesis
669
- if (aiResponse && provider) {
670
- try {
671
- const ttsResult = await TTSProcessor.synthesize(aiResponse, provider, options.tts);
672
- // Add audio to enhanced result (TTSProcessor already includes latency in metadata)
673
- enhancedResult = {
674
- ...enhancedResult,
675
- audio: ttsResult,
676
- };
677
- }
678
- catch (ttsError) {
679
- // Log TTS error but continue with text-only result
680
- logger.error(`TTS synthesis failed in Mode 2 (AI response synthesis):`, ttsError);
681
- // enhancedResult remains unchanged (no audio field added)
682
- }
683
- }
684
- else {
685
- logger.warn(`TTS synthesis skipped despite being enabled`, {
686
- provider: this.providerName,
687
- hasAiResponse: !!aiResponse,
688
- aiResponseLength: aiResponse?.length ?? 0,
689
- hasProvider: !!provider,
690
- ttsConfig: {
691
- enabled: options.tts?.enabled,
692
- useAiResponse: options.tts?.useAiResponse,
693
- },
694
- reason: !aiResponse
695
- ? "AI response is empty or undefined"
696
- : "Provider is missing",
697
- });
698
- }
699
- }
700
- // Observability: record successful generate span with token/cost data
701
- let enrichedGenerateSpan = { ...metricsSpan };
702
- if (enhancedResult?.usage) {
703
- enrichedGenerateSpan = SpanSerializer.enrichWithTokenUsage(enrichedGenerateSpan, {
704
- promptTokens: enhancedResult.usage.input || 0,
705
- completionTokens: enhancedResult.usage.output || 0,
706
- totalTokens: enhancedResult.usage.total || 0,
707
- });
708
- const cost = calculateCost(this.providerName, this.modelName, {
709
- input: enhancedResult.usage.input || 0,
710
- output: enhancedResult.usage.output || 0,
711
- total: enhancedResult.usage.total || 0,
712
- });
713
- if (cost && cost > 0) {
714
- enrichedGenerateSpan = SpanSerializer.enrichWithCost(enrichedGenerateSpan, {
715
- totalCost: cost,
716
- });
717
- }
718
- }
719
- const _endedGenerateSpan = SpanSerializer.endSpan(enrichedGenerateSpan, SpanStatus.OK);
720
- // Note: Do NOT record to getMetricsAggregator() here — the neurolink.ts
721
- // generation:end listener creates an authoritative span with richer context
722
- // (provider name, model, input/output) and records to both aggregators.
723
- // Recording here would double-count cost and token metrics.
724
- return await this.enhanceResult(enhancedResult, options, startTime);
725
- }
726
- catch (error) {
727
- // Observability: record failed generate span
728
- const _endedGenerateSpan = SpanSerializer.endSpan(metricsSpan, SpanStatus.ERROR, error instanceof Error ? error.message : String(error));
729
- // Note: Do NOT record to getMetricsAggregator() here — neurolink.ts
730
- // handles authoritative metrics recording to avoid double-counting.
731
- otelSpan.setStatus({
732
- code: SpanStatusCode.ERROR,
733
- message: error instanceof Error ? error.message : String(error),
734
- });
735
- otelSpan.end();
736
- otelSpanEnded = true;
737
- // Abort errors are expected when a generation is cancelled — log at info, not error
738
- if (isAbortError(error)) {
739
- logger.info(`Generate aborted for ${this.providerName}`, {
740
- error: error instanceof Error ? error.message : String(error),
741
- });
742
- }
743
- else {
744
- logger.error(`Generate failed for ${this.providerName}:`, error);
745
- }
746
- throw this.handleProviderError(error);
747
- }
748
- finally {
749
- if (!otelSpanEnded) {
750
- otelSpan.setStatus({ code: SpanStatusCode.OK });
751
- otelSpan.end();
752
- }
753
- }
754
- }); // end context.with
755
- }
756
- /**
757
- * Alias for generate method - implements AIProvider interface
758
- */
759
- async gen(optionsOrPrompt, analysisSchema) {
760
- return this.generate(optionsOrPrompt, analysisSchema);
761
- }
762
- /**
763
- * BACKWARD COMPATIBILITY: Legacy generateText method
764
- * Converts EnhancedGenerateResult to TextGenerationResult format
765
- * Ensures existing scripts using createAIProvider().generateText() continue to work
766
- */
767
- async generateText(options) {
768
- // Validate required parameters for backward compatibility - support both prompt and input.text
769
- const promptText = options.prompt || options.input?.text;
770
- if (!promptText ||
771
- typeof promptText !== "string" ||
772
- promptText.trim() === "") {
773
- throw new Error("GenerateText options must include prompt or input.text as a non-empty string");
774
- }
775
- // Call the main generate method
776
- const result = await this.generate(options);
777
- if (!result) {
778
- throw new Error("Generation failed: No result returned");
779
- }
780
- // Convert EnhancedGenerateResult to TextGenerationResult format
781
- return {
782
- content: result.content || "",
783
- provider: result.provider || this.providerName,
784
- model: result.model || this.modelName,
785
- usage: result.usage || {
786
- input: 0,
787
- output: 0,
788
- total: 0,
789
- },
790
- responseTime: 0, // BaseProvider doesn't track response time directly
791
- toolsUsed: result.toolsUsed || [],
792
- enhancedWithTools: !!(result.toolsUsed && result.toolsUsed.length > 0),
793
- analytics: result.analytics,
794
- evaluation: result.evaluation,
795
- audio: result.audio,
796
- };
797
- }
798
- /**
799
- * Generate embeddings for text
800
- *
801
- * This is a default implementation that throws an error.
802
- * Providers that support embeddings (OpenAI, Google Vertex, Amazon Bedrock)
803
- * should override this method with their specific implementation.
804
- *
805
- * @param text - The text to embed
806
- * @param _modelName - Optional embedding model name (provider-specific)
807
- * @returns Promise resolving to the embedding vector (array of numbers)
808
- * @throws Error if the provider does not support embeddings
809
- *
810
- * @example
811
- * ```typescript
812
- * const provider = await ProviderFactory.createProvider('openai', 'text-embedding-3-small');
813
- * const embedding = await provider.embed('Hello world');
814
- * console.log(embedding); // [0.123, -0.456, ...]
815
- * ```
816
- */
817
- async embed(text, _modelName) {
818
- logger.warn(`embed() called on ${this.providerName} which does not have a native implementation`, {
819
- textLength: text.length,
820
- });
821
- throw new Error(`Embedding generation is not supported by the ${this.providerName} provider. ` +
822
- `Supported providers: openai, vertex/google, bedrock. ` +
823
- `Use an embedding model like text-embedding-3-small (OpenAI), text-embedding-004 (Vertex), ` +
824
- `or amazon.titan-embed-text-v2:0 (Bedrock).`);
825
- }
826
- /**
827
- * Generate embeddings for multiple texts in a single batch
828
- *
829
- * This is a default implementation that throws an error.
830
- * Providers that support embeddings should override this method.
831
- * The AI SDK's embedMany automatically handles chunking for models with batch limits.
832
- *
833
- * @param texts - The texts to embed
834
- * @param _modelName - Optional embedding model name (provider-specific)
835
- * @returns Promise resolving to an array of embedding vectors
836
- * @throws Error if the provider does not support embeddings
837
- */
838
- async embedMany(texts, _modelName) {
839
- logger.warn(`embedMany() called on ${this.providerName} which does not have a native implementation`, {
840
- count: texts.length,
841
- });
842
- throw new Error(`Batch embedding generation is not supported by the ${this.providerName} provider. ` +
843
- `Supported providers: openai, googleAiStudio, vertex/google, bedrock. ` +
844
- `Use an embedding model like text-embedding-3-small (OpenAI), gemini-embedding-001 (Google AI), ` +
845
- `text-embedding-004 (Vertex), or amazon.titan-embed-text-v2:0 (Bedrock).`);
846
- }
847
- /**
848
- * Get the default embedding model for this provider
849
- *
850
- * Override in subclasses to provide provider-specific defaults.
851
- * Returns undefined for providers that don't support embeddings.
852
- *
853
- * @returns The default embedding model name, or undefined if not supported
854
- */
855
- getDefaultEmbeddingModel() {
856
- // Default implementation returns undefined - providers override this
857
- return undefined;
858
- }
859
- /**
860
- * Get AI SDK model with middleware applied
861
- * This method wraps the base model with any configured middleware
862
- * TODO: Implement global level middlewares that can be used
863
- */
864
- async getAISDKModelWithMiddleware(options = {}) {
865
- // Get the base model
866
- const baseModel = await this.getAISDKModel();
867
- logger.debug(`Retrieved base model for ${this.providerName}`, {
868
- provider: this.providerName,
869
- model: this.modelName,
870
- hasMiddlewareConfig: !!this.middlewareOptions,
871
- timestamp: Date.now(),
872
- });
873
- // Check if middleware should be applied
874
- const middlewareOptions = this.extractMiddlewareOptions(options);
875
- logger.debug(`Middleware extraction result`, {
876
- provider: this.providerName,
877
- model: this.modelName,
878
- middlewareOptions,
879
- });
880
- if (!middlewareOptions) {
881
- return baseModel;
882
- }
883
- try {
884
- logger.debug(`Applying middleware to ${this.providerName} model`, {
885
- provider: this.providerName,
886
- model: this.modelName,
887
- middlewareOptions,
888
- });
889
- // Create a new factory instance with the specified options
890
- const factory = new MiddlewareFactory(middlewareOptions);
891
- // Create middleware context
892
- const context = factory.createContext(this.providerName, this.modelName, options, {
893
- sessionId: this.sessionId,
894
- userId: this.userId,
895
- });
896
- // Apply middleware to the model
897
- const wrappedModel = factory.applyMiddleware(baseModel, context, middlewareOptions);
898
- logger.debug(`Applied middleware to ${this.providerName} model`, {
899
- provider: this.providerName,
900
- model: this.modelName,
901
- hasMiddleware: true,
902
- });
903
- return wrappedModel;
904
- }
905
- catch (error) {
906
- logger.warn(`Failed to apply middleware to ${this.providerName}, using base model`, {
907
- error: error instanceof Error ? error.message : String(error),
908
- });
909
- // Return base model on middleware failure to maintain functionality
910
- return baseModel;
911
- }
912
- }
913
- /**
914
- * Extract middleware options - delegated to Utilities
915
- */
916
- extractMiddlewareOptions(options) {
917
- return this.utilities.extractMiddlewareOptions(options);
918
- }
919
- // ===================
920
- // TOOL MANAGEMENT
921
- // ===================
922
- /**
923
- * Check if a schema is a Zod schema - delegated to Utilities
924
- */
925
- isZodSchema(schema) {
926
- return this.utilities.isZodSchema(schema);
927
- }
928
- /**
929
- * Convert tool execution result - delegated to Utilities
930
- */
931
- async convertToolResult(result) {
932
- return this.utilities.convertToolResult(result);
933
- }
934
- /**
935
- * Fix JSON Schema for OpenAI strict mode - delegated to Utilities
936
- */
937
- fixSchemaForOpenAIStrictMode(schema) {
938
- return this.utilities.fixSchemaForOpenAIStrictMode(schema);
939
- }
940
- /**
941
- * Get all available tools - delegated to ToolsManager
942
- */
943
- async getAllTools() {
944
- return this.toolsManager.getAllTools();
945
- }
946
- /**
947
- * Calculate actual cost - delegated to TelemetryHandler
948
- */
949
- async calculateActualCost(usage) {
950
- return this.telemetryHandler.calculateActualCost(usage);
951
- }
952
- /**
953
- * Create a permissive Zod schema - delegated to Utilities
954
- */
955
- createPermissiveZodSchema() {
956
- return this.utilities.createPermissiveZodSchema();
957
- }
958
- /**
959
- * Set session context for MCP tools - delegated to ToolsManager
960
- */
961
- setSessionContext(sessionId, userId) {
962
- this.sessionId = sessionId;
963
- this.userId = userId;
964
- this.toolsManager.setSessionContext(sessionId, userId);
965
- }
966
- /**
967
- * Handle provider errors with abort passthrough.
968
- * AbortErrors are never wrapped — they must propagate with their
969
- * original identity so that isAbortError() can detect them in
970
- * retry/fallback loops (directProviderGeneration, performMCPGenerationRetries).
971
- */
972
- handleProviderError(error) {
973
- if (isAbortError(error)) {
974
- // Preserve AbortError identity — never wrap in provider-specific formatting
975
- return error instanceof Error
976
- ? error
977
- : new DOMException("The operation was aborted", "AbortError");
978
- }
979
- return this.formatProviderError(error);
980
- }
981
- /**
982
- * Image generation method. Providers that support it should override this.
983
- * By default, it throws an error indicating that the functionality is not supported.
984
- * @param _options The generation options.
985
- * @returns A promise that resolves to the generation result.
986
- */
987
- async executeImageGeneration(_options) {
988
- throw new Error(`Image generation is not supported by the ${this.providerName} provider or the selected model.`);
989
- }
990
- // ===================
991
- // CONSOLIDATED PROVIDER METHODS - MOVED FROM INDIVIDUAL PROVIDERS
992
- // ===================
993
- /**
994
- * Execute operation with timeout and proper cleanup
995
- * Consolidates identical timeout handling from 8/10 providers
996
- */
997
- async executeWithTimeout(operation, options) {
998
- const timeout = this.getTimeout(options);
999
- const timeoutController = createTimeoutController(timeout, this.providerName, options.operationType || "generate");
1000
- try {
1001
- if (timeoutController) {
1002
- return await Promise.race([
1003
- operation(),
1004
- new Promise((_, reject) => {
1005
- timeoutController.controller.signal.addEventListener("abort", () => {
1006
- reject(new TimeoutError(`${this.providerName} operation timed out`, timeoutController.timeoutMs, this.providerName, options.operationType ||
1007
- "generate"));
1008
- });
1009
- }),
1010
- ]);
1011
- }
1012
- else {
1013
- return await operation();
1014
- }
1015
- }
1016
- finally {
1017
- timeoutController?.cleanup();
1018
- }
1019
- }
1020
- /**
1021
- * Validate stream options - delegated to StreamHandler
1022
- */
1023
- validateStreamOptions(options) {
1024
- this.streamHandler.validateStreamOptions(options);
1025
- }
1026
- /**
1027
- * Create text stream transformation - delegated to StreamHandler
1028
- */
1029
- createTextStream(result) {
1030
- return this.streamHandler.createTextStream(result);
1031
- }
1032
- /**
1033
- * Create standardized stream result - delegated to StreamHandler
1034
- */
1035
- createStreamResult(stream, additionalProps = {}) {
1036
- return this.streamHandler.createStreamResult(stream, additionalProps);
1037
- }
1038
- /**
1039
- * Create stream analytics - delegated to StreamHandler
1040
- */
1041
- async createStreamAnalytics(result, startTime, options) {
1042
- return this.streamHandler.createStreamAnalytics(result, startTime, options);
1043
- }
1044
- /**
1045
- * Handle common error patterns - delegated to Utilities
1046
- */
1047
- handleCommonErrors(error) {
1048
- return this.utilities.handleCommonErrors(error);
1049
- }
1050
- /**
1051
- * Set up tool executor - delegated to ToolsManager
1052
- * @param sdk - The NeuroLinkSDK instance for tool execution
1053
- * @param functionTag - Function name for logging
1054
- */
1055
- setupToolExecutor(sdk, functionTag) {
1056
- this.toolsManager.setupToolExecutor(sdk, functionTag);
1057
- }
1058
- // ===================
1059
- // TEMPLATE METHODS - COMMON FUNCTIONALITY
1060
- // ===================
1061
- /**
1062
- * Normalize text generation options - delegated to Utilities
1063
- */
1064
- normalizeTextOptions(optionsOrPrompt) {
1065
- return this.utilities.normalizeTextOptions(optionsOrPrompt);
1066
- }
1067
- /**
1068
- * Normalize stream options - delegated to Utilities
1069
- */
1070
- normalizeStreamOptions(optionsOrPrompt) {
1071
- return this.utilities.normalizeStreamOptions(optionsOrPrompt);
1072
- }
1073
- async enhanceResult(result, options, startTime) {
1074
- const responseTime = Date.now() - startTime;
1075
- // CRITICAL FIX: Store imageOutput separately to ensure it's preserved
1076
- const imageOutput = result.imageOutput;
1077
- let enhancedResult = { ...result };
1078
- if (options.enableAnalytics) {
1079
- try {
1080
- const analytics = await this.createAnalytics(result, responseTime, options);
1081
- // Preserve ALL fields including imageOutput when adding analytics
1082
- enhancedResult = { ...enhancedResult, analytics, imageOutput };
1083
- }
1084
- catch (error) {
1085
- logger.warn(`Analytics creation failed for ${this.providerName}:`, error);
1086
- }
1087
- }
1088
- if (options.enableEvaluation) {
1089
- try {
1090
- const evaluation = await this.createEvaluation(result, options);
1091
- // Preserve ALL fields including imageOutput when adding evaluation
1092
- enhancedResult = { ...enhancedResult, evaluation, imageOutput };
1093
- }
1094
- catch (error) {
1095
- logger.warn(`Evaluation creation failed for ${this.providerName}:`, error);
1096
- }
1097
- }
1098
- // CRITICAL FIX: Always restore imageOutput if it existed in the original result
1099
- if (imageOutput) {
1100
- enhancedResult.imageOutput = imageOutput;
1101
- }
1102
- return enhancedResult;
1103
- }
1104
- /**
1105
- * Handle video generation mode
1106
- *
1107
- * Generates video from input image + text prompt using Vertex AI Veo 3.1.
1108
- *
1109
- * @param options - Text generation options with video configuration
1110
- * @param startTime - Generation start timestamp for metrics
1111
- * @returns Enhanced result with video data
1112
- *
1113
- * @example
1114
- * ```typescript
1115
- * const result = await provider.generate({
1116
- * input: { text: "Product showcase", images: [imageBuffer] },
1117
- * output: { mode: "video", video: { resolution: "1080p" } }
1118
- * });
1119
- * // result.video contains the generated video
1120
- * ```
1121
- */
1122
- async handleVideoGeneration(options, startTime) {
1123
- // Dynamic imports to avoid loading video dependencies unless needed
1124
- const { generateVideoWithVertex, VideoError, VIDEO_ERROR_CODES } = await import("../adapters/video/vertexVideoHandler.js");
1125
- const { validateVideoGenerationInput, validateImageForVideo, validateDirectorModeInput, } = await import("../utils/parameterValidation.js");
1126
- const { ErrorFactory } = await import("../utils/errorHandling.js");
1127
- // Build GenerateOptions for validation
1128
- const generateOptions = {
1129
- input: options.input || { text: options.prompt || "" },
1130
- output: options.output,
1131
- provider: options.provider,
1132
- model: options.model,
1133
- };
1134
- // ===== DIRECTOR MODE =====
1135
- // Route to Director pipeline when segments are provided
1136
- if (generateOptions.input?.segments &&
1137
- Array.isArray(generateOptions.input.segments) &&
1138
- generateOptions.input.segments.length > 0) {
1139
- // Type narrowing: segments is guaranteed to exist here
1140
- const segments = generateOptions.input.segments;
1141
- const directorValidation = validateDirectorModeInput(generateOptions);
1142
- if (!directorValidation.isValid) {
1143
- throw ErrorFactory.invalidParameters("director-mode", new Error(directorValidation.errors
1144
- .map((e) => e.message)
1145
- .join("; ")), { errors: directorValidation.errors });
1146
- }
1147
- if (directorValidation.warnings.length > 0) {
1148
- for (const warning of directorValidation.warnings) {
1149
- logger.warn(`Director Mode warning: ${warning}`);
1150
- }
1151
- }
1152
- const { executeDirectorPipeline, DIRECTOR_PIPELINE_TIMEOUT_MS } = await import("../adapters/video/directorPipeline.js");
1153
- // Use caller's timeout if provided, otherwise use default Director timeout
1154
- const directorTimeout = options.timeout ?? DIRECTOR_PIPELINE_TIMEOUT_MS;
1155
- const videoResult = await this.executeWithTimeout(() => executeDirectorPipeline(segments, generateOptions.output?.video ?? {}, generateOptions.output?.director ?? {}, options.region), { timeout: directorTimeout, operationType: "generate" });
1156
- // Build content summary with metadata
1157
- const joinedPrompts = generateOptions.input.segments
1158
- .map((s) => s.prompt)
1159
- .join(" → ");
1160
- const segmentCount = videoResult.metadata?.segmentCount ??
1161
- generateOptions.input.segments.length;
1162
- const transitionCount = videoResult.metadata?.transitionCount ?? Math.max(0, segmentCount - 1);
1163
- const totalDuration = videoResult.metadata?.duration ?? 0;
1164
- const contentSummary = `${joinedPrompts} — duration: ${totalDuration}s, segments: ${segmentCount}, transitions: ${transitionCount}`;
1165
- const baseResult = {
1166
- content: contentSummary,
1167
- provider: "vertex",
1168
- model: options.model || "veo-3.1-generate-001",
1169
- usage: { input: 0, output: 0, total: 0 },
1170
- video: videoResult,
1171
- };
1172
- return await this.enhanceResult(baseResult, options, startTime);
1173
- }
1174
- // ===== STANDARD SINGLE-CLIP VIDEO GENERATION =====
1175
- // Validate video generation input
1176
- const validation = validateVideoGenerationInput(generateOptions);
1177
- if (!validation.isValid) {
1178
- throw ErrorFactory.invalidParameters("video-generation", new Error(validation.errors.map((e) => e.message).join("; ")), { errors: validation.errors });
1179
- }
1180
- // Log warnings if any
1181
- if (validation.warnings.length > 0) {
1182
- for (const warning of validation.warnings) {
1183
- logger.warn(`Video generation warning: ${warning}`);
1184
- }
1185
- }
1186
- // Extract image from input
1187
- const imageInput = options.input?.images?.[0];
1188
- if (!imageInput) {
1189
- throw new VideoError({
1190
- code: VIDEO_ERROR_CODES.INVALID_INPUT,
1191
- message: "Video generation requires an input image. Provide via input.images array.",
1192
- retriable: false,
1193
- context: { field: "input.images" },
1194
- });
1195
- }
1196
- // Timeout for image IO operations (15 seconds)
1197
- const IMAGE_IO_TIMEOUT_MS = 15000;
1198
- // Load image buffer if path/URL
1199
- let imageBuffer;
1200
- if (typeof imageInput === "string") {
1201
- if (imageInput.startsWith("http://") ||
1202
- imageInput.startsWith("https://")) {
1203
- // URL - fetch the image with timeout
1204
- logger.debug("Fetching image from URL for video generation", {
1205
- url: imageInput.substring(0, 100),
1206
- });
1207
- let response;
1208
- try {
1209
- response = await this.executeWithTimeout(() => fetch(imageInput), {
1210
- timeout: IMAGE_IO_TIMEOUT_MS,
1211
- operationType: "generate", // Part of video generation flow
1212
- });
1213
- }
1214
- catch (error) {
1215
- throw new VideoError({
1216
- code: VIDEO_ERROR_CODES.INVALID_INPUT,
1217
- message: `Failed to fetch image from URL: ${error instanceof Error ? error.message : "Request timed out"}`,
1218
- retriable: true,
1219
- context: { url: imageInput, timeout: IMAGE_IO_TIMEOUT_MS },
1220
- originalError: error instanceof Error ? error : undefined,
1221
- });
1222
- }
1223
- if (!response.ok) {
1224
- throw new VideoError({
1225
- code: VIDEO_ERROR_CODES.INVALID_INPUT,
1226
- message: `Failed to fetch image from URL: ${response.status} ${response.statusText}`,
1227
- retriable: response.status >= 500,
1228
- context: { url: imageInput, status: response.status },
1229
- });
1230
- }
1231
- imageBuffer = Buffer.from(await response.arrayBuffer());
1232
- }
1233
- else {
1234
- // File path - read from disk with timeout
1235
- logger.debug("Reading image from path for video generation", {
1236
- path: imageInput,
1237
- });
1238
- const fs = await import("node:fs/promises");
1239
- try {
1240
- imageBuffer = await this.executeWithTimeout(() => fs.readFile(imageInput), { timeout: IMAGE_IO_TIMEOUT_MS, operationType: "generate" });
1241
- }
1242
- catch (error) {
1243
- throw new VideoError({
1244
- code: VIDEO_ERROR_CODES.INVALID_INPUT,
1245
- message: `Failed to read image file: ${error instanceof Error ? error.message : String(error)}`,
1246
- retriable: false,
1247
- context: { path: imageInput, timeout: IMAGE_IO_TIMEOUT_MS },
1248
- originalError: error instanceof Error ? error : undefined,
1249
- });
1250
- }
1251
- }
1252
- }
1253
- else if (Buffer.isBuffer(imageInput)) {
1254
- imageBuffer = imageInput;
1255
- }
1256
- else if (typeof imageInput === "object" && "data" in imageInput) {
1257
- // ImageWithAltText type
1258
- const imgData = imageInput.data;
1259
- if (typeof imgData === "string") {
1260
- imageBuffer = Buffer.from(imgData, "base64");
1261
- }
1262
- else if (Buffer.isBuffer(imgData)) {
1263
- imageBuffer = imgData;
1264
- }
1265
- else {
1266
- throw new VideoError({
1267
- code: VIDEO_ERROR_CODES.INVALID_INPUT,
1268
- message: "ImageWithAltText.data must be a base64 string or Buffer.",
1269
- retriable: false,
1270
- context: { field: "input.images[0].data", type: typeof imgData },
1271
- });
1272
- }
1273
- }
1274
- else {
1275
- throw new VideoError({
1276
- code: VIDEO_ERROR_CODES.INVALID_INPUT,
1277
- message: "Invalid image input type. Provide Buffer, path string, URL, or ImageWithAltText.",
1278
- retriable: false,
1279
- context: { field: "input.images[0]", type: typeof imageInput },
1280
- });
1281
- }
1282
- // Validate image format and size (for Buffer inputs)
1283
- const imageValidation = validateImageForVideo(imageBuffer);
1284
- if (imageValidation) {
1285
- throw ErrorFactory.invalidParameters("video-generation", new Error(imageValidation.message), {
1286
- field: "input.images[0]",
1287
- validation: imageValidation,
1288
- });
1289
- }
1290
- // Get prompt text
1291
- const prompt = options.prompt || options.input?.text || "";
1292
- logger.info("Starting video generation", {
1293
- provider: "vertex",
1294
- model: options.model || "veo-3.1-generate-001",
1295
- promptLength: prompt.length,
1296
- imageSize: imageBuffer.length,
1297
- resolution: options.output?.video?.resolution || "720p",
1298
- duration: options.output?.video?.length || 6,
1299
- });
1300
- // Generate video using Vertex handler (no processor abstraction)
1301
- const videoResult = await generateVideoWithVertex(imageBuffer, prompt, options.output?.video, options.region);
1302
- logger.info("Video generation complete", {
1303
- videoSize: videoResult.data.length,
1304
- duration: videoResult.metadata?.duration,
1305
- processingTime: videoResult.metadata?.processingTime,
1306
- });
1307
- // Build result
1308
- const baseResult = {
1309
- content: prompt, // Echo the prompt as content
1310
- provider: "vertex",
1311
- model: options.model || "veo-3.1-generate-001",
1312
- usage: { input: 0, output: 0, total: 0 },
1313
- video: videoResult,
1314
- };
1315
- return await this.enhanceResult(baseResult, options, startTime);
1316
- }
1317
- /**
1318
- * Create analytics - delegated to TelemetryHandler
1319
- */
1320
- async createAnalytics(result, responseTime, options) {
1321
- return this.telemetryHandler.createAnalytics(result, responseTime, options.context);
1322
- }
1323
- /**
1324
- * Create evaluation - delegated to TelemetryHandler
1325
- */
1326
- async createEvaluation(result, options) {
1327
- return this.telemetryHandler.createEvaluation(result, options);
1328
- }
1329
- /**
1330
- * Validate text generation options - delegated to Utilities
1331
- */
1332
- validateOptions(options) {
1333
- this.utilities.validateOptions(options);
1334
- }
1335
- /**
1336
- * Get provider information - delegated to Utilities
1337
- */
1338
- getProviderInfo() {
1339
- return this.utilities.getProviderInfo();
1340
- }
1341
- /**
1342
- * Get timeout value in milliseconds - delegated to Utilities
1343
- */
1344
- getTimeout(options) {
1345
- return this.utilities.getTimeout(options);
1346
- }
1347
- /**
1348
- * Check if tool executions should be stored and handle storage
1349
- */
1350
- async handleToolExecutionStorage(toolCalls, toolResults, options, currentTime) {
1351
- return this.telemetryHandler.handleToolExecutionStorage(toolCalls, toolResults, options, currentTime);
1352
- }
1353
- /**
1354
- * Utility method to chunk large prompts into smaller pieces
1355
- * @param prompt The prompt to chunk
1356
- * @param maxChunkSize Maximum size per chunk (default: 900,000 characters)
1357
- * @param overlap Overlap between chunks to maintain context (default: 100 characters)
1358
- * @returns Array of prompt chunks
1359
- */
1360
- static chunkPrompt(prompt, maxChunkSize = 900000, overlap = 100) {
1361
- if (prompt.length <= maxChunkSize) {
1362
- return [prompt];
1363
- }
1364
- const chunks = [];
1365
- let start = 0;
1366
- while (start < prompt.length) {
1367
- const end = Math.min(start + maxChunkSize, prompt.length);
1368
- chunks.push(prompt.slice(start, end));
1369
- // Break if we've reached the end
1370
- if (end >= prompt.length) {
1371
- break;
1372
- }
1373
- // Move start forward, accounting for overlap
1374
- const nextStart = end - overlap;
1375
- // Ensure we make progress (avoid infinite loops)
1376
- if (nextStart <= start) {
1377
- start = end;
1378
- }
1379
- else {
1380
- start = Math.max(nextStart, 0);
1381
- }
1382
- }
1383
- return chunks;
1384
- }
1385
- }