@juspay/neurolink 9.32.0 → 9.33.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (475) hide show
  1. package/CHANGELOG.md +12 -0
  2. package/dist/auth/anthropicOAuth.js +1 -1
  3. package/dist/cli/commands/proxy.js +18 -5
  4. package/dist/client/aiSdkAdapter.js +1 -1
  5. package/dist/client/index.js +137 -501
  6. package/dist/core/factory.js +0 -1
  7. package/dist/core/redisConversationMemoryManager.js +1 -1
  8. package/dist/features/ppt/slideGenerator.js +0 -1
  9. package/dist/features/ppt/utils.js +0 -1
  10. package/dist/lib/neurolink.d.ts +10 -0
  11. package/dist/lib/neurolink.js +41 -7
  12. package/dist/lib/server/routes/claudeProxyRoutes.js +45 -9
  13. package/dist/lib/types/generateTypes.d.ts +16 -0
  14. package/dist/lib/types/streamTypes.d.ts +15 -0
  15. package/dist/mcp/elicitationProtocol.js +1 -1
  16. package/dist/mcp/servers/agent/directToolsServer.js +0 -1
  17. package/dist/neurolink.d.ts +10 -0
  18. package/dist/neurolink.js +41 -7
  19. package/dist/providers/azureOpenai.js +1 -1
  20. package/dist/providers/huggingFace.js +0 -1
  21. package/dist/providers/openaiCompatible.js +0 -1
  22. package/dist/sdk/toolRegistration.js +0 -1
  23. package/dist/server/openapi/generator.js +1 -1
  24. package/dist/server/routes/claudeProxyRoutes.js +45 -9
  25. package/dist/types/configTypes.js +0 -5
  26. package/dist/types/generateTypes.d.ts +16 -0
  27. package/dist/types/modelTypes.js +0 -1
  28. package/dist/types/streamTypes.d.ts +15 -0
  29. package/dist/types/tools.js +0 -1
  30. package/dist/types/typeAliases.js +0 -1
  31. package/dist/types/utilities.js +1 -1
  32. package/dist/types/workflowTypes.js +0 -1
  33. package/dist/utils/providerRetry.js +0 -1
  34. package/dist/utils/providerUtils.js +0 -1
  35. package/package.json +2 -2
  36. package/dist/client/adapters/providerImageAdapter.js +0 -588
  37. package/dist/client/adapters/tts/googleTTSHandler.js +0 -344
  38. package/dist/client/adapters/video/directorPipeline.js +0 -516
  39. package/dist/client/adapters/video/ffmpegAdapter.js +0 -206
  40. package/dist/client/adapters/video/frameExtractor.js +0 -143
  41. package/dist/client/adapters/video/vertexVideoHandler.js +0 -763
  42. package/dist/client/adapters/video/videoAnalyzer.js +0 -238
  43. package/dist/client/adapters/video/videoMerger.js +0 -171
  44. package/dist/client/agent/directTools.js +0 -840
  45. package/dist/client/auth/AuthProviderFactory.js +0 -111
  46. package/dist/client/auth/AuthProviderRegistry.js +0 -190
  47. package/dist/client/auth/RequestContext.js +0 -78
  48. package/dist/client/auth/accountPool.js +0 -178
  49. package/dist/client/auth/anthropicOAuth.js +0 -974
  50. package/dist/client/auth/authContext.js +0 -314
  51. package/dist/client/auth/errors.js +0 -39
  52. package/dist/client/auth/index.js +0 -61
  53. package/dist/client/auth/middleware/AuthMiddleware.js +0 -519
  54. package/dist/client/auth/middleware/rateLimitByUser.js +0 -554
  55. package/dist/client/auth/providers/BaseAuthProvider.js +0 -723
  56. package/dist/client/auth/providers/CognitoProvider.js +0 -304
  57. package/dist/client/auth/providers/KeycloakProvider.js +0 -393
  58. package/dist/client/auth/providers/auth0.js +0 -274
  59. package/dist/client/auth/providers/betterAuth.js +0 -182
  60. package/dist/client/auth/providers/clerk.js +0 -317
  61. package/dist/client/auth/providers/custom.js +0 -112
  62. package/dist/client/auth/providers/firebase.js +0 -226
  63. package/dist/client/auth/providers/jwt.js +0 -212
  64. package/dist/client/auth/providers/oauth2.js +0 -303
  65. package/dist/client/auth/providers/supabase.js +0 -259
  66. package/dist/client/auth/providers/workos.js +0 -284
  67. package/dist/client/auth/serverBridge.js +0 -25
  68. package/dist/client/auth/sessionManager.js +0 -437
  69. package/dist/client/auth/tokenStore.js +0 -799
  70. package/dist/client/client/aiSdkAdapter.js +0 -487
  71. package/dist/client/client/auth.js +0 -473
  72. package/dist/client/client/errors.js +0 -552
  73. package/dist/client/client/httpClient.js +0 -837
  74. package/dist/client/client/index.js +0 -172
  75. package/dist/client/client/interceptors.js +0 -601
  76. package/dist/client/client/sseClient.js +0 -545
  77. package/dist/client/client/streamingClient.js +0 -917
  78. package/dist/client/client/wsClient.js +0 -369
  79. package/dist/client/config/configManager.js +0 -303
  80. package/dist/client/config/conversationMemory.js +0 -86
  81. package/dist/client/config/taskClassificationConfig.js +0 -148
  82. package/dist/client/constants/contextWindows.js +0 -295
  83. package/dist/client/constants/enums.js +0 -853
  84. package/dist/client/constants/index.js +0 -207
  85. package/dist/client/constants/performance.js +0 -389
  86. package/dist/client/constants/retry.js +0 -266
  87. package/dist/client/constants/timeouts.js +0 -182
  88. package/dist/client/constants/tokens.js +0 -380
  89. package/dist/client/constants/videoErrors.js +0 -46
  90. package/dist/client/context/budgetChecker.js +0 -98
  91. package/dist/client/context/contextCompactor.js +0 -205
  92. package/dist/client/context/emergencyTruncation.js +0 -88
  93. package/dist/client/context/errorDetection.js +0 -171
  94. package/dist/client/context/errors.js +0 -21
  95. package/dist/client/context/fileTokenBudget.js +0 -127
  96. package/dist/client/context/prompts/summarizationPrompt.js +0 -117
  97. package/dist/client/context/stages/fileReadDeduplicator.js +0 -66
  98. package/dist/client/context/stages/slidingWindowTruncator.js +0 -190
  99. package/dist/client/context/stages/structuredSummarizer.js +0 -99
  100. package/dist/client/context/stages/toolOutputPruner.js +0 -52
  101. package/dist/client/context/summarizationEngine.js +0 -136
  102. package/dist/client/context/toolOutputLimits.js +0 -78
  103. package/dist/client/context/toolPairRepair.js +0 -66
  104. package/dist/client/core/analytics.js +0 -88
  105. package/dist/client/core/baseProvider.js +0 -1385
  106. package/dist/client/core/constants.js +0 -140
  107. package/dist/client/core/conversationMemoryFactory.js +0 -141
  108. package/dist/client/core/conversationMemoryInitializer.js +0 -128
  109. package/dist/client/core/conversationMemoryManager.js +0 -344
  110. package/dist/client/core/dynamicModels.js +0 -358
  111. package/dist/client/core/evaluation.js +0 -309
  112. package/dist/client/core/evaluationProviders.js +0 -248
  113. package/dist/client/core/factory.js +0 -412
  114. package/dist/client/core/infrastructure/baseError.js +0 -22
  115. package/dist/client/core/infrastructure/baseFactory.js +0 -54
  116. package/dist/client/core/infrastructure/baseRegistry.js +0 -53
  117. package/dist/client/core/infrastructure/index.js +0 -5
  118. package/dist/client/core/infrastructure/retry.js +0 -20
  119. package/dist/client/core/infrastructure/typedEventEmitter.js +0 -23
  120. package/dist/client/core/modelConfiguration.js +0 -851
  121. package/dist/client/core/modules/GenerationHandler.js +0 -588
  122. package/dist/client/core/modules/MessageBuilder.js +0 -273
  123. package/dist/client/core/modules/StreamHandler.js +0 -185
  124. package/dist/client/core/modules/TelemetryHandler.js +0 -203
  125. package/dist/client/core/modules/ToolsManager.js +0 -499
  126. package/dist/client/core/modules/Utilities.js +0 -331
  127. package/dist/client/core/redisConversationMemoryManager.js +0 -1435
  128. package/dist/client/core/streamAnalytics.js +0 -131
  129. package/dist/client/evaluation/contextBuilder.js +0 -134
  130. package/dist/client/evaluation/index.js +0 -61
  131. package/dist/client/evaluation/prompts.js +0 -73
  132. package/dist/client/evaluation/ragasEvaluator.js +0 -110
  133. package/dist/client/evaluation/retryManager.js +0 -78
  134. package/dist/client/evaluation/scoring.js +0 -61
  135. package/dist/client/factories/providerFactory.js +0 -166
  136. package/dist/client/factories/providerRegistry.js +0 -166
  137. package/dist/client/features/ppt/constants.js +0 -896
  138. package/dist/client/features/ppt/contentPlanner.js +0 -529
  139. package/dist/client/features/ppt/presentationOrchestrator.js +0 -236
  140. package/dist/client/features/ppt/slideGenerator.js +0 -532
  141. package/dist/client/features/ppt/slideRenderers.js +0 -2383
  142. package/dist/client/features/ppt/slideTypeInference.js +0 -405
  143. package/dist/client/features/ppt/types.js +0 -13
  144. package/dist/client/features/ppt/utils.js +0 -443
  145. package/dist/client/files/fileReferenceRegistry.js +0 -1543
  146. package/dist/client/files/fileTools.js +0 -450
  147. package/dist/client/files/streamingReader.js +0 -321
  148. package/dist/client/files/types.js +0 -23
  149. package/dist/client/hitl/hitlErrors.js +0 -54
  150. package/dist/client/hitl/hitlManager.js +0 -460
  151. package/dist/client/mcp/agentExposure.js +0 -356
  152. package/dist/client/mcp/auth/index.js +0 -11
  153. package/dist/client/mcp/auth/oauthClientProvider.js +0 -325
  154. package/dist/client/mcp/auth/tokenStorage.js +0 -134
  155. package/dist/client/mcp/batching/index.js +0 -10
  156. package/dist/client/mcp/batching/requestBatcher.js +0 -441
  157. package/dist/client/mcp/caching/index.js +0 -10
  158. package/dist/client/mcp/caching/toolCache.js +0 -433
  159. package/dist/client/mcp/elicitation/elicitationManager.js +0 -376
  160. package/dist/client/mcp/elicitation/index.js +0 -11
  161. package/dist/client/mcp/elicitation/types.js +0 -10
  162. package/dist/client/mcp/elicitationProtocol.js +0 -375
  163. package/dist/client/mcp/enhancedToolDiscovery.js +0 -481
  164. package/dist/client/mcp/externalServerManager.js +0 -1478
  165. package/dist/client/mcp/factory.js +0 -161
  166. package/dist/client/mcp/flexibleToolValidator.js +0 -161
  167. package/dist/client/mcp/httpRateLimiter.js +0 -391
  168. package/dist/client/mcp/httpRetryHandler.js +0 -178
  169. package/dist/client/mcp/index.js +0 -74
  170. package/dist/client/mcp/mcpCircuitBreaker.js +0 -427
  171. package/dist/client/mcp/mcpClientFactory.js +0 -708
  172. package/dist/client/mcp/mcpRegistryClient.js +0 -488
  173. package/dist/client/mcp/mcpServerBase.js +0 -373
  174. package/dist/client/mcp/multiServerManager.js +0 -579
  175. package/dist/client/mcp/registry.js +0 -158
  176. package/dist/client/mcp/routing/index.js +0 -10
  177. package/dist/client/mcp/routing/toolRouter.js +0 -416
  178. package/dist/client/mcp/serverCapabilities.js +0 -502
  179. package/dist/client/mcp/servers/agent/directToolsServer.js +0 -150
  180. package/dist/client/mcp/toolAnnotations.js +0 -239
  181. package/dist/client/mcp/toolConverter.js +0 -258
  182. package/dist/client/mcp/toolDiscoveryService.js +0 -798
  183. package/dist/client/mcp/toolIntegration.js +0 -334
  184. package/dist/client/mcp/toolRegistry.js +0 -729
  185. package/dist/client/memory/hippocampusInitializer.js +0 -19
  186. package/dist/client/memory/memoryRetrievalTools.js +0 -166
  187. package/dist/client/middleware/builtin/analytics.js +0 -132
  188. package/dist/client/middleware/builtin/autoEvaluation.js +0 -203
  189. package/dist/client/middleware/builtin/guardrails.js +0 -109
  190. package/dist/client/middleware/builtin/lifecycle.js +0 -168
  191. package/dist/client/middleware/factory.js +0 -327
  192. package/dist/client/middleware/registry.js +0 -295
  193. package/dist/client/middleware/utils/guardrailsUtils.js +0 -396
  194. package/dist/client/models/anthropicModels.js +0 -527
  195. package/dist/client/neurolink.js +0 -8233
  196. package/dist/client/observability/exporterRegistry.js +0 -413
  197. package/dist/client/observability/exporters/arizeExporter.js +0 -138
  198. package/dist/client/observability/exporters/baseExporter.js +0 -190
  199. package/dist/client/observability/exporters/braintrustExporter.js +0 -154
  200. package/dist/client/observability/exporters/datadogExporter.js +0 -196
  201. package/dist/client/observability/exporters/laminarExporter.js +0 -302
  202. package/dist/client/observability/exporters/langfuseExporter.js +0 -209
  203. package/dist/client/observability/exporters/langsmithExporter.js +0 -143
  204. package/dist/client/observability/exporters/otelExporter.js +0 -164
  205. package/dist/client/observability/exporters/posthogExporter.js +0 -287
  206. package/dist/client/observability/exporters/sentryExporter.js +0 -165
  207. package/dist/client/observability/index.js +0 -31
  208. package/dist/client/observability/metricsAggregator.js +0 -556
  209. package/dist/client/observability/otelBridge.js +0 -131
  210. package/dist/client/observability/retryPolicy.js +0 -383
  211. package/dist/client/observability/sampling/samplers.js +0 -216
  212. package/dist/client/observability/spanProcessor.js +0 -303
  213. package/dist/client/observability/tokenTracker.js +0 -413
  214. package/dist/client/observability/types/exporterTypes.js +0 -5
  215. package/dist/client/observability/types/index.js +0 -4
  216. package/dist/client/observability/types/spanTypes.js +0 -92
  217. package/dist/client/observability/utils/safeMetadata.js +0 -25
  218. package/dist/client/observability/utils/spanSerializer.js +0 -292
  219. package/dist/client/processors/archive/ArchiveProcessor.js +0 -1308
  220. package/dist/client/processors/base/BaseFileProcessor.js +0 -614
  221. package/dist/client/processors/base/types.js +0 -82
  222. package/dist/client/processors/config/fileTypes.js +0 -520
  223. package/dist/client/processors/config/index.js +0 -92
  224. package/dist/client/processors/config/languageMap.js +0 -410
  225. package/dist/client/processors/config/mimeTypes.js +0 -363
  226. package/dist/client/processors/config/sizeLimits.js +0 -258
  227. package/dist/client/processors/document/ExcelProcessor.js +0 -590
  228. package/dist/client/processors/document/OpenDocumentProcessor.js +0 -212
  229. package/dist/client/processors/document/PptxProcessor.js +0 -157
  230. package/dist/client/processors/document/RtfProcessor.js +0 -361
  231. package/dist/client/processors/document/WordProcessor.js +0 -353
  232. package/dist/client/processors/errors/FileErrorCode.js +0 -255
  233. package/dist/client/processors/errors/errorHelpers.js +0 -386
  234. package/dist/client/processors/errors/errorSerializer.js +0 -507
  235. package/dist/client/processors/errors/index.js +0 -49
  236. package/dist/client/processors/markup/SvgProcessor.js +0 -240
  237. package/dist/client/processors/media/AudioProcessor.js +0 -707
  238. package/dist/client/processors/media/VideoProcessor.js +0 -1045
  239. package/dist/client/providers/amazonBedrock.js +0 -1512
  240. package/dist/client/providers/amazonSagemaker.js +0 -162
  241. package/dist/client/providers/anthropic.js +0 -831
  242. package/dist/client/providers/azureOpenai.js +0 -143
  243. package/dist/client/providers/googleAiStudio.js +0 -1200
  244. package/dist/client/providers/googleNativeGemini3.js +0 -543
  245. package/dist/client/providers/googleVertex.js +0 -2936
  246. package/dist/client/providers/huggingFace.js +0 -315
  247. package/dist/client/providers/litellm.js +0 -488
  248. package/dist/client/providers/mistral.js +0 -157
  249. package/dist/client/providers/ollama.js +0 -1579
  250. package/dist/client/providers/openAI.js +0 -627
  251. package/dist/client/providers/openRouter.js +0 -543
  252. package/dist/client/providers/openaiCompatible.js +0 -290
  253. package/dist/client/providers/providerTypeUtils.js +0 -46
  254. package/dist/client/providers/sagemaker/adaptive-semaphore.js +0 -215
  255. package/dist/client/providers/sagemaker/client.js +0 -472
  256. package/dist/client/providers/sagemaker/config.js +0 -317
  257. package/dist/client/providers/sagemaker/detection.js +0 -606
  258. package/dist/client/providers/sagemaker/error-constants.js +0 -227
  259. package/dist/client/providers/sagemaker/errors.js +0 -299
  260. package/dist/client/providers/sagemaker/language-model.js +0 -775
  261. package/dist/client/providers/sagemaker/parsers.js +0 -634
  262. package/dist/client/providers/sagemaker/streaming.js +0 -331
  263. package/dist/client/providers/sagemaker/structured-parser.js +0 -625
  264. package/dist/client/proxy/accountQuota.js +0 -162
  265. package/dist/client/proxy/claudeFormat.js +0 -595
  266. package/dist/client/proxy/modelRouter.js +0 -29
  267. package/dist/client/proxy/oauthFetch.js +0 -367
  268. package/dist/client/proxy/proxyFetch.js +0 -586
  269. package/dist/client/proxy/requestLogger.js +0 -207
  270. package/dist/client/proxy/tokenRefresh.js +0 -124
  271. package/dist/client/proxy/usageStats.js +0 -74
  272. package/dist/client/proxy/utils/noProxyUtils.js +0 -149
  273. package/dist/client/rag/ChunkerFactory.js +0 -320
  274. package/dist/client/rag/ChunkerRegistry.js +0 -421
  275. package/dist/client/rag/chunkers/BaseChunker.js +0 -143
  276. package/dist/client/rag/chunkers/CharacterChunker.js +0 -28
  277. package/dist/client/rag/chunkers/HTMLChunker.js +0 -38
  278. package/dist/client/rag/chunkers/JSONChunker.js +0 -68
  279. package/dist/client/rag/chunkers/LaTeXChunker.js +0 -63
  280. package/dist/client/rag/chunkers/MarkdownChunker.js +0 -306
  281. package/dist/client/rag/chunkers/RecursiveChunker.js +0 -139
  282. package/dist/client/rag/chunkers/SemanticMarkdownChunker.js +0 -138
  283. package/dist/client/rag/chunkers/SentenceChunker.js +0 -66
  284. package/dist/client/rag/chunkers/TokenChunker.js +0 -61
  285. package/dist/client/rag/chunkers/index.js +0 -15
  286. package/dist/client/rag/chunking/characterChunker.js +0 -142
  287. package/dist/client/rag/chunking/chunkerRegistry.js +0 -194
  288. package/dist/client/rag/chunking/htmlChunker.js +0 -247
  289. package/dist/client/rag/chunking/index.js +0 -17
  290. package/dist/client/rag/chunking/jsonChunker.js +0 -281
  291. package/dist/client/rag/chunking/latexChunker.js +0 -251
  292. package/dist/client/rag/chunking/markdownChunker.js +0 -373
  293. package/dist/client/rag/chunking/recursiveChunker.js +0 -148
  294. package/dist/client/rag/chunking/semanticChunker.js +0 -306
  295. package/dist/client/rag/chunking/sentenceChunker.js +0 -230
  296. package/dist/client/rag/chunking/tokenChunker.js +0 -183
  297. package/dist/client/rag/document/MDocument.js +0 -392
  298. package/dist/client/rag/document/index.js +0 -5
  299. package/dist/client/rag/document/loaders.js +0 -500
  300. package/dist/client/rag/errors/RAGError.js +0 -274
  301. package/dist/client/rag/errors/index.js +0 -6
  302. package/dist/client/rag/graphRag/graphRAG.js +0 -401
  303. package/dist/client/rag/graphRag/index.js +0 -4
  304. package/dist/client/rag/index.js +0 -141
  305. package/dist/client/rag/metadata/MetadataExtractorFactory.js +0 -418
  306. package/dist/client/rag/metadata/MetadataExtractorRegistry.js +0 -362
  307. package/dist/client/rag/metadata/index.js +0 -9
  308. package/dist/client/rag/metadata/metadataExtractor.js +0 -280
  309. package/dist/client/rag/pipeline/RAGPipeline.js +0 -436
  310. package/dist/client/rag/pipeline/contextAssembly.js +0 -341
  311. package/dist/client/rag/pipeline/index.js +0 -5
  312. package/dist/client/rag/ragIntegration.js +0 -321
  313. package/dist/client/rag/reranker/RerankerFactory.js +0 -430
  314. package/dist/client/rag/reranker/RerankerRegistry.js +0 -402
  315. package/dist/client/rag/reranker/index.js +0 -9
  316. package/dist/client/rag/reranker/reranker.js +0 -277
  317. package/dist/client/rag/resilience/CircuitBreaker.js +0 -431
  318. package/dist/client/rag/resilience/RetryHandler.js +0 -304
  319. package/dist/client/rag/resilience/index.js +0 -7
  320. package/dist/client/rag/retrieval/hybridSearch.js +0 -335
  321. package/dist/client/rag/retrieval/index.js +0 -5
  322. package/dist/client/rag/retrieval/vectorQueryTool.js +0 -307
  323. package/dist/client/rag/types.js +0 -8
  324. package/dist/client/sdk/toolRegistration.js +0 -377
  325. package/dist/client/server/abstract/baseServerAdapter.js +0 -575
  326. package/dist/client/server/adapters/expressAdapter.js +0 -486
  327. package/dist/client/server/adapters/fastifyAdapter.js +0 -472
  328. package/dist/client/server/adapters/honoAdapter.js +0 -632
  329. package/dist/client/server/adapters/koaAdapter.js +0 -510
  330. package/dist/client/server/errors.js +0 -486
  331. package/dist/client/server/factory/serverAdapterFactory.js +0 -160
  332. package/dist/client/server/index.js +0 -108
  333. package/dist/client/server/middleware/abortSignal.js +0 -111
  334. package/dist/client/server/middleware/auth.js +0 -388
  335. package/dist/client/server/middleware/cache.js +0 -359
  336. package/dist/client/server/middleware/common.js +0 -281
  337. package/dist/client/server/middleware/deprecation.js +0 -190
  338. package/dist/client/server/middleware/mcpBodyAttachment.js +0 -63
  339. package/dist/client/server/middleware/rateLimit.js +0 -227
  340. package/dist/client/server/middleware/validation.js +0 -388
  341. package/dist/client/server/openapi/generator.js +0 -398
  342. package/dist/client/server/openapi/index.js +0 -36
  343. package/dist/client/server/openapi/schemas.js +0 -695
  344. package/dist/client/server/openapi/templates.js +0 -374
  345. package/dist/client/server/routes/agentRoutes.js +0 -189
  346. package/dist/client/server/routes/claudeProxyRoutes.js +0 -1600
  347. package/dist/client/server/routes/healthRoutes.js +0 -187
  348. package/dist/client/server/routes/index.js +0 -57
  349. package/dist/client/server/routes/mcpRoutes.js +0 -342
  350. package/dist/client/server/routes/memoryRoutes.js +0 -350
  351. package/dist/client/server/routes/openApiRoutes.js +0 -126
  352. package/dist/client/server/routes/toolRoutes.js +0 -199
  353. package/dist/client/server/streaming/dataStream.js +0 -486
  354. package/dist/client/server/streaming/index.js +0 -11
  355. package/dist/client/server/types.js +0 -67
  356. package/dist/client/server/utils/redaction.js +0 -334
  357. package/dist/client/server/utils/validation.js +0 -243
  358. package/dist/client/server/websocket/WebSocketHandler.js +0 -383
  359. package/dist/client/server/websocket/index.js +0 -4
  360. package/dist/client/services/server/ai/observability/instrumentation.js +0 -808
  361. package/dist/client/telemetry/attributes.js +0 -100
  362. package/dist/client/telemetry/index.js +0 -26
  363. package/dist/client/telemetry/telemetryService.js +0 -308
  364. package/dist/client/telemetry/tracers.js +0 -17
  365. package/dist/client/telemetry/withSpan.js +0 -34
  366. package/dist/client/types/actionTypes.js +0 -6
  367. package/dist/client/types/analytics.js +0 -5
  368. package/dist/client/types/authTypes.js +0 -9
  369. package/dist/client/types/circuitBreakerErrors.js +0 -34
  370. package/dist/client/types/cli.js +0 -21
  371. package/dist/client/types/clientTypes.js +0 -10
  372. package/dist/client/types/common.js +0 -51
  373. package/dist/client/types/configTypes.js +0 -49
  374. package/dist/client/types/content.js +0 -19
  375. package/dist/client/types/contextTypes.js +0 -400
  376. package/dist/client/types/conversation.js +0 -47
  377. package/dist/client/types/conversationMemoryInterface.js +0 -6
  378. package/dist/client/types/domainTypes.js +0 -5
  379. package/dist/client/types/errors.js +0 -167
  380. package/dist/client/types/evaluation.js +0 -5
  381. package/dist/client/types/evaluationProviders.js +0 -5
  382. package/dist/client/types/evaluationTypes.js +0 -1
  383. package/dist/client/types/externalMcp.js +0 -6
  384. package/dist/client/types/fileReferenceTypes.js +0 -8
  385. package/dist/client/types/fileTypes.js +0 -4
  386. package/dist/client/types/generateTypes.js +0 -1
  387. package/dist/client/types/guardrails.js +0 -1
  388. package/dist/client/types/hitlTypes.js +0 -8
  389. package/dist/client/types/index.js +0 -57
  390. package/dist/client/types/mcpTypes.js +0 -5
  391. package/dist/client/types/middlewareTypes.js +0 -1
  392. package/dist/client/types/modelTypes.js +0 -30
  393. package/dist/client/types/multimodal.js +0 -135
  394. package/dist/client/types/observability.js +0 -6
  395. package/dist/client/types/pptTypes.js +0 -82
  396. package/dist/client/types/providers.js +0 -111
  397. package/dist/client/types/proxyTypes.js +0 -16
  398. package/dist/client/types/ragTypes.js +0 -7
  399. package/dist/client/types/sdkTypes.js +0 -8
  400. package/dist/client/types/serviceTypes.js +0 -5
  401. package/dist/client/types/streamTypes.js +0 -1
  402. package/dist/client/types/subscriptionTypes.js +0 -9
  403. package/dist/client/types/taskClassificationTypes.js +0 -5
  404. package/dist/client/types/tools.js +0 -24
  405. package/dist/client/types/ttsTypes.js +0 -57
  406. package/dist/client/types/typeAliases.js +0 -48
  407. package/dist/client/types/utilities.js +0 -4
  408. package/dist/client/types/workflowTypes.js +0 -30
  409. package/dist/client/utils/async/withTimeout.js +0 -98
  410. package/dist/client/utils/asyncMutex.js +0 -60
  411. package/dist/client/utils/conversationMemory.js +0 -431
  412. package/dist/client/utils/csvProcessor.js +0 -846
  413. package/dist/client/utils/errorHandling.js +0 -936
  414. package/dist/client/utils/evaluationUtils.js +0 -131
  415. package/dist/client/utils/factoryProcessing.js +0 -589
  416. package/dist/client/utils/fileDetector.js +0 -2161
  417. package/dist/client/utils/imageCache.js +0 -376
  418. package/dist/client/utils/imageProcessor.js +0 -704
  419. package/dist/client/utils/logger.js +0 -491
  420. package/dist/client/utils/mcpDefaults.js +0 -134
  421. package/dist/client/utils/messageBuilder.js +0 -1653
  422. package/dist/client/utils/modelAliasResolver.js +0 -54
  423. package/dist/client/utils/modelDetection.js +0 -80
  424. package/dist/client/utils/modelRouter.js +0 -292
  425. package/dist/client/utils/multimodalOptionsBuilder.js +0 -65
  426. package/dist/client/utils/observabilityHelpers.js +0 -47
  427. package/dist/client/utils/parameterValidation.js +0 -966
  428. package/dist/client/utils/pdfProcessor.js +0 -410
  429. package/dist/client/utils/performance.js +0 -222
  430. package/dist/client/utils/pricing.js +0 -340
  431. package/dist/client/utils/promptRedaction.js +0 -62
  432. package/dist/client/utils/providerConfig.js +0 -1009
  433. package/dist/client/utils/providerHealth.js +0 -1237
  434. package/dist/client/utils/providerRetry.js +0 -112
  435. package/dist/client/utils/providerUtils.js +0 -434
  436. package/dist/client/utils/rateLimiter.js +0 -200
  437. package/dist/client/utils/redis.js +0 -368
  438. package/dist/client/utils/retryHandler.js +0 -269
  439. package/dist/client/utils/retryability.js +0 -22
  440. package/dist/client/utils/sanitizers/svg.js +0 -481
  441. package/dist/client/utils/schemaConversion.js +0 -255
  442. package/dist/client/utils/taskClassificationUtils.js +0 -149
  443. package/dist/client/utils/taskClassifier.js +0 -94
  444. package/dist/client/utils/thinkingConfig.js +0 -104
  445. package/dist/client/utils/timeout.js +0 -359
  446. package/dist/client/utils/tokenEstimation.js +0 -142
  447. package/dist/client/utils/tokenLimits.js +0 -125
  448. package/dist/client/utils/tokenUtils.js +0 -239
  449. package/dist/client/utils/toolUtils.js +0 -75
  450. package/dist/client/utils/transformationUtils.js +0 -554
  451. package/dist/client/utils/ttsProcessor.js +0 -286
  452. package/dist/client/utils/typeUtils.js +0 -97
  453. package/dist/client/utils/videoAnalysisProcessor.js +0 -67
  454. package/dist/client/workflow/config.js +0 -398
  455. package/dist/client/workflow/core/ensembleExecutor.js +0 -407
  456. package/dist/client/workflow/core/judgeScorer.js +0 -544
  457. package/dist/client/workflow/core/responseConditioner.js +0 -225
  458. package/dist/client/workflow/core/types/conditionerTypes.js +0 -7
  459. package/dist/client/workflow/core/types/ensembleTypes.js +0 -7
  460. package/dist/client/workflow/core/types/index.js +0 -7
  461. package/dist/client/workflow/core/types/judgeTypes.js +0 -7
  462. package/dist/client/workflow/core/types/layerTypes.js +0 -7
  463. package/dist/client/workflow/core/types/registryTypes.js +0 -7
  464. package/dist/client/workflow/core/workflowRegistry.js +0 -304
  465. package/dist/client/workflow/core/workflowRunner.js +0 -586
  466. package/dist/client/workflow/index.js +0 -50
  467. package/dist/client/workflow/types.js +0 -9
  468. package/dist/client/workflow/utils/types/index.js +0 -7
  469. package/dist/client/workflow/utils/workflowMetrics.js +0 -311
  470. package/dist/client/workflow/utils/workflowValidation.js +0 -420
  471. package/dist/client/workflow/workflows/adaptiveWorkflow.js +0 -366
  472. package/dist/client/workflow/workflows/consensusWorkflow.js +0 -192
  473. package/dist/client/workflow/workflows/fallbackWorkflow.js +0 -225
  474. package/dist/client/workflow/workflows/multiJudgeWorkflow.js +0 -351
  475. /package/dist/client/{client/reactHooks.js → reactHooks.js} +0 -0
@@ -1,1200 +0,0 @@
1
- import { createGoogleGenerativeAI } from "@ai-sdk/google";
2
- import { embed, embedMany, stepCountIs, streamText, } from "ai";
3
- import { ErrorCategory, ErrorSeverity, GoogleAIModels, } from "../constants/enums.js";
4
- import { BaseProvider } from "../core/baseProvider.js";
5
- import { DEFAULT_MAX_STEPS } from "../core/constants.js";
6
- import { streamAnalyticsCollector } from "../core/streamAnalytics.js";
7
- import { ATTR, tracers, withClientSpan } from "../telemetry/index.js";
8
- import { AuthenticationError, NetworkError, ProviderError, RateLimitError, } from "../types/errors.js";
9
- import { ERROR_CODES, NeuroLinkError } from "../utils/errorHandling.js";
10
- import { logger } from "../utils/logger.js";
11
- import { isGemini3Model } from "../utils/modelDetection.js";
12
- import { composeAbortSignals, createTimeoutController, TimeoutError, } from "../utils/timeout.js";
13
- import { estimateTokens } from "../utils/tokenEstimation.js";
14
- import { buildNativeConfig, buildNativeToolDeclarations, collectStreamChunks, collectStreamChunksIncremental, computeMaxSteps, createTextChannel, executeNativeToolCalls, extractTextFromParts, handleMaxStepsTermination, pushModelResponseToHistory, sanitizeToolsForGemini, } from "./googleNativeGemini3.js";
15
- import { toAnalyticsStreamResult } from "./providerTypeUtils.js";
16
- // Google AI Live API types now imported from ../types/providerSpecific.js
17
- // Import proper types for multimodal message handling
18
- // Create Google GenAI client
19
- async function createGoogleGenAIClient(apiKey) {
20
- const mod = await import("@google/genai");
21
- const ctor = mod.GoogleGenAI;
22
- if (!ctor) {
23
- throw new NeuroLinkError({
24
- code: ERROR_CODES.INVALID_CONFIGURATION,
25
- message: "@google/genai does not export GoogleGenAI",
26
- category: ErrorCategory.CONFIGURATION,
27
- severity: ErrorSeverity.CRITICAL,
28
- retriable: false,
29
- context: { module: "@google/genai", expectedExport: "GoogleGenAI" },
30
- });
31
- }
32
- const Ctor = ctor;
33
- return new Ctor({ apiKey });
34
- }
35
- // Environment variable setup
36
- if (!process.env.GOOGLE_GENERATIVE_AI_API_KEY &&
37
- process.env.GOOGLE_AI_API_KEY) {
38
- process.env.GOOGLE_GENERATIVE_AI_API_KEY = process.env.GOOGLE_AI_API_KEY;
39
- }
40
- /**
41
- * Google AI Studio provider implementation using BaseProvider
42
- * Migrated from original GoogleAIStudio class to new factory pattern
43
- *
44
- * @important Structured Output Limitation
45
- * Google Gemini models cannot combine function calling (tools) with structured
46
- * output (JSON schema). When using schemas with output.format: "json", you MUST
47
- * set disableTools: true.
48
- *
49
- * Error without disableTools:
50
- * "Function calling with a response mime type: 'application/json' is unsupported"
51
- *
52
- * This is a Google API limitation documented at:
53
- * https://ai.google.dev/gemini-api/docs/function-calling
54
- *
55
- * @example
56
- * ```typescript
57
- * // ✅ Correct usage with schemas
58
- * const provider = new GoogleAIStudioProvider("gemini-2.5-flash");
59
- * const result = await provider.generate({
60
- * input: { text: "Analyze data" },
61
- * schema: MySchema,
62
- * output: { format: "json" },
63
- * disableTools: true // Required
64
- * });
65
- * ```
66
- *
67
- * @note Gemini 3 Pro Preview (November 2025) will support combining tools + schemas
68
- * @note "Too many states for serving" errors can occur with complex schemas + tools.
69
- * Solution: Simplify schema or use disableTools: true
70
- */
71
- export class GoogleAIStudioProvider extends BaseProvider {
72
- constructor(modelName, sdk) {
73
- super(modelName, "google-ai", sdk);
74
- logger.debug("GoogleAIStudioProvider initialized", {
75
- model: this.modelName,
76
- provider: this.providerName,
77
- sdkProvided: !!sdk,
78
- });
79
- }
80
- // ===================
81
- // ABSTRACT METHOD IMPLEMENTATIONS
82
- // ===================
83
- getProviderName() {
84
- return "google-ai";
85
- }
86
- getDefaultModel() {
87
- return process.env.GOOGLE_AI_MODEL || GoogleAIModels.GEMINI_2_5_FLASH;
88
- }
89
- /**
90
- * 🔧 PHASE 2: Return AI SDK model instance for tool calling
91
- */
92
- getAISDKModel() {
93
- const apiKey = this.getApiKey();
94
- const google = createGoogleGenerativeAI({ apiKey });
95
- return google(this.modelName);
96
- }
97
- formatProviderError(error) {
98
- if (error instanceof TimeoutError) {
99
- return new NetworkError(error.message, this.providerName);
100
- }
101
- const errorRecord = error;
102
- const message = typeof errorRecord?.message === "string"
103
- ? errorRecord.message
104
- : "Unknown error";
105
- if (message.includes("API_KEY_INVALID")) {
106
- return new AuthenticationError("Invalid Google AI API key. Please check your GOOGLE_AI_API_KEY environment variable.", this.providerName);
107
- }
108
- if (message.includes("RATE_LIMIT_EXCEEDED")) {
109
- return new RateLimitError("Google AI rate limit exceeded. Please try again later.", this.providerName);
110
- }
111
- return new ProviderError(`Google AI error: ${message}`, this.providerName);
112
- }
113
- /**
114
- * Overrides the BaseProvider's image generation method to implement it for Google AI.
115
- * This method calls the Google AI API to generate an image from a prompt.
116
- * @param options The generation options containing the prompt.
117
- * @returns A promise that resolves to the generation result, including the image data.
118
- */
119
- async executeImageGeneration(options) {
120
- const prompt = options.prompt || options.input?.text || "";
121
- const imageModelName = options.model || this.modelName;
122
- const startTime = Date.now();
123
- const apiKey = this.getApiKey();
124
- logger.info("🎨 Starting Google AI Studio image generation", {
125
- model: imageModelName,
126
- prompt: prompt.substring(0, 100),
127
- provider: this.providerName,
128
- });
129
- // Use the @google/genai client for image generation
130
- let client;
131
- try {
132
- client = await createGoogleGenAIClient(apiKey);
133
- }
134
- catch {
135
- throw new AuthenticationError("Missing '@google/genai'. Install with: npm install @google/genai", this.providerName);
136
- }
137
- try {
138
- // Build content array with multimodal support
139
- const imageParts = await Promise.all((options.input?.images || []).map(async (image) => {
140
- // Handle ImageWithAltText objects
141
- if (typeof image === "object" && "url" in image) {
142
- const imageUrl = image.url;
143
- if (imageUrl.startsWith("http")) {
144
- const response = await fetch(imageUrl);
145
- if (!response.ok) {
146
- throw new Error(`Failed to fetch image from ${imageUrl}: ${response.status} ${response.statusText}`);
147
- }
148
- const arrayBuffer = await response.arrayBuffer();
149
- const buffer = Buffer.from(arrayBuffer);
150
- const mimeType = this.detectImageType(buffer);
151
- logger.debug(`Downloaded and detected image MIME type: ${mimeType}`);
152
- return {
153
- inlineData: {
154
- mimeType,
155
- data: buffer.toString("base64"),
156
- },
157
- };
158
- }
159
- // Base64 URL in ImageWithAltText
160
- const buffer = Buffer.from(imageUrl, "base64");
161
- const mimeType = this.detectImageType(buffer);
162
- return {
163
- inlineData: {
164
- mimeType,
165
- data: buffer.toString("base64"),
166
- },
167
- };
168
- }
169
- // Handle string URLs
170
- if (typeof image === "string" && image.startsWith("http")) {
171
- const response = await fetch(image);
172
- if (!response.ok) {
173
- throw new Error(`Failed to fetch image from ${image}: ${response.status} ${response.statusText}`);
174
- }
175
- const arrayBuffer = await response.arrayBuffer();
176
- const buffer = Buffer.from(arrayBuffer);
177
- const mimeType = this.detectImageType(buffer);
178
- logger.debug(`Downloaded and detected image MIME type: ${mimeType}`);
179
- return {
180
- inlineData: {
181
- mimeType,
182
- data: buffer.toString("base64"),
183
- },
184
- };
185
- }
186
- // Handle Buffer or base64 string
187
- const buffer = Buffer.isBuffer(image)
188
- ? image
189
- : typeof image === "string"
190
- ? Buffer.from(image, "base64")
191
- : Buffer.from(""); // Fallback for unexpected types
192
- const mimeType = this.detectImageType(buffer);
193
- logger.debug(`Detected image MIME type: ${mimeType}`);
194
- return {
195
- inlineData: {
196
- mimeType,
197
- data: buffer.toString("base64"),
198
- },
199
- };
200
- }));
201
- const contents = [
202
- {
203
- role: "user",
204
- parts: [{ text: prompt }, ...imageParts],
205
- },
206
- ];
207
- // Configure for image generation
208
- const generateConfig = {
209
- responseModalities: ["IMAGE", "TEXT"], // This is the key setting for image generation
210
- };
211
- logger.debug("Starting image generation request", {
212
- model: imageModelName,
213
- contentParts: contents[0].parts.length,
214
- responseModalities: generateConfig.responseModalities,
215
- });
216
- // Try streaming approach first
217
- let imageData = null;
218
- let textContent = "";
219
- try {
220
- // Await the Promise to get the AsyncIterable
221
- const stream = await client.models.generateContentStream({
222
- model: imageModelName,
223
- contents: contents,
224
- config: generateConfig,
225
- });
226
- // Process the stream
227
- for await (const chunk of stream) {
228
- logger.debug("Received chunk", {
229
- hasCandidate: !!chunk.candidates?.[0],
230
- hasContent: !!chunk.candidates?.[0]?.content,
231
- hasParts: !!chunk.candidates?.[0]?.content?.parts,
232
- });
233
- const candidate = chunk.candidates?.[0];
234
- if (candidate?.content?.parts) {
235
- for (const part of candidate.content.parts) {
236
- // Check for image data
237
- if ("inlineData" in part && part.inlineData?.data) {
238
- const foundImageData = part.inlineData.data;
239
- imageData = foundImageData;
240
- const mimeType = part.inlineData.mimeType || "image/png";
241
- logger.info("Image generation successful", {
242
- model: imageModelName,
243
- mimeType,
244
- dataLength: foundImageData.length,
245
- responseTime: Date.now() - startTime,
246
- });
247
- const result = {
248
- content: `Generated image using ${imageModelName} (${mimeType})`,
249
- imageOutput: {
250
- base64: foundImageData,
251
- },
252
- provider: this.providerName,
253
- model: imageModelName,
254
- usage: {
255
- input: this.estimateTokenCount(prompt),
256
- output: 0,
257
- total: this.estimateTokenCount(prompt),
258
- },
259
- };
260
- return await this.enhanceResult(result, options, startTime);
261
- }
262
- // Check for text content
263
- if ("text" in part && part.text) {
264
- textContent += part.text;
265
- logger.debug("Received text content", {
266
- text: part.text.substring(0, 100),
267
- });
268
- }
269
- }
270
- }
271
- }
272
- }
273
- catch (streamError) {
274
- logger.debug("Streaming failed, trying non-streaming approach", {
275
- error: streamError instanceof Error
276
- ? streamError.message
277
- : String(streamError),
278
- });
279
- }
280
- // If no image was found, try non-streaming approach
281
- if (!imageData) {
282
- logger.debug("Trying non-streaming approach");
283
- const response = await client.models.generateContent({
284
- model: imageModelName,
285
- contents: contents,
286
- config: generateConfig,
287
- });
288
- const candidate = response.candidates?.[0];
289
- if (candidate?.content?.parts) {
290
- for (const part of candidate.content.parts) {
291
- if ("inlineData" in part && part.inlineData?.data) {
292
- const foundImageData = part.inlineData.data;
293
- imageData = foundImageData;
294
- const mimeType = part.inlineData.mimeType || "image/png";
295
- logger.info("Image generation successful (non-streaming)", {
296
- model: imageModelName,
297
- mimeType,
298
- dataLength: foundImageData.length,
299
- responseTime: Date.now() - startTime,
300
- });
301
- const result = {
302
- content: `Generated image using ${imageModelName} (${mimeType})`,
303
- imageOutput: {
304
- base64: foundImageData,
305
- },
306
- provider: this.providerName,
307
- model: imageModelName,
308
- usage: {
309
- input: this.estimateTokenCount(prompt),
310
- output: 0,
311
- total: this.estimateTokenCount(prompt),
312
- },
313
- };
314
- return await this.enhanceResult(result, options, startTime);
315
- }
316
- if ("text" in part && part.text) {
317
- textContent += part.text;
318
- }
319
- }
320
- }
321
- }
322
- // If we reach here, no image was generated
323
- logger.warn("No image data found in response", {
324
- model: imageModelName,
325
- prompt: prompt.substring(0, 100),
326
- hasTextContent: !!textContent,
327
- textContent: textContent.substring(0, 200),
328
- });
329
- throw new ProviderError(textContent ||
330
- `Image generation completed but no image data was returned. This may indicate an issue with the model "${imageModelName}" or the prompt: "${prompt}". Please try again or use a different model.`, this.providerName);
331
- }
332
- catch (error) {
333
- logger.error("Image generation failed", {
334
- error: error instanceof Error ? error.message : String(error),
335
- model: imageModelName,
336
- prompt: prompt.substring(0, 100),
337
- });
338
- throw this.handleProviderError(error);
339
- }
340
- }
341
- /**
342
- * Detect image MIME type from buffer
343
- */
344
- detectImageType(buffer) {
345
- // Check PNG signature
346
- if (buffer.length >= 8 &&
347
- buffer[0] === 0x89 &&
348
- buffer[1] === 0x50 &&
349
- buffer[2] === 0x4e &&
350
- buffer[3] === 0x47) {
351
- return "image/png";
352
- }
353
- // Check JPEG signature
354
- if (buffer.length >= 3 &&
355
- buffer[0] === 0xff &&
356
- buffer[1] === 0xd8 &&
357
- buffer[2] === 0xff) {
358
- return "image/jpeg";
359
- }
360
- // Check WebP signature
361
- if (buffer.length >= 12 &&
362
- buffer[0] === 0x52 &&
363
- buffer[1] === 0x49 &&
364
- buffer[2] === 0x46 &&
365
- buffer[3] === 0x46 &&
366
- buffer[8] === 0x57 &&
367
- buffer[9] === 0x45 &&
368
- buffer[10] === 0x42 &&
369
- buffer[11] === 0x50) {
370
- return "image/webp";
371
- }
372
- // Check GIF signature
373
- if (buffer.length >= 6 &&
374
- buffer[0] === 0x47 &&
375
- buffer[1] === 0x49 &&
376
- buffer[2] === 0x46) {
377
- return "image/gif";
378
- }
379
- // Default to PNG if unknown
380
- return "image/png";
381
- }
382
- /**
383
- * Estimate token count from text using centralized estimation with provider multipliers
384
- */
385
- estimateTokenCount(text) {
386
- return estimateTokens(text, "google-ai");
387
- }
388
- // executeGenerate removed - BaseProvider handles all generation with tools
389
- async executeStream(options, analysisSchema) {
390
- // Check if this is a Gemini 3 model with tools - use native SDK for thought_signature
391
- const gemini3CheckModelName = options.model || this.modelName;
392
- // Structured output (analysisSchema, JSON format, or schema) is incompatible with tools on Gemini.
393
- // Compute once and reuse in both the native Gemini 3 gate and the streamText fallback path.
394
- const wantsStructuredOutput = analysisSchema || options.output?.format === "json" || options.schema;
395
- // Check for tools from options AND from SDK (MCP tools)
396
- // Need to check early if we should route to native SDK
397
- const gemini3CheckShouldUseTools = !options.disableTools && this.supportsTools() && !wantsStructuredOutput;
398
- const optionTools = options.tools || {};
399
- const sdkTools = gemini3CheckShouldUseTools ? await this.getAllTools() : {};
400
- const combinedToolCount = Object.keys(optionTools).length + Object.keys(sdkTools).length;
401
- const hasTools = gemini3CheckShouldUseTools && combinedToolCount > 0;
402
- if (isGemini3Model(gemini3CheckModelName) && hasTools) {
403
- // Merge SDK tools into options for native SDK path
404
- let mergedOptions = {
405
- ...options,
406
- tools: { ...sdkTools, ...optionTools },
407
- };
408
- // Check for tools + JSON schema conflict (Gemini limitation)
409
- const wantsJsonOutput = options.output?.format === "json" || options.schema;
410
- if (wantsJsonOutput &&
411
- mergedOptions.tools &&
412
- Object.keys(mergedOptions.tools).length > 0 &&
413
- !mergedOptions.disableTools) {
414
- logger.warn("[GoogleAIStudio] Gemini does not support tools and JSON schema output simultaneously. Disabling tools for this request.");
415
- mergedOptions = { ...mergedOptions, disableTools: true, tools: {} };
416
- }
417
- // Only route to native path if tools are still active after conflict check
418
- const hasActiveTools = !mergedOptions.disableTools &&
419
- mergedOptions.tools &&
420
- Object.keys(mergedOptions.tools).length > 0;
421
- if (hasActiveTools) {
422
- logger.info("[GoogleAIStudio] Routing Gemini 3 to native SDK for tool calling", {
423
- model: gemini3CheckModelName,
424
- optionToolCount: Object.keys(optionTools).length,
425
- sdkToolCount: Object.keys(sdkTools).length,
426
- totalToolCount: combinedToolCount,
427
- });
428
- return this.executeNativeGemini3Stream(mergedOptions);
429
- }
430
- // Fall through to standard stream path using merged options (tools disabled for schema)
431
- options = mergedOptions;
432
- }
433
- // Phase 1: if audio input present, bridge to Gemini Live (Studio) using @google/genai
434
- if (options.input?.audio) {
435
- return await this.executeAudioStreamViaGeminiLive(options);
436
- }
437
- this.validateStreamOptions(options);
438
- const startTime = Date.now();
439
- const apiKey = this.getApiKey();
440
- // Ensure environment variable is set for @ai-sdk/google
441
- if (!process.env.GOOGLE_GENERATIVE_AI_API_KEY) {
442
- process.env.GOOGLE_GENERATIVE_AI_API_KEY = apiKey;
443
- }
444
- const model = await this.getAISDKModelWithMiddleware(options);
445
- const timeout = this.getTimeout(options);
446
- const timeoutController = createTimeoutController(timeout, this.providerName, "stream");
447
- try {
448
- // Get tools consistently with generate method (include user-provided RAG tools)
449
- // wantsStructuredOutput already computed before the Gemini 3 native-routing gate
450
- if (wantsStructuredOutput &&
451
- !options.disableTools &&
452
- this.supportsTools()) {
453
- logger.warn("[GoogleAIStudio] Structured output active — disabling tools (Gemini limitation).");
454
- }
455
- const shouldUseTools = !options.disableTools && this.supportsTools() && !wantsStructuredOutput;
456
- const baseTools = shouldUseTools ? await this.getAllTools() : {};
457
- const rawTools = shouldUseTools
458
- ? { ...baseTools, ...(options.tools || {}) }
459
- : {};
460
- // Sanitize tool schemas for Gemini proto compatibility (converts anyOf/oneOf unions to string)
461
- let tools;
462
- if (Object.keys(rawTools).length > 0) {
463
- const sanitized = sanitizeToolsForGemini(rawTools);
464
- if (sanitized.dropped.length > 0) {
465
- logger.warn(`[GoogleAIStudio] Dropped ${sanitized.dropped.length} incompatible tool(s): ${sanitized.dropped.join(", ")}`);
466
- }
467
- tools =
468
- Object.keys(sanitized.tools).length > 0 ? sanitized.tools : undefined;
469
- }
470
- else {
471
- tools = undefined;
472
- }
473
- // Build message array from options with multimodal support
474
- // Using protected helper from BaseProvider to eliminate code duplication
475
- const messages = await this.buildMessagesForStream(options);
476
- const result = await streamText({
477
- model,
478
- messages: messages,
479
- temperature: options.temperature,
480
- maxOutputTokens: options.maxTokens, // No default limit - unlimited unless specified
481
- tools,
482
- stopWhen: stepCountIs(options.maxSteps || DEFAULT_MAX_STEPS),
483
- toolChoice: shouldUseTools ? "auto" : "none",
484
- abortSignal: composeAbortSignals(options.abortSignal, timeoutController?.controller.signal),
485
- experimental_telemetry: this.telemetryHandler.getTelemetryConfig(options),
486
- // Gemini 3: use thinkingLevel via providerOptions
487
- // Gemini 2.5: use thinkingBudget via providerOptions
488
- ...(options.thinkingConfig?.enabled && {
489
- providerOptions: {
490
- google: {
491
- thinkingConfig: {
492
- ...(options.thinkingConfig.thinkingLevel && {
493
- thinkingLevel: options.thinkingConfig.thinkingLevel,
494
- }),
495
- ...(options.thinkingConfig.budgetTokens &&
496
- !options.thinkingConfig.thinkingLevel && {
497
- thinkingBudget: options.thinkingConfig.budgetTokens,
498
- }),
499
- includeThoughts: true,
500
- },
501
- },
502
- },
503
- }),
504
- onStepFinish: ({ toolCalls, toolResults }) => {
505
- this.handleToolExecutionStorage(toolCalls, toolResults, options, new Date()).catch((error) => {
506
- logger.warn("[GoogleAiStudioProvider] Failed to store tool executions", {
507
- provider: this.providerName,
508
- error: error instanceof Error ? error.message : String(error),
509
- });
510
- });
511
- },
512
- });
513
- // Defer timeout cleanup until the stream completes or errors.
514
- // Guard against NoOutputGeneratedError becoming an unhandled rejection.
515
- Promise.resolve(result.text)
516
- .catch((err) => {
517
- logger.debug("Stream text promise rejected (expected for empty streams)", {
518
- error: err instanceof Error ? err.message : String(err),
519
- });
520
- })
521
- .finally(() => timeoutController?.cleanup());
522
- // Transform string stream to content object stream using BaseProvider method
523
- const transformedStream = this.createTextStream(result);
524
- // Create analytics promise that resolves after stream completion
525
- const analyticsPromise = streamAnalyticsCollector.createAnalytics(this.providerName, this.modelName, toAnalyticsStreamResult(result), Date.now() - startTime, {
526
- requestId: `google-ai-stream-${Date.now()}`,
527
- streamingMode: true,
528
- });
529
- return {
530
- stream: transformedStream,
531
- provider: this.providerName,
532
- model: this.modelName,
533
- analytics: analyticsPromise,
534
- metadata: {
535
- startTime,
536
- streamId: `google-ai-${Date.now()}`,
537
- },
538
- };
539
- }
540
- catch (error) {
541
- timeoutController?.cleanup();
542
- throw this.handleProviderError(error);
543
- }
544
- }
545
- /**
546
- * Execute stream using native @google/genai SDK for Gemini 3 models
547
- * This bypasses @ai-sdk/google to properly handle thought_signature
548
- */
549
- async executeNativeGemini3Stream(options) {
550
- const modelName = options.model || this.modelName;
551
- return withClientSpan({
552
- name: "neurolink.provider.stream",
553
- tracer: tracers.provider,
554
- attributes: {
555
- [ATTR.GEN_AI_SYSTEM]: "google-ai",
556
- [ATTR.GEN_AI_MODEL]: modelName,
557
- [ATTR.GEN_AI_OPERATION]: "stream",
558
- [ATTR.NL_PROVIDER]: this.providerName,
559
- },
560
- }, async (span) => {
561
- const startTime = Date.now();
562
- const timeout = this.getTimeout(options);
563
- const timeoutController = createTimeoutController(timeout, this.providerName, "stream");
564
- try {
565
- const apiKey = this.getApiKey();
566
- const client = await createGoogleGenAIClient(apiKey);
567
- logger.debug("[GoogleAIStudio] Using native @google/genai for Gemini 3", {
568
- model: modelName,
569
- hasTools: !!options.tools && Object.keys(options.tools).length > 0,
570
- });
571
- // Build contents from input
572
- const currentContents = [{ role: "user", parts: [{ text: options.input.text }] }];
573
- // Convert tools
574
- let toolsConfig;
575
- let executeMap = new Map();
576
- if (options.tools &&
577
- Object.keys(options.tools).length > 0 &&
578
- !options.disableTools) {
579
- const result = buildNativeToolDeclarations(options.tools);
580
- toolsConfig = result.toolsConfig;
581
- executeMap = result.executeMap;
582
- logger.debug("[GoogleAIStudio] Converted tools for native SDK", {
583
- toolCount: toolsConfig[0].functionDeclarations.length,
584
- toolNames: toolsConfig[0].functionDeclarations.map((t) => t.name),
585
- });
586
- }
587
- const config = buildNativeConfig(options, toolsConfig);
588
- const maxSteps = computeMaxSteps(options.maxSteps);
589
- // Compose abort signal from user signal + timeout
590
- const composedSignal = composeAbortSignals(options.abortSignal, timeoutController?.controller.signal);
591
- // Create a push-based text channel so the caller receives tokens as
592
- // they arrive from the network rather than after full buffering.
593
- const channel = createTextChannel();
594
- // Shared mutable state updated by the background agentic loop.
595
- const allToolCalls = [];
596
- // analyticsResolvers lets the background loop settle the analytics
597
- // promise once token counts are known (after the loop completes).
598
- let analyticsResolve;
599
- let analyticsReject;
600
- const analyticsPromise = new Promise((res, rej) => {
601
- analyticsResolve = res;
602
- analyticsReject = rej;
603
- });
604
- // Shared metadata object mutated by the background loop so the
605
- // returned object reflects the final values after stream completion.
606
- const metadata = {
607
- streamId: `native-${Date.now()}`,
608
- startTime,
609
- responseTime: 0,
610
- totalToolExecutions: 0,
611
- };
612
- // Run the agentic loop in the background without awaiting it here,
613
- // so we can return the StreamResult (with channel.iterable) immediately.
614
- const loopPromise = (async () => {
615
- let lastStepText = "";
616
- let totalInputTokens = 0;
617
- let totalOutputTokens = 0;
618
- let step = 0;
619
- let completedWithFinalAnswer = false;
620
- const failedTools = new Map();
621
- try {
622
- // Agentic loop for tool calling
623
- while (step < maxSteps) {
624
- if (composedSignal?.aborted) {
625
- throw composedSignal.reason instanceof Error
626
- ? composedSignal.reason
627
- : new Error("Request aborted");
628
- }
629
- step++;
630
- logger.debug(`[GoogleAIStudio] Native SDK step ${step}/${maxSteps}`);
631
- try {
632
- const rawStream = await client.models.generateContentStream({
633
- model: modelName,
634
- contents: currentContents,
635
- config,
636
- ...(composedSignal
637
- ? { httpOptions: { signal: composedSignal } }
638
- : {}),
639
- });
640
- // For every step, use incremental collection so text parts
641
- // are pushed to the channel as they arrive. For intermediate
642
- // steps (those that produce function calls) we still need the
643
- // complete rawResponseParts for pushModelResponseToHistory,
644
- // which collectStreamChunksIncremental provides at stream end.
645
- const chunkResult = await collectStreamChunksIncremental(rawStream, channel);
646
- totalInputTokens += chunkResult.inputTokens;
647
- totalOutputTokens += chunkResult.outputTokens;
648
- const stepText = extractTextFromParts(chunkResult.rawResponseParts);
649
- // If no function calls, this was the final step — channel
650
- // already received all text parts incrementally.
651
- if (chunkResult.stepFunctionCalls.length === 0) {
652
- completedWithFinalAnswer = true;
653
- break;
654
- }
655
- lastStepText = stepText;
656
- // Record tool call events on the span
657
- for (const fc of chunkResult.stepFunctionCalls) {
658
- span.addEvent("gen_ai.tool_call", {
659
- "tool.name": fc.name,
660
- "tool.step": step,
661
- });
662
- }
663
- logger.debug(`[GoogleAIStudio] Executing ${chunkResult.stepFunctionCalls.length} function calls`);
664
- // Add model response with ALL parts (including thoughtSignature) to history
665
- pushModelResponseToHistory(currentContents, chunkResult.rawResponseParts, chunkResult.stepFunctionCalls);
666
- const functionResponses = await executeNativeToolCalls("[GoogleAIStudio]", chunkResult.stepFunctionCalls, executeMap, failedTools, allToolCalls, { abortSignal: composedSignal });
667
- // Add function responses to history — the @google/genai SDK
668
- // only accepts "user" and "model" as valid roles in contents.
669
- // Function/tool responses must use role: "user" (matching the
670
- // SDK's own automaticFunctionCalling implementation).
671
- currentContents.push({
672
- role: "user",
673
- parts: functionResponses,
674
- });
675
- }
676
- catch (error) {
677
- logger.error("[GoogleAIStudio] Native SDK error", error);
678
- throw this.handleProviderError(error);
679
- }
680
- }
681
- // Handle max-steps termination: if the model was still calling
682
- // tools when we hit the limit, push a synthetic final message.
683
- const hitStepLimitWithoutFinalAnswer = step >= maxSteps && !completedWithFinalAnswer;
684
- if (hitStepLimitWithoutFinalAnswer) {
685
- const fallback = handleMaxStepsTermination("[GoogleAIStudio]", step, maxSteps, "", // finalText is empty — model didn't stop on its own
686
- lastStepText);
687
- if (fallback) {
688
- channel.push(fallback);
689
- }
690
- }
691
- const responseTime = Date.now() - startTime;
692
- // Update shared metadata so the returned object reflects final values.
693
- metadata.responseTime = responseTime;
694
- metadata.totalToolExecutions = allToolCalls.length;
695
- // Set token usage and finish reason on the span
696
- span.setAttribute(ATTR.GEN_AI_INPUT_TOKENS, totalInputTokens);
697
- span.setAttribute(ATTR.GEN_AI_OUTPUT_TOKENS, totalOutputTokens);
698
- span.setAttribute(ATTR.GEN_AI_FINISH_REASON, hitStepLimitWithoutFinalAnswer ? "max_steps" : "stop");
699
- analyticsResolve({
700
- provider: this.providerName,
701
- model: modelName,
702
- tokenUsage: {
703
- input: totalInputTokens,
704
- output: totalOutputTokens,
705
- total: totalInputTokens + totalOutputTokens,
706
- },
707
- requestDuration: responseTime,
708
- timestamp: new Date().toISOString(),
709
- });
710
- channel.close();
711
- }
712
- catch (err) {
713
- channel.error(err);
714
- analyticsReject(err);
715
- }
716
- finally {
717
- timeoutController?.cleanup();
718
- }
719
- })();
720
- // Suppress unhandled-rejection warnings on loopPromise — errors are
721
- // forwarded to the channel and will surface when the caller iterates.
722
- loopPromise.catch(() => undefined);
723
- return {
724
- stream: channel.iterable,
725
- provider: this.providerName,
726
- model: modelName,
727
- toolCalls: allToolCalls,
728
- analytics: analyticsPromise,
729
- metadata,
730
- };
731
- }
732
- finally {
733
- // Timeout controller cleanup is managed inside the background loop
734
- }
735
- });
736
- }
737
- /**
738
- * Execute generate using native @google/genai SDK for Gemini 3 models
739
- * This bypasses @ai-sdk/google to properly handle thought_signature
740
- */
741
- async executeNativeGemini3Generate(options) {
742
- const modelName = options.model || this.modelName;
743
- return withClientSpan({
744
- name: "neurolink.provider.generate",
745
- tracer: tracers.provider,
746
- attributes: {
747
- [ATTR.GEN_AI_SYSTEM]: "google-ai",
748
- [ATTR.GEN_AI_MODEL]: modelName,
749
- [ATTR.GEN_AI_OPERATION]: "generate",
750
- [ATTR.NL_PROVIDER]: this.providerName,
751
- },
752
- }, async (span) => {
753
- const startTime = Date.now();
754
- const timeout = this.getTimeout(options);
755
- const timeoutController = createTimeoutController(timeout, this.providerName, "generate");
756
- try {
757
- const apiKey = this.getApiKey();
758
- const client = await createGoogleGenAIClient(apiKey);
759
- logger.debug("[GoogleAIStudio] Using native @google/genai for Gemini 3 generate", {
760
- model: modelName,
761
- hasTools: !!options.tools && Object.keys(options.tools).length > 0,
762
- });
763
- // Build contents from input
764
- // Prefer input.text over prompt — processCSVFilesForNativeSDK enriches
765
- // input.text with inlined CSV data, so using prompt first would discard it.
766
- const promptText = options.input?.text || options.prompt || "";
767
- const currentContents = [{ role: "user", parts: [{ text: promptText }] }];
768
- // Convert tools (merge SDK tools with options.tools)
769
- let toolsConfig;
770
- let executeMap = new Map();
771
- const shouldUseTools = !options.disableTools;
772
- if (shouldUseTools) {
773
- const sdkTools = await this.getAllTools();
774
- const mergedTools = { ...sdkTools, ...(options.tools || {}) };
775
- if (Object.keys(mergedTools).length > 0) {
776
- const result = buildNativeToolDeclarations(mergedTools);
777
- toolsConfig = result.toolsConfig;
778
- executeMap = result.executeMap;
779
- logger.debug("[GoogleAIStudio] Converted tools for native SDK generate", {
780
- toolCount: toolsConfig[0].functionDeclarations.length,
781
- toolNames: toolsConfig[0].functionDeclarations.map((t) => t.name),
782
- });
783
- }
784
- }
785
- const config = buildNativeConfig(options, toolsConfig);
786
- const composedSignal = composeAbortSignals(options.abortSignal, timeoutController?.controller.signal);
787
- const maxSteps = computeMaxSteps(options.maxSteps);
788
- let finalText = "";
789
- let lastStepText = "";
790
- let totalInputTokens = 0;
791
- let totalOutputTokens = 0;
792
- const allToolCalls = [];
793
- const toolExecutions = [];
794
- let step = 0;
795
- const failedTools = new Map();
796
- // Agentic loop for tool calling
797
- while (step < maxSteps) {
798
- if (composedSignal?.aborted) {
799
- throw composedSignal.reason instanceof Error
800
- ? composedSignal.reason
801
- : new Error("Request aborted");
802
- }
803
- step++;
804
- logger.debug(`[GoogleAIStudio] Native SDK generate step ${step}/${maxSteps}`);
805
- try {
806
- const stream = await client.models.generateContentStream({
807
- model: modelName,
808
- contents: currentContents,
809
- config,
810
- ...(composedSignal
811
- ? { httpOptions: { signal: composedSignal } }
812
- : {}),
813
- });
814
- const chunkResult = await collectStreamChunks(stream);
815
- totalInputTokens += chunkResult.inputTokens;
816
- totalOutputTokens += chunkResult.outputTokens;
817
- const stepText = extractTextFromParts(chunkResult.rawResponseParts);
818
- // If no function calls, we're done
819
- if (chunkResult.stepFunctionCalls.length === 0) {
820
- finalText = stepText;
821
- break;
822
- }
823
- lastStepText = stepText;
824
- // Record tool call events on the span
825
- for (const fc of chunkResult.stepFunctionCalls) {
826
- span.addEvent("gen_ai.tool_call", {
827
- "tool.name": fc.name,
828
- "tool.step": step,
829
- });
830
- }
831
- logger.debug(`[GoogleAIStudio] Executing ${chunkResult.stepFunctionCalls.length} function calls in generate`);
832
- // Add model response with ALL parts (including thoughtSignature) to history
833
- // This is critical for Gemini 3 - it requires thought signatures in subsequent turns
834
- pushModelResponseToHistory(currentContents, chunkResult.rawResponseParts, chunkResult.stepFunctionCalls);
835
- const functionResponses = await executeNativeToolCalls("[GoogleAIStudio]", chunkResult.stepFunctionCalls, executeMap, failedTools, allToolCalls, { toolExecutions, abortSignal: composedSignal });
836
- // Add function responses to history — the @google/genai SDK
837
- // only accepts "user" and "model" as valid roles in contents.
838
- // Function/tool responses must use role: "user" (matching the
839
- // SDK's own automaticFunctionCalling implementation).
840
- currentContents.push({
841
- role: "user",
842
- parts: functionResponses,
843
- });
844
- }
845
- catch (error) {
846
- logger.error("[GoogleAIStudio] Native SDK generate error", error);
847
- throw this.handleProviderError(error);
848
- }
849
- }
850
- finalText = handleMaxStepsTermination("[GoogleAIStudio]", step, maxSteps, finalText, lastStepText);
851
- const responseTime = Date.now() - startTime;
852
- // Set token usage and finish reason on the span
853
- span.setAttribute(ATTR.GEN_AI_INPUT_TOKENS, totalInputTokens);
854
- span.setAttribute(ATTR.GEN_AI_OUTPUT_TOKENS, totalOutputTokens);
855
- span.setAttribute(ATTR.GEN_AI_FINISH_REASON, step >= maxSteps ? "max_steps" : "stop");
856
- // Build EnhancedGenerateResult
857
- return {
858
- content: finalText,
859
- provider: this.providerName,
860
- model: modelName,
861
- usage: {
862
- input: totalInputTokens,
863
- output: totalOutputTokens,
864
- total: totalInputTokens + totalOutputTokens,
865
- },
866
- responseTime,
867
- toolsUsed: allToolCalls.map((tc) => tc.toolName),
868
- toolExecutions: toolExecutions,
869
- enhancedWithTools: allToolCalls.length > 0,
870
- };
871
- }
872
- finally {
873
- timeoutController?.cleanup();
874
- }
875
- });
876
- }
877
- /**
878
- * Override generate to route Gemini 3 models with tools to native SDK
879
- */
880
- async generate(optionsOrPrompt) {
881
- // Normalize options
882
- const options = typeof optionsOrPrompt === "string"
883
- ? { prompt: optionsOrPrompt }
884
- : optionsOrPrompt;
885
- const modelName = options.model || this.modelName;
886
- // Check if we should use native SDK for Gemini 3 with tools
887
- const shouldUseTools = !options.disableTools && this.supportsTools();
888
- const sdkTools = shouldUseTools ? await this.getAllTools() : {};
889
- const hasTools = shouldUseTools &&
890
- (Object.keys(sdkTools).length > 0 ||
891
- (options.tools && Object.keys(options.tools).length > 0));
892
- if (isGemini3Model(modelName) && hasTools) {
893
- // Merge SDK tools into options for native SDK path
894
- let mergedOptions = {
895
- ...options,
896
- tools: { ...sdkTools, ...(options.tools || {}) },
897
- };
898
- // Check for tools + JSON schema conflict (Gemini limitation)
899
- const wantsJsonOutput = options.output?.format === "json" || options.schema;
900
- if (wantsJsonOutput &&
901
- mergedOptions.tools &&
902
- Object.keys(mergedOptions.tools).length > 0 &&
903
- !mergedOptions.disableTools) {
904
- logger.warn("[GoogleAIStudio] Gemini does not support tools and JSON schema output simultaneously. Disabling tools for this request.");
905
- mergedOptions = { ...mergedOptions, disableTools: true, tools: {} };
906
- }
907
- // Only route to native path if tools are still active after conflict check
908
- const hasActiveTools = !mergedOptions.disableTools &&
909
- mergedOptions.tools &&
910
- Object.keys(mergedOptions.tools).length > 0;
911
- if (hasActiveTools) {
912
- logger.info("[GoogleAIStudio] Routing Gemini 3 generate to native SDK for tool calling", {
913
- model: modelName,
914
- sdkToolCount: Object.keys(sdkTools).length,
915
- optionToolCount: Object.keys(options.tools || {}).length,
916
- totalToolCount: Object.keys(sdkTools).length +
917
- Object.keys(options.tools || {}).length,
918
- });
919
- return this.executeNativeGemini3Generate(mergedOptions);
920
- }
921
- // Fall through to standard generate path using merged options (tools disabled for schema)
922
- return super.generate(mergedOptions);
923
- }
924
- // Fall back to BaseProvider implementation
925
- return super.generate(options);
926
- }
927
- // ===================
928
- // HELPER METHODS
929
- // ===================
930
- async executeAudioStreamViaGeminiLive(options) {
931
- const startTime = Date.now();
932
- const apiKey = this.getApiKey();
933
- // Dynamic import to avoid hard dependency unless audio streaming is used
934
- let client;
935
- try {
936
- client = await createGoogleGenAIClient(apiKey);
937
- }
938
- catch {
939
- throw new AuthenticationError("Missing '@google/genai'. Install with: pnpm add @google/genai", this.providerName);
940
- }
941
- const model = this.modelName ||
942
- process.env.GOOGLE_VOICE_AI_MODEL ||
943
- "gemini-2.5-flash-preview-native-audio-dialog";
944
- const queue = [];
945
- let resolveNext = null;
946
- let done = false;
947
- const push = (item) => {
948
- if (done) {
949
- return;
950
- }
951
- if (item.type === "audio") {
952
- if (resolveNext) {
953
- const fn = resolveNext;
954
- resolveNext = null;
955
- fn({ value: { type: "audio", audio: item.audio }, done: false });
956
- return;
957
- }
958
- }
959
- queue.push(item);
960
- };
961
- const session = await client.live.connect({
962
- model,
963
- callbacks: {
964
- onopen: () => {
965
- // no-op
966
- },
967
- onmessage: async (message) => {
968
- try {
969
- const audio = message?.serverContent?.modelTurn?.parts?.[0]?.inlineData;
970
- if (audio?.data) {
971
- const buf = Buffer.from(String(audio.data), "base64");
972
- const chunk = {
973
- data: buf,
974
- sampleRateHz: 24000,
975
- channels: 1,
976
- encoding: "PCM16LE",
977
- };
978
- push({ type: "audio", audio: chunk });
979
- }
980
- if (message?.serverContent?.interrupted) {
981
- // allow consumer to handle; no special action required here
982
- }
983
- }
984
- catch (e) {
985
- push({ type: "error", error: e });
986
- }
987
- },
988
- onerror: (e) => {
989
- push({ type: "error", error: e });
990
- },
991
- onclose: (_e) => {
992
- push({ type: "end" });
993
- },
994
- },
995
- config: {
996
- responseModalities: ["AUDIO"],
997
- speechConfig: {
998
- voiceConfig: { prebuiltVoiceConfig: { voiceName: "Orus" } },
999
- },
1000
- },
1001
- });
1002
- // Feed upstream audio frames concurrently
1003
- (async () => {
1004
- try {
1005
- const spec = options.input?.audio;
1006
- if (!spec) {
1007
- logger.debug("[GeminiLive] No audio spec found on input; skipping upstream send");
1008
- return;
1009
- }
1010
- for await (const frame of spec.frames) {
1011
- // Zero-length frame acts as a 'flush' control signal
1012
- if (!frame || frame.byteLength === 0) {
1013
- try {
1014
- if (session.sendInput) {
1015
- await session.sendInput({ event: "flush" });
1016
- }
1017
- else if (session.sendRealtimeInput) {
1018
- await session.sendRealtimeInput({ event: "flush" });
1019
- }
1020
- }
1021
- catch (err) {
1022
- logger.debug("[GeminiLive] flush control failed (non-fatal)", {
1023
- error: err instanceof Error ? err.message : String(err),
1024
- });
1025
- }
1026
- continue;
1027
- }
1028
- // Convert PCM16LE buffer to base64 and wrap in genai Blob-like object
1029
- const base64 = frame.toString("base64");
1030
- const mimeType = `audio/pcm;rate=${spec.sampleRateHz || 16000}`;
1031
- await session.sendRealtimeInput?.({
1032
- media: { data: base64, mimeType },
1033
- });
1034
- }
1035
- // Best-effort flush signal if supported
1036
- try {
1037
- if (session.sendInput) {
1038
- await session.sendInput({ event: "flush" });
1039
- }
1040
- else if (session.sendRealtimeInput) {
1041
- await session.sendRealtimeInput({ event: "flush" });
1042
- }
1043
- }
1044
- catch (err) {
1045
- logger.debug("[GeminiLive] final flush failed (non-fatal)", {
1046
- error: err instanceof Error ? err.message : String(err),
1047
- });
1048
- }
1049
- }
1050
- catch (e) {
1051
- push({ type: "error", error: e });
1052
- }
1053
- })().catch(() => {
1054
- // ignore
1055
- });
1056
- // AsyncIterable for stream events
1057
- const asyncIterable = {
1058
- [Symbol.asyncIterator]() {
1059
- return {
1060
- async next() {
1061
- if (queue.length > 0) {
1062
- const item = queue.shift();
1063
- if (!item) {
1064
- return {
1065
- value: undefined,
1066
- done: true,
1067
- };
1068
- }
1069
- if (item.type === "audio") {
1070
- return {
1071
- value: { type: "audio", audio: item.audio },
1072
- done: false,
1073
- };
1074
- }
1075
- if (item.type === "end") {
1076
- done = true;
1077
- return {
1078
- value: undefined,
1079
- done: true,
1080
- };
1081
- }
1082
- if (item.type === "error") {
1083
- done = true;
1084
- throw item.error instanceof Error
1085
- ? item.error
1086
- : new Error(String(item.error));
1087
- }
1088
- }
1089
- if (done) {
1090
- return {
1091
- value: undefined,
1092
- done: true,
1093
- };
1094
- }
1095
- return await new Promise((resolve) => {
1096
- resolveNext = resolve;
1097
- });
1098
- },
1099
- };
1100
- },
1101
- };
1102
- return {
1103
- stream: asyncIterable,
1104
- provider: this.providerName,
1105
- model: model,
1106
- metadata: {
1107
- startTime,
1108
- streamId: `google-ai-audio-${Date.now()}`,
1109
- },
1110
- };
1111
- }
1112
- getDefaultEmbeddingModel() {
1113
- return (process.env.GOOGLE_AI_EMBEDDING_MODEL ||
1114
- process.env.GOOGLE_EMBEDDING_MODEL ||
1115
- "gemini-embedding-001");
1116
- }
1117
- /**
1118
- * Generate embeddings for text using Google AI Studio embedding models
1119
- * @param text - The text to embed
1120
- * @param modelName - The embedding model to use (default: gemini-embedding-001)
1121
- * @returns Promise resolving to the embedding vector
1122
- */
1123
- async embed(text, modelName) {
1124
- const embeddingModelName = modelName || this.getDefaultEmbeddingModel() || "gemini-embedding-001";
1125
- logger.debug("Generating embedding", {
1126
- provider: this.providerName,
1127
- model: embeddingModelName,
1128
- textLength: text.length,
1129
- });
1130
- try {
1131
- const apiKey = this.getApiKey();
1132
- const google = createGoogleGenerativeAI({ apiKey });
1133
- const embeddingModel = google.textEmbeddingModel(embeddingModelName);
1134
- const result = await embed({
1135
- model: embeddingModel,
1136
- value: text,
1137
- });
1138
- logger.debug("Embedding generated successfully", {
1139
- provider: this.providerName,
1140
- model: embeddingModelName,
1141
- embeddingDimension: result.embedding.length,
1142
- });
1143
- return result.embedding;
1144
- }
1145
- catch (error) {
1146
- logger.error("Embedding generation failed", {
1147
- error: error instanceof Error ? error.message : String(error),
1148
- model: embeddingModelName,
1149
- textLength: text.length,
1150
- });
1151
- throw this.handleProviderError(error);
1152
- }
1153
- }
1154
- /**
1155
- * Generate embeddings for multiple texts in a single batch
1156
- * @param texts - The texts to embed
1157
- * @param modelName - The embedding model to use (default: gemini-embedding-001)
1158
- * @returns Promise resolving to an array of embedding vectors
1159
- */
1160
- async embedMany(texts, modelName) {
1161
- const embeddingModelName = modelName || this.getDefaultEmbeddingModel() || "gemini-embedding-001";
1162
- logger.debug("Generating batch embeddings", {
1163
- provider: this.providerName,
1164
- model: embeddingModelName,
1165
- count: texts.length,
1166
- });
1167
- try {
1168
- const apiKey = this.getApiKey();
1169
- const google = createGoogleGenerativeAI({ apiKey });
1170
- const embeddingModel = google.textEmbeddingModel(embeddingModelName);
1171
- const result = await embedMany({
1172
- model: embeddingModel,
1173
- values: texts,
1174
- });
1175
- logger.debug("Batch embeddings generated successfully", {
1176
- provider: this.providerName,
1177
- model: embeddingModelName,
1178
- count: result.embeddings.length,
1179
- embeddingDimension: result.embeddings[0]?.length,
1180
- });
1181
- return result.embeddings;
1182
- }
1183
- catch (error) {
1184
- logger.error("Batch embedding generation failed", {
1185
- error: error instanceof Error ? error.message : String(error),
1186
- model: embeddingModelName,
1187
- count: texts.length,
1188
- });
1189
- throw this.handleProviderError(error);
1190
- }
1191
- }
1192
- getApiKey() {
1193
- const apiKey = process.env.GOOGLE_AI_API_KEY || process.env.GOOGLE_GENERATIVE_AI_API_KEY;
1194
- if (!apiKey) {
1195
- throw new AuthenticationError("GOOGLE_AI_API_KEY or GOOGLE_GENERATIVE_AI_API_KEY environment variable is not set", this.providerName);
1196
- }
1197
- return apiKey;
1198
- }
1199
- }
1200
- export default GoogleAIStudioProvider;