@juspay/neurolink 9.32.0 → 9.32.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (467) hide show
  1. package/CHANGELOG.md +6 -0
  2. package/dist/auth/anthropicOAuth.js +1 -1
  3. package/dist/cli/commands/proxy.js +18 -5
  4. package/dist/client/aiSdkAdapter.js +1 -1
  5. package/dist/client/index.js +137 -501
  6. package/dist/core/factory.js +0 -1
  7. package/dist/core/redisConversationMemoryManager.js +1 -1
  8. package/dist/features/ppt/slideGenerator.js +0 -1
  9. package/dist/features/ppt/utils.js +0 -1
  10. package/dist/lib/server/routes/claudeProxyRoutes.js +45 -9
  11. package/dist/mcp/elicitationProtocol.js +1 -1
  12. package/dist/mcp/servers/agent/directToolsServer.js +0 -1
  13. package/dist/providers/azureOpenai.js +1 -1
  14. package/dist/providers/huggingFace.js +0 -1
  15. package/dist/providers/openaiCompatible.js +0 -1
  16. package/dist/sdk/toolRegistration.js +0 -1
  17. package/dist/server/openapi/generator.js +1 -1
  18. package/dist/server/routes/claudeProxyRoutes.js +45 -9
  19. package/dist/types/configTypes.js +0 -5
  20. package/dist/types/modelTypes.js +0 -1
  21. package/dist/types/tools.js +0 -1
  22. package/dist/types/typeAliases.js +0 -1
  23. package/dist/types/utilities.js +1 -1
  24. package/dist/types/workflowTypes.js +0 -1
  25. package/dist/utils/providerRetry.js +0 -1
  26. package/dist/utils/providerUtils.js +0 -1
  27. package/package.json +2 -2
  28. package/dist/client/adapters/providerImageAdapter.js +0 -588
  29. package/dist/client/adapters/tts/googleTTSHandler.js +0 -344
  30. package/dist/client/adapters/video/directorPipeline.js +0 -516
  31. package/dist/client/adapters/video/ffmpegAdapter.js +0 -206
  32. package/dist/client/adapters/video/frameExtractor.js +0 -143
  33. package/dist/client/adapters/video/vertexVideoHandler.js +0 -763
  34. package/dist/client/adapters/video/videoAnalyzer.js +0 -238
  35. package/dist/client/adapters/video/videoMerger.js +0 -171
  36. package/dist/client/agent/directTools.js +0 -840
  37. package/dist/client/auth/AuthProviderFactory.js +0 -111
  38. package/dist/client/auth/AuthProviderRegistry.js +0 -190
  39. package/dist/client/auth/RequestContext.js +0 -78
  40. package/dist/client/auth/accountPool.js +0 -178
  41. package/dist/client/auth/anthropicOAuth.js +0 -974
  42. package/dist/client/auth/authContext.js +0 -314
  43. package/dist/client/auth/errors.js +0 -39
  44. package/dist/client/auth/index.js +0 -61
  45. package/dist/client/auth/middleware/AuthMiddleware.js +0 -519
  46. package/dist/client/auth/middleware/rateLimitByUser.js +0 -554
  47. package/dist/client/auth/providers/BaseAuthProvider.js +0 -723
  48. package/dist/client/auth/providers/CognitoProvider.js +0 -304
  49. package/dist/client/auth/providers/KeycloakProvider.js +0 -393
  50. package/dist/client/auth/providers/auth0.js +0 -274
  51. package/dist/client/auth/providers/betterAuth.js +0 -182
  52. package/dist/client/auth/providers/clerk.js +0 -317
  53. package/dist/client/auth/providers/custom.js +0 -112
  54. package/dist/client/auth/providers/firebase.js +0 -226
  55. package/dist/client/auth/providers/jwt.js +0 -212
  56. package/dist/client/auth/providers/oauth2.js +0 -303
  57. package/dist/client/auth/providers/supabase.js +0 -259
  58. package/dist/client/auth/providers/workos.js +0 -284
  59. package/dist/client/auth/serverBridge.js +0 -25
  60. package/dist/client/auth/sessionManager.js +0 -437
  61. package/dist/client/auth/tokenStore.js +0 -799
  62. package/dist/client/client/aiSdkAdapter.js +0 -487
  63. package/dist/client/client/auth.js +0 -473
  64. package/dist/client/client/errors.js +0 -552
  65. package/dist/client/client/httpClient.js +0 -837
  66. package/dist/client/client/index.js +0 -172
  67. package/dist/client/client/interceptors.js +0 -601
  68. package/dist/client/client/sseClient.js +0 -545
  69. package/dist/client/client/streamingClient.js +0 -917
  70. package/dist/client/client/wsClient.js +0 -369
  71. package/dist/client/config/configManager.js +0 -303
  72. package/dist/client/config/conversationMemory.js +0 -86
  73. package/dist/client/config/taskClassificationConfig.js +0 -148
  74. package/dist/client/constants/contextWindows.js +0 -295
  75. package/dist/client/constants/enums.js +0 -853
  76. package/dist/client/constants/index.js +0 -207
  77. package/dist/client/constants/performance.js +0 -389
  78. package/dist/client/constants/retry.js +0 -266
  79. package/dist/client/constants/timeouts.js +0 -182
  80. package/dist/client/constants/tokens.js +0 -380
  81. package/dist/client/constants/videoErrors.js +0 -46
  82. package/dist/client/context/budgetChecker.js +0 -98
  83. package/dist/client/context/contextCompactor.js +0 -205
  84. package/dist/client/context/emergencyTruncation.js +0 -88
  85. package/dist/client/context/errorDetection.js +0 -171
  86. package/dist/client/context/errors.js +0 -21
  87. package/dist/client/context/fileTokenBudget.js +0 -127
  88. package/dist/client/context/prompts/summarizationPrompt.js +0 -117
  89. package/dist/client/context/stages/fileReadDeduplicator.js +0 -66
  90. package/dist/client/context/stages/slidingWindowTruncator.js +0 -190
  91. package/dist/client/context/stages/structuredSummarizer.js +0 -99
  92. package/dist/client/context/stages/toolOutputPruner.js +0 -52
  93. package/dist/client/context/summarizationEngine.js +0 -136
  94. package/dist/client/context/toolOutputLimits.js +0 -78
  95. package/dist/client/context/toolPairRepair.js +0 -66
  96. package/dist/client/core/analytics.js +0 -88
  97. package/dist/client/core/baseProvider.js +0 -1385
  98. package/dist/client/core/constants.js +0 -140
  99. package/dist/client/core/conversationMemoryFactory.js +0 -141
  100. package/dist/client/core/conversationMemoryInitializer.js +0 -128
  101. package/dist/client/core/conversationMemoryManager.js +0 -344
  102. package/dist/client/core/dynamicModels.js +0 -358
  103. package/dist/client/core/evaluation.js +0 -309
  104. package/dist/client/core/evaluationProviders.js +0 -248
  105. package/dist/client/core/factory.js +0 -412
  106. package/dist/client/core/infrastructure/baseError.js +0 -22
  107. package/dist/client/core/infrastructure/baseFactory.js +0 -54
  108. package/dist/client/core/infrastructure/baseRegistry.js +0 -53
  109. package/dist/client/core/infrastructure/index.js +0 -5
  110. package/dist/client/core/infrastructure/retry.js +0 -20
  111. package/dist/client/core/infrastructure/typedEventEmitter.js +0 -23
  112. package/dist/client/core/modelConfiguration.js +0 -851
  113. package/dist/client/core/modules/GenerationHandler.js +0 -588
  114. package/dist/client/core/modules/MessageBuilder.js +0 -273
  115. package/dist/client/core/modules/StreamHandler.js +0 -185
  116. package/dist/client/core/modules/TelemetryHandler.js +0 -203
  117. package/dist/client/core/modules/ToolsManager.js +0 -499
  118. package/dist/client/core/modules/Utilities.js +0 -331
  119. package/dist/client/core/redisConversationMemoryManager.js +0 -1435
  120. package/dist/client/core/streamAnalytics.js +0 -131
  121. package/dist/client/evaluation/contextBuilder.js +0 -134
  122. package/dist/client/evaluation/index.js +0 -61
  123. package/dist/client/evaluation/prompts.js +0 -73
  124. package/dist/client/evaluation/ragasEvaluator.js +0 -110
  125. package/dist/client/evaluation/retryManager.js +0 -78
  126. package/dist/client/evaluation/scoring.js +0 -61
  127. package/dist/client/factories/providerFactory.js +0 -166
  128. package/dist/client/factories/providerRegistry.js +0 -166
  129. package/dist/client/features/ppt/constants.js +0 -896
  130. package/dist/client/features/ppt/contentPlanner.js +0 -529
  131. package/dist/client/features/ppt/presentationOrchestrator.js +0 -236
  132. package/dist/client/features/ppt/slideGenerator.js +0 -532
  133. package/dist/client/features/ppt/slideRenderers.js +0 -2383
  134. package/dist/client/features/ppt/slideTypeInference.js +0 -405
  135. package/dist/client/features/ppt/types.js +0 -13
  136. package/dist/client/features/ppt/utils.js +0 -443
  137. package/dist/client/files/fileReferenceRegistry.js +0 -1543
  138. package/dist/client/files/fileTools.js +0 -450
  139. package/dist/client/files/streamingReader.js +0 -321
  140. package/dist/client/files/types.js +0 -23
  141. package/dist/client/hitl/hitlErrors.js +0 -54
  142. package/dist/client/hitl/hitlManager.js +0 -460
  143. package/dist/client/mcp/agentExposure.js +0 -356
  144. package/dist/client/mcp/auth/index.js +0 -11
  145. package/dist/client/mcp/auth/oauthClientProvider.js +0 -325
  146. package/dist/client/mcp/auth/tokenStorage.js +0 -134
  147. package/dist/client/mcp/batching/index.js +0 -10
  148. package/dist/client/mcp/batching/requestBatcher.js +0 -441
  149. package/dist/client/mcp/caching/index.js +0 -10
  150. package/dist/client/mcp/caching/toolCache.js +0 -433
  151. package/dist/client/mcp/elicitation/elicitationManager.js +0 -376
  152. package/dist/client/mcp/elicitation/index.js +0 -11
  153. package/dist/client/mcp/elicitation/types.js +0 -10
  154. package/dist/client/mcp/elicitationProtocol.js +0 -375
  155. package/dist/client/mcp/enhancedToolDiscovery.js +0 -481
  156. package/dist/client/mcp/externalServerManager.js +0 -1478
  157. package/dist/client/mcp/factory.js +0 -161
  158. package/dist/client/mcp/flexibleToolValidator.js +0 -161
  159. package/dist/client/mcp/httpRateLimiter.js +0 -391
  160. package/dist/client/mcp/httpRetryHandler.js +0 -178
  161. package/dist/client/mcp/index.js +0 -74
  162. package/dist/client/mcp/mcpCircuitBreaker.js +0 -427
  163. package/dist/client/mcp/mcpClientFactory.js +0 -708
  164. package/dist/client/mcp/mcpRegistryClient.js +0 -488
  165. package/dist/client/mcp/mcpServerBase.js +0 -373
  166. package/dist/client/mcp/multiServerManager.js +0 -579
  167. package/dist/client/mcp/registry.js +0 -158
  168. package/dist/client/mcp/routing/index.js +0 -10
  169. package/dist/client/mcp/routing/toolRouter.js +0 -416
  170. package/dist/client/mcp/serverCapabilities.js +0 -502
  171. package/dist/client/mcp/servers/agent/directToolsServer.js +0 -150
  172. package/dist/client/mcp/toolAnnotations.js +0 -239
  173. package/dist/client/mcp/toolConverter.js +0 -258
  174. package/dist/client/mcp/toolDiscoveryService.js +0 -798
  175. package/dist/client/mcp/toolIntegration.js +0 -334
  176. package/dist/client/mcp/toolRegistry.js +0 -729
  177. package/dist/client/memory/hippocampusInitializer.js +0 -19
  178. package/dist/client/memory/memoryRetrievalTools.js +0 -166
  179. package/dist/client/middleware/builtin/analytics.js +0 -132
  180. package/dist/client/middleware/builtin/autoEvaluation.js +0 -203
  181. package/dist/client/middleware/builtin/guardrails.js +0 -109
  182. package/dist/client/middleware/builtin/lifecycle.js +0 -168
  183. package/dist/client/middleware/factory.js +0 -327
  184. package/dist/client/middleware/registry.js +0 -295
  185. package/dist/client/middleware/utils/guardrailsUtils.js +0 -396
  186. package/dist/client/models/anthropicModels.js +0 -527
  187. package/dist/client/neurolink.js +0 -8233
  188. package/dist/client/observability/exporterRegistry.js +0 -413
  189. package/dist/client/observability/exporters/arizeExporter.js +0 -138
  190. package/dist/client/observability/exporters/baseExporter.js +0 -190
  191. package/dist/client/observability/exporters/braintrustExporter.js +0 -154
  192. package/dist/client/observability/exporters/datadogExporter.js +0 -196
  193. package/dist/client/observability/exporters/laminarExporter.js +0 -302
  194. package/dist/client/observability/exporters/langfuseExporter.js +0 -209
  195. package/dist/client/observability/exporters/langsmithExporter.js +0 -143
  196. package/dist/client/observability/exporters/otelExporter.js +0 -164
  197. package/dist/client/observability/exporters/posthogExporter.js +0 -287
  198. package/dist/client/observability/exporters/sentryExporter.js +0 -165
  199. package/dist/client/observability/index.js +0 -31
  200. package/dist/client/observability/metricsAggregator.js +0 -556
  201. package/dist/client/observability/otelBridge.js +0 -131
  202. package/dist/client/observability/retryPolicy.js +0 -383
  203. package/dist/client/observability/sampling/samplers.js +0 -216
  204. package/dist/client/observability/spanProcessor.js +0 -303
  205. package/dist/client/observability/tokenTracker.js +0 -413
  206. package/dist/client/observability/types/exporterTypes.js +0 -5
  207. package/dist/client/observability/types/index.js +0 -4
  208. package/dist/client/observability/types/spanTypes.js +0 -92
  209. package/dist/client/observability/utils/safeMetadata.js +0 -25
  210. package/dist/client/observability/utils/spanSerializer.js +0 -292
  211. package/dist/client/processors/archive/ArchiveProcessor.js +0 -1308
  212. package/dist/client/processors/base/BaseFileProcessor.js +0 -614
  213. package/dist/client/processors/base/types.js +0 -82
  214. package/dist/client/processors/config/fileTypes.js +0 -520
  215. package/dist/client/processors/config/index.js +0 -92
  216. package/dist/client/processors/config/languageMap.js +0 -410
  217. package/dist/client/processors/config/mimeTypes.js +0 -363
  218. package/dist/client/processors/config/sizeLimits.js +0 -258
  219. package/dist/client/processors/document/ExcelProcessor.js +0 -590
  220. package/dist/client/processors/document/OpenDocumentProcessor.js +0 -212
  221. package/dist/client/processors/document/PptxProcessor.js +0 -157
  222. package/dist/client/processors/document/RtfProcessor.js +0 -361
  223. package/dist/client/processors/document/WordProcessor.js +0 -353
  224. package/dist/client/processors/errors/FileErrorCode.js +0 -255
  225. package/dist/client/processors/errors/errorHelpers.js +0 -386
  226. package/dist/client/processors/errors/errorSerializer.js +0 -507
  227. package/dist/client/processors/errors/index.js +0 -49
  228. package/dist/client/processors/markup/SvgProcessor.js +0 -240
  229. package/dist/client/processors/media/AudioProcessor.js +0 -707
  230. package/dist/client/processors/media/VideoProcessor.js +0 -1045
  231. package/dist/client/providers/amazonBedrock.js +0 -1512
  232. package/dist/client/providers/amazonSagemaker.js +0 -162
  233. package/dist/client/providers/anthropic.js +0 -831
  234. package/dist/client/providers/azureOpenai.js +0 -143
  235. package/dist/client/providers/googleAiStudio.js +0 -1200
  236. package/dist/client/providers/googleNativeGemini3.js +0 -543
  237. package/dist/client/providers/googleVertex.js +0 -2936
  238. package/dist/client/providers/huggingFace.js +0 -315
  239. package/dist/client/providers/litellm.js +0 -488
  240. package/dist/client/providers/mistral.js +0 -157
  241. package/dist/client/providers/ollama.js +0 -1579
  242. package/dist/client/providers/openAI.js +0 -627
  243. package/dist/client/providers/openRouter.js +0 -543
  244. package/dist/client/providers/openaiCompatible.js +0 -290
  245. package/dist/client/providers/providerTypeUtils.js +0 -46
  246. package/dist/client/providers/sagemaker/adaptive-semaphore.js +0 -215
  247. package/dist/client/providers/sagemaker/client.js +0 -472
  248. package/dist/client/providers/sagemaker/config.js +0 -317
  249. package/dist/client/providers/sagemaker/detection.js +0 -606
  250. package/dist/client/providers/sagemaker/error-constants.js +0 -227
  251. package/dist/client/providers/sagemaker/errors.js +0 -299
  252. package/dist/client/providers/sagemaker/language-model.js +0 -775
  253. package/dist/client/providers/sagemaker/parsers.js +0 -634
  254. package/dist/client/providers/sagemaker/streaming.js +0 -331
  255. package/dist/client/providers/sagemaker/structured-parser.js +0 -625
  256. package/dist/client/proxy/accountQuota.js +0 -162
  257. package/dist/client/proxy/claudeFormat.js +0 -595
  258. package/dist/client/proxy/modelRouter.js +0 -29
  259. package/dist/client/proxy/oauthFetch.js +0 -367
  260. package/dist/client/proxy/proxyFetch.js +0 -586
  261. package/dist/client/proxy/requestLogger.js +0 -207
  262. package/dist/client/proxy/tokenRefresh.js +0 -124
  263. package/dist/client/proxy/usageStats.js +0 -74
  264. package/dist/client/proxy/utils/noProxyUtils.js +0 -149
  265. package/dist/client/rag/ChunkerFactory.js +0 -320
  266. package/dist/client/rag/ChunkerRegistry.js +0 -421
  267. package/dist/client/rag/chunkers/BaseChunker.js +0 -143
  268. package/dist/client/rag/chunkers/CharacterChunker.js +0 -28
  269. package/dist/client/rag/chunkers/HTMLChunker.js +0 -38
  270. package/dist/client/rag/chunkers/JSONChunker.js +0 -68
  271. package/dist/client/rag/chunkers/LaTeXChunker.js +0 -63
  272. package/dist/client/rag/chunkers/MarkdownChunker.js +0 -306
  273. package/dist/client/rag/chunkers/RecursiveChunker.js +0 -139
  274. package/dist/client/rag/chunkers/SemanticMarkdownChunker.js +0 -138
  275. package/dist/client/rag/chunkers/SentenceChunker.js +0 -66
  276. package/dist/client/rag/chunkers/TokenChunker.js +0 -61
  277. package/dist/client/rag/chunkers/index.js +0 -15
  278. package/dist/client/rag/chunking/characterChunker.js +0 -142
  279. package/dist/client/rag/chunking/chunkerRegistry.js +0 -194
  280. package/dist/client/rag/chunking/htmlChunker.js +0 -247
  281. package/dist/client/rag/chunking/index.js +0 -17
  282. package/dist/client/rag/chunking/jsonChunker.js +0 -281
  283. package/dist/client/rag/chunking/latexChunker.js +0 -251
  284. package/dist/client/rag/chunking/markdownChunker.js +0 -373
  285. package/dist/client/rag/chunking/recursiveChunker.js +0 -148
  286. package/dist/client/rag/chunking/semanticChunker.js +0 -306
  287. package/dist/client/rag/chunking/sentenceChunker.js +0 -230
  288. package/dist/client/rag/chunking/tokenChunker.js +0 -183
  289. package/dist/client/rag/document/MDocument.js +0 -392
  290. package/dist/client/rag/document/index.js +0 -5
  291. package/dist/client/rag/document/loaders.js +0 -500
  292. package/dist/client/rag/errors/RAGError.js +0 -274
  293. package/dist/client/rag/errors/index.js +0 -6
  294. package/dist/client/rag/graphRag/graphRAG.js +0 -401
  295. package/dist/client/rag/graphRag/index.js +0 -4
  296. package/dist/client/rag/index.js +0 -141
  297. package/dist/client/rag/metadata/MetadataExtractorFactory.js +0 -418
  298. package/dist/client/rag/metadata/MetadataExtractorRegistry.js +0 -362
  299. package/dist/client/rag/metadata/index.js +0 -9
  300. package/dist/client/rag/metadata/metadataExtractor.js +0 -280
  301. package/dist/client/rag/pipeline/RAGPipeline.js +0 -436
  302. package/dist/client/rag/pipeline/contextAssembly.js +0 -341
  303. package/dist/client/rag/pipeline/index.js +0 -5
  304. package/dist/client/rag/ragIntegration.js +0 -321
  305. package/dist/client/rag/reranker/RerankerFactory.js +0 -430
  306. package/dist/client/rag/reranker/RerankerRegistry.js +0 -402
  307. package/dist/client/rag/reranker/index.js +0 -9
  308. package/dist/client/rag/reranker/reranker.js +0 -277
  309. package/dist/client/rag/resilience/CircuitBreaker.js +0 -431
  310. package/dist/client/rag/resilience/RetryHandler.js +0 -304
  311. package/dist/client/rag/resilience/index.js +0 -7
  312. package/dist/client/rag/retrieval/hybridSearch.js +0 -335
  313. package/dist/client/rag/retrieval/index.js +0 -5
  314. package/dist/client/rag/retrieval/vectorQueryTool.js +0 -307
  315. package/dist/client/rag/types.js +0 -8
  316. package/dist/client/sdk/toolRegistration.js +0 -377
  317. package/dist/client/server/abstract/baseServerAdapter.js +0 -575
  318. package/dist/client/server/adapters/expressAdapter.js +0 -486
  319. package/dist/client/server/adapters/fastifyAdapter.js +0 -472
  320. package/dist/client/server/adapters/honoAdapter.js +0 -632
  321. package/dist/client/server/adapters/koaAdapter.js +0 -510
  322. package/dist/client/server/errors.js +0 -486
  323. package/dist/client/server/factory/serverAdapterFactory.js +0 -160
  324. package/dist/client/server/index.js +0 -108
  325. package/dist/client/server/middleware/abortSignal.js +0 -111
  326. package/dist/client/server/middleware/auth.js +0 -388
  327. package/dist/client/server/middleware/cache.js +0 -359
  328. package/dist/client/server/middleware/common.js +0 -281
  329. package/dist/client/server/middleware/deprecation.js +0 -190
  330. package/dist/client/server/middleware/mcpBodyAttachment.js +0 -63
  331. package/dist/client/server/middleware/rateLimit.js +0 -227
  332. package/dist/client/server/middleware/validation.js +0 -388
  333. package/dist/client/server/openapi/generator.js +0 -398
  334. package/dist/client/server/openapi/index.js +0 -36
  335. package/dist/client/server/openapi/schemas.js +0 -695
  336. package/dist/client/server/openapi/templates.js +0 -374
  337. package/dist/client/server/routes/agentRoutes.js +0 -189
  338. package/dist/client/server/routes/claudeProxyRoutes.js +0 -1600
  339. package/dist/client/server/routes/healthRoutes.js +0 -187
  340. package/dist/client/server/routes/index.js +0 -57
  341. package/dist/client/server/routes/mcpRoutes.js +0 -342
  342. package/dist/client/server/routes/memoryRoutes.js +0 -350
  343. package/dist/client/server/routes/openApiRoutes.js +0 -126
  344. package/dist/client/server/routes/toolRoutes.js +0 -199
  345. package/dist/client/server/streaming/dataStream.js +0 -486
  346. package/dist/client/server/streaming/index.js +0 -11
  347. package/dist/client/server/types.js +0 -67
  348. package/dist/client/server/utils/redaction.js +0 -334
  349. package/dist/client/server/utils/validation.js +0 -243
  350. package/dist/client/server/websocket/WebSocketHandler.js +0 -383
  351. package/dist/client/server/websocket/index.js +0 -4
  352. package/dist/client/services/server/ai/observability/instrumentation.js +0 -808
  353. package/dist/client/telemetry/attributes.js +0 -100
  354. package/dist/client/telemetry/index.js +0 -26
  355. package/dist/client/telemetry/telemetryService.js +0 -308
  356. package/dist/client/telemetry/tracers.js +0 -17
  357. package/dist/client/telemetry/withSpan.js +0 -34
  358. package/dist/client/types/actionTypes.js +0 -6
  359. package/dist/client/types/analytics.js +0 -5
  360. package/dist/client/types/authTypes.js +0 -9
  361. package/dist/client/types/circuitBreakerErrors.js +0 -34
  362. package/dist/client/types/cli.js +0 -21
  363. package/dist/client/types/clientTypes.js +0 -10
  364. package/dist/client/types/common.js +0 -51
  365. package/dist/client/types/configTypes.js +0 -49
  366. package/dist/client/types/content.js +0 -19
  367. package/dist/client/types/contextTypes.js +0 -400
  368. package/dist/client/types/conversation.js +0 -47
  369. package/dist/client/types/conversationMemoryInterface.js +0 -6
  370. package/dist/client/types/domainTypes.js +0 -5
  371. package/dist/client/types/errors.js +0 -167
  372. package/dist/client/types/evaluation.js +0 -5
  373. package/dist/client/types/evaluationProviders.js +0 -5
  374. package/dist/client/types/evaluationTypes.js +0 -1
  375. package/dist/client/types/externalMcp.js +0 -6
  376. package/dist/client/types/fileReferenceTypes.js +0 -8
  377. package/dist/client/types/fileTypes.js +0 -4
  378. package/dist/client/types/generateTypes.js +0 -1
  379. package/dist/client/types/guardrails.js +0 -1
  380. package/dist/client/types/hitlTypes.js +0 -8
  381. package/dist/client/types/index.js +0 -57
  382. package/dist/client/types/mcpTypes.js +0 -5
  383. package/dist/client/types/middlewareTypes.js +0 -1
  384. package/dist/client/types/modelTypes.js +0 -30
  385. package/dist/client/types/multimodal.js +0 -135
  386. package/dist/client/types/observability.js +0 -6
  387. package/dist/client/types/pptTypes.js +0 -82
  388. package/dist/client/types/providers.js +0 -111
  389. package/dist/client/types/proxyTypes.js +0 -16
  390. package/dist/client/types/ragTypes.js +0 -7
  391. package/dist/client/types/sdkTypes.js +0 -8
  392. package/dist/client/types/serviceTypes.js +0 -5
  393. package/dist/client/types/streamTypes.js +0 -1
  394. package/dist/client/types/subscriptionTypes.js +0 -9
  395. package/dist/client/types/taskClassificationTypes.js +0 -5
  396. package/dist/client/types/tools.js +0 -24
  397. package/dist/client/types/ttsTypes.js +0 -57
  398. package/dist/client/types/typeAliases.js +0 -48
  399. package/dist/client/types/utilities.js +0 -4
  400. package/dist/client/types/workflowTypes.js +0 -30
  401. package/dist/client/utils/async/withTimeout.js +0 -98
  402. package/dist/client/utils/asyncMutex.js +0 -60
  403. package/dist/client/utils/conversationMemory.js +0 -431
  404. package/dist/client/utils/csvProcessor.js +0 -846
  405. package/dist/client/utils/errorHandling.js +0 -936
  406. package/dist/client/utils/evaluationUtils.js +0 -131
  407. package/dist/client/utils/factoryProcessing.js +0 -589
  408. package/dist/client/utils/fileDetector.js +0 -2161
  409. package/dist/client/utils/imageCache.js +0 -376
  410. package/dist/client/utils/imageProcessor.js +0 -704
  411. package/dist/client/utils/logger.js +0 -491
  412. package/dist/client/utils/mcpDefaults.js +0 -134
  413. package/dist/client/utils/messageBuilder.js +0 -1653
  414. package/dist/client/utils/modelAliasResolver.js +0 -54
  415. package/dist/client/utils/modelDetection.js +0 -80
  416. package/dist/client/utils/modelRouter.js +0 -292
  417. package/dist/client/utils/multimodalOptionsBuilder.js +0 -65
  418. package/dist/client/utils/observabilityHelpers.js +0 -47
  419. package/dist/client/utils/parameterValidation.js +0 -966
  420. package/dist/client/utils/pdfProcessor.js +0 -410
  421. package/dist/client/utils/performance.js +0 -222
  422. package/dist/client/utils/pricing.js +0 -340
  423. package/dist/client/utils/promptRedaction.js +0 -62
  424. package/dist/client/utils/providerConfig.js +0 -1009
  425. package/dist/client/utils/providerHealth.js +0 -1237
  426. package/dist/client/utils/providerRetry.js +0 -112
  427. package/dist/client/utils/providerUtils.js +0 -434
  428. package/dist/client/utils/rateLimiter.js +0 -200
  429. package/dist/client/utils/redis.js +0 -368
  430. package/dist/client/utils/retryHandler.js +0 -269
  431. package/dist/client/utils/retryability.js +0 -22
  432. package/dist/client/utils/sanitizers/svg.js +0 -481
  433. package/dist/client/utils/schemaConversion.js +0 -255
  434. package/dist/client/utils/taskClassificationUtils.js +0 -149
  435. package/dist/client/utils/taskClassifier.js +0 -94
  436. package/dist/client/utils/thinkingConfig.js +0 -104
  437. package/dist/client/utils/timeout.js +0 -359
  438. package/dist/client/utils/tokenEstimation.js +0 -142
  439. package/dist/client/utils/tokenLimits.js +0 -125
  440. package/dist/client/utils/tokenUtils.js +0 -239
  441. package/dist/client/utils/toolUtils.js +0 -75
  442. package/dist/client/utils/transformationUtils.js +0 -554
  443. package/dist/client/utils/ttsProcessor.js +0 -286
  444. package/dist/client/utils/typeUtils.js +0 -97
  445. package/dist/client/utils/videoAnalysisProcessor.js +0 -67
  446. package/dist/client/workflow/config.js +0 -398
  447. package/dist/client/workflow/core/ensembleExecutor.js +0 -407
  448. package/dist/client/workflow/core/judgeScorer.js +0 -544
  449. package/dist/client/workflow/core/responseConditioner.js +0 -225
  450. package/dist/client/workflow/core/types/conditionerTypes.js +0 -7
  451. package/dist/client/workflow/core/types/ensembleTypes.js +0 -7
  452. package/dist/client/workflow/core/types/index.js +0 -7
  453. package/dist/client/workflow/core/types/judgeTypes.js +0 -7
  454. package/dist/client/workflow/core/types/layerTypes.js +0 -7
  455. package/dist/client/workflow/core/types/registryTypes.js +0 -7
  456. package/dist/client/workflow/core/workflowRegistry.js +0 -304
  457. package/dist/client/workflow/core/workflowRunner.js +0 -586
  458. package/dist/client/workflow/index.js +0 -50
  459. package/dist/client/workflow/types.js +0 -9
  460. package/dist/client/workflow/utils/types/index.js +0 -7
  461. package/dist/client/workflow/utils/workflowMetrics.js +0 -311
  462. package/dist/client/workflow/utils/workflowValidation.js +0 -420
  463. package/dist/client/workflow/workflows/adaptiveWorkflow.js +0 -366
  464. package/dist/client/workflow/workflows/consensusWorkflow.js +0 -192
  465. package/dist/client/workflow/workflows/fallbackWorkflow.js +0 -225
  466. package/dist/client/workflow/workflows/multiJudgeWorkflow.js +0 -351
  467. /package/dist/client/{client/reactHooks.js → reactHooks.js} +0 -0
@@ -1,1200 +0,0 @@
1
- import { createGoogleGenerativeAI } from "@ai-sdk/google";
2
- import { embed, embedMany, stepCountIs, streamText, } from "ai";
3
- import { ErrorCategory, ErrorSeverity, GoogleAIModels, } from "../constants/enums.js";
4
- import { BaseProvider } from "../core/baseProvider.js";
5
- import { DEFAULT_MAX_STEPS } from "../core/constants.js";
6
- import { streamAnalyticsCollector } from "../core/streamAnalytics.js";
7
- import { ATTR, tracers, withClientSpan } from "../telemetry/index.js";
8
- import { AuthenticationError, NetworkError, ProviderError, RateLimitError, } from "../types/errors.js";
9
- import { ERROR_CODES, NeuroLinkError } from "../utils/errorHandling.js";
10
- import { logger } from "../utils/logger.js";
11
- import { isGemini3Model } from "../utils/modelDetection.js";
12
- import { composeAbortSignals, createTimeoutController, TimeoutError, } from "../utils/timeout.js";
13
- import { estimateTokens } from "../utils/tokenEstimation.js";
14
- import { buildNativeConfig, buildNativeToolDeclarations, collectStreamChunks, collectStreamChunksIncremental, computeMaxSteps, createTextChannel, executeNativeToolCalls, extractTextFromParts, handleMaxStepsTermination, pushModelResponseToHistory, sanitizeToolsForGemini, } from "./googleNativeGemini3.js";
15
- import { toAnalyticsStreamResult } from "./providerTypeUtils.js";
16
- // Google AI Live API types now imported from ../types/providerSpecific.js
17
- // Import proper types for multimodal message handling
18
- // Create Google GenAI client
19
- async function createGoogleGenAIClient(apiKey) {
20
- const mod = await import("@google/genai");
21
- const ctor = mod.GoogleGenAI;
22
- if (!ctor) {
23
- throw new NeuroLinkError({
24
- code: ERROR_CODES.INVALID_CONFIGURATION,
25
- message: "@google/genai does not export GoogleGenAI",
26
- category: ErrorCategory.CONFIGURATION,
27
- severity: ErrorSeverity.CRITICAL,
28
- retriable: false,
29
- context: { module: "@google/genai", expectedExport: "GoogleGenAI" },
30
- });
31
- }
32
- const Ctor = ctor;
33
- return new Ctor({ apiKey });
34
- }
35
- // Environment variable setup
36
- if (!process.env.GOOGLE_GENERATIVE_AI_API_KEY &&
37
- process.env.GOOGLE_AI_API_KEY) {
38
- process.env.GOOGLE_GENERATIVE_AI_API_KEY = process.env.GOOGLE_AI_API_KEY;
39
- }
40
- /**
41
- * Google AI Studio provider implementation using BaseProvider
42
- * Migrated from original GoogleAIStudio class to new factory pattern
43
- *
44
- * @important Structured Output Limitation
45
- * Google Gemini models cannot combine function calling (tools) with structured
46
- * output (JSON schema). When using schemas with output.format: "json", you MUST
47
- * set disableTools: true.
48
- *
49
- * Error without disableTools:
50
- * "Function calling with a response mime type: 'application/json' is unsupported"
51
- *
52
- * This is a Google API limitation documented at:
53
- * https://ai.google.dev/gemini-api/docs/function-calling
54
- *
55
- * @example
56
- * ```typescript
57
- * // ✅ Correct usage with schemas
58
- * const provider = new GoogleAIStudioProvider("gemini-2.5-flash");
59
- * const result = await provider.generate({
60
- * input: { text: "Analyze data" },
61
- * schema: MySchema,
62
- * output: { format: "json" },
63
- * disableTools: true // Required
64
- * });
65
- * ```
66
- *
67
- * @note Gemini 3 Pro Preview (November 2025) will support combining tools + schemas
68
- * @note "Too many states for serving" errors can occur with complex schemas + tools.
69
- * Solution: Simplify schema or use disableTools: true
70
- */
71
- export class GoogleAIStudioProvider extends BaseProvider {
72
- constructor(modelName, sdk) {
73
- super(modelName, "google-ai", sdk);
74
- logger.debug("GoogleAIStudioProvider initialized", {
75
- model: this.modelName,
76
- provider: this.providerName,
77
- sdkProvided: !!sdk,
78
- });
79
- }
80
- // ===================
81
- // ABSTRACT METHOD IMPLEMENTATIONS
82
- // ===================
83
- getProviderName() {
84
- return "google-ai";
85
- }
86
- getDefaultModel() {
87
- return process.env.GOOGLE_AI_MODEL || GoogleAIModels.GEMINI_2_5_FLASH;
88
- }
89
- /**
90
- * 🔧 PHASE 2: Return AI SDK model instance for tool calling
91
- */
92
- getAISDKModel() {
93
- const apiKey = this.getApiKey();
94
- const google = createGoogleGenerativeAI({ apiKey });
95
- return google(this.modelName);
96
- }
97
- formatProviderError(error) {
98
- if (error instanceof TimeoutError) {
99
- return new NetworkError(error.message, this.providerName);
100
- }
101
- const errorRecord = error;
102
- const message = typeof errorRecord?.message === "string"
103
- ? errorRecord.message
104
- : "Unknown error";
105
- if (message.includes("API_KEY_INVALID")) {
106
- return new AuthenticationError("Invalid Google AI API key. Please check your GOOGLE_AI_API_KEY environment variable.", this.providerName);
107
- }
108
- if (message.includes("RATE_LIMIT_EXCEEDED")) {
109
- return new RateLimitError("Google AI rate limit exceeded. Please try again later.", this.providerName);
110
- }
111
- return new ProviderError(`Google AI error: ${message}`, this.providerName);
112
- }
113
- /**
114
- * Overrides the BaseProvider's image generation method to implement it for Google AI.
115
- * This method calls the Google AI API to generate an image from a prompt.
116
- * @param options The generation options containing the prompt.
117
- * @returns A promise that resolves to the generation result, including the image data.
118
- */
119
- async executeImageGeneration(options) {
120
- const prompt = options.prompt || options.input?.text || "";
121
- const imageModelName = options.model || this.modelName;
122
- const startTime = Date.now();
123
- const apiKey = this.getApiKey();
124
- logger.info("🎨 Starting Google AI Studio image generation", {
125
- model: imageModelName,
126
- prompt: prompt.substring(0, 100),
127
- provider: this.providerName,
128
- });
129
- // Use the @google/genai client for image generation
130
- let client;
131
- try {
132
- client = await createGoogleGenAIClient(apiKey);
133
- }
134
- catch {
135
- throw new AuthenticationError("Missing '@google/genai'. Install with: npm install @google/genai", this.providerName);
136
- }
137
- try {
138
- // Build content array with multimodal support
139
- const imageParts = await Promise.all((options.input?.images || []).map(async (image) => {
140
- // Handle ImageWithAltText objects
141
- if (typeof image === "object" && "url" in image) {
142
- const imageUrl = image.url;
143
- if (imageUrl.startsWith("http")) {
144
- const response = await fetch(imageUrl);
145
- if (!response.ok) {
146
- throw new Error(`Failed to fetch image from ${imageUrl}: ${response.status} ${response.statusText}`);
147
- }
148
- const arrayBuffer = await response.arrayBuffer();
149
- const buffer = Buffer.from(arrayBuffer);
150
- const mimeType = this.detectImageType(buffer);
151
- logger.debug(`Downloaded and detected image MIME type: ${mimeType}`);
152
- return {
153
- inlineData: {
154
- mimeType,
155
- data: buffer.toString("base64"),
156
- },
157
- };
158
- }
159
- // Base64 URL in ImageWithAltText
160
- const buffer = Buffer.from(imageUrl, "base64");
161
- const mimeType = this.detectImageType(buffer);
162
- return {
163
- inlineData: {
164
- mimeType,
165
- data: buffer.toString("base64"),
166
- },
167
- };
168
- }
169
- // Handle string URLs
170
- if (typeof image === "string" && image.startsWith("http")) {
171
- const response = await fetch(image);
172
- if (!response.ok) {
173
- throw new Error(`Failed to fetch image from ${image}: ${response.status} ${response.statusText}`);
174
- }
175
- const arrayBuffer = await response.arrayBuffer();
176
- const buffer = Buffer.from(arrayBuffer);
177
- const mimeType = this.detectImageType(buffer);
178
- logger.debug(`Downloaded and detected image MIME type: ${mimeType}`);
179
- return {
180
- inlineData: {
181
- mimeType,
182
- data: buffer.toString("base64"),
183
- },
184
- };
185
- }
186
- // Handle Buffer or base64 string
187
- const buffer = Buffer.isBuffer(image)
188
- ? image
189
- : typeof image === "string"
190
- ? Buffer.from(image, "base64")
191
- : Buffer.from(""); // Fallback for unexpected types
192
- const mimeType = this.detectImageType(buffer);
193
- logger.debug(`Detected image MIME type: ${mimeType}`);
194
- return {
195
- inlineData: {
196
- mimeType,
197
- data: buffer.toString("base64"),
198
- },
199
- };
200
- }));
201
- const contents = [
202
- {
203
- role: "user",
204
- parts: [{ text: prompt }, ...imageParts],
205
- },
206
- ];
207
- // Configure for image generation
208
- const generateConfig = {
209
- responseModalities: ["IMAGE", "TEXT"], // This is the key setting for image generation
210
- };
211
- logger.debug("Starting image generation request", {
212
- model: imageModelName,
213
- contentParts: contents[0].parts.length,
214
- responseModalities: generateConfig.responseModalities,
215
- });
216
- // Try streaming approach first
217
- let imageData = null;
218
- let textContent = "";
219
- try {
220
- // Await the Promise to get the AsyncIterable
221
- const stream = await client.models.generateContentStream({
222
- model: imageModelName,
223
- contents: contents,
224
- config: generateConfig,
225
- });
226
- // Process the stream
227
- for await (const chunk of stream) {
228
- logger.debug("Received chunk", {
229
- hasCandidate: !!chunk.candidates?.[0],
230
- hasContent: !!chunk.candidates?.[0]?.content,
231
- hasParts: !!chunk.candidates?.[0]?.content?.parts,
232
- });
233
- const candidate = chunk.candidates?.[0];
234
- if (candidate?.content?.parts) {
235
- for (const part of candidate.content.parts) {
236
- // Check for image data
237
- if ("inlineData" in part && part.inlineData?.data) {
238
- const foundImageData = part.inlineData.data;
239
- imageData = foundImageData;
240
- const mimeType = part.inlineData.mimeType || "image/png";
241
- logger.info("Image generation successful", {
242
- model: imageModelName,
243
- mimeType,
244
- dataLength: foundImageData.length,
245
- responseTime: Date.now() - startTime,
246
- });
247
- const result = {
248
- content: `Generated image using ${imageModelName} (${mimeType})`,
249
- imageOutput: {
250
- base64: foundImageData,
251
- },
252
- provider: this.providerName,
253
- model: imageModelName,
254
- usage: {
255
- input: this.estimateTokenCount(prompt),
256
- output: 0,
257
- total: this.estimateTokenCount(prompt),
258
- },
259
- };
260
- return await this.enhanceResult(result, options, startTime);
261
- }
262
- // Check for text content
263
- if ("text" in part && part.text) {
264
- textContent += part.text;
265
- logger.debug("Received text content", {
266
- text: part.text.substring(0, 100),
267
- });
268
- }
269
- }
270
- }
271
- }
272
- }
273
- catch (streamError) {
274
- logger.debug("Streaming failed, trying non-streaming approach", {
275
- error: streamError instanceof Error
276
- ? streamError.message
277
- : String(streamError),
278
- });
279
- }
280
- // If no image was found, try non-streaming approach
281
- if (!imageData) {
282
- logger.debug("Trying non-streaming approach");
283
- const response = await client.models.generateContent({
284
- model: imageModelName,
285
- contents: contents,
286
- config: generateConfig,
287
- });
288
- const candidate = response.candidates?.[0];
289
- if (candidate?.content?.parts) {
290
- for (const part of candidate.content.parts) {
291
- if ("inlineData" in part && part.inlineData?.data) {
292
- const foundImageData = part.inlineData.data;
293
- imageData = foundImageData;
294
- const mimeType = part.inlineData.mimeType || "image/png";
295
- logger.info("Image generation successful (non-streaming)", {
296
- model: imageModelName,
297
- mimeType,
298
- dataLength: foundImageData.length,
299
- responseTime: Date.now() - startTime,
300
- });
301
- const result = {
302
- content: `Generated image using ${imageModelName} (${mimeType})`,
303
- imageOutput: {
304
- base64: foundImageData,
305
- },
306
- provider: this.providerName,
307
- model: imageModelName,
308
- usage: {
309
- input: this.estimateTokenCount(prompt),
310
- output: 0,
311
- total: this.estimateTokenCount(prompt),
312
- },
313
- };
314
- return await this.enhanceResult(result, options, startTime);
315
- }
316
- if ("text" in part && part.text) {
317
- textContent += part.text;
318
- }
319
- }
320
- }
321
- }
322
- // If we reach here, no image was generated
323
- logger.warn("No image data found in response", {
324
- model: imageModelName,
325
- prompt: prompt.substring(0, 100),
326
- hasTextContent: !!textContent,
327
- textContent: textContent.substring(0, 200),
328
- });
329
- throw new ProviderError(textContent ||
330
- `Image generation completed but no image data was returned. This may indicate an issue with the model "${imageModelName}" or the prompt: "${prompt}". Please try again or use a different model.`, this.providerName);
331
- }
332
- catch (error) {
333
- logger.error("Image generation failed", {
334
- error: error instanceof Error ? error.message : String(error),
335
- model: imageModelName,
336
- prompt: prompt.substring(0, 100),
337
- });
338
- throw this.handleProviderError(error);
339
- }
340
- }
341
- /**
342
- * Detect image MIME type from buffer
343
- */
344
- detectImageType(buffer) {
345
- // Check PNG signature
346
- if (buffer.length >= 8 &&
347
- buffer[0] === 0x89 &&
348
- buffer[1] === 0x50 &&
349
- buffer[2] === 0x4e &&
350
- buffer[3] === 0x47) {
351
- return "image/png";
352
- }
353
- // Check JPEG signature
354
- if (buffer.length >= 3 &&
355
- buffer[0] === 0xff &&
356
- buffer[1] === 0xd8 &&
357
- buffer[2] === 0xff) {
358
- return "image/jpeg";
359
- }
360
- // Check WebP signature
361
- if (buffer.length >= 12 &&
362
- buffer[0] === 0x52 &&
363
- buffer[1] === 0x49 &&
364
- buffer[2] === 0x46 &&
365
- buffer[3] === 0x46 &&
366
- buffer[8] === 0x57 &&
367
- buffer[9] === 0x45 &&
368
- buffer[10] === 0x42 &&
369
- buffer[11] === 0x50) {
370
- return "image/webp";
371
- }
372
- // Check GIF signature
373
- if (buffer.length >= 6 &&
374
- buffer[0] === 0x47 &&
375
- buffer[1] === 0x49 &&
376
- buffer[2] === 0x46) {
377
- return "image/gif";
378
- }
379
- // Default to PNG if unknown
380
- return "image/png";
381
- }
382
- /**
383
- * Estimate token count from text using centralized estimation with provider multipliers
384
- */
385
- estimateTokenCount(text) {
386
- return estimateTokens(text, "google-ai");
387
- }
388
- // executeGenerate removed - BaseProvider handles all generation with tools
389
- async executeStream(options, analysisSchema) {
390
- // Check if this is a Gemini 3 model with tools - use native SDK for thought_signature
391
- const gemini3CheckModelName = options.model || this.modelName;
392
- // Structured output (analysisSchema, JSON format, or schema) is incompatible with tools on Gemini.
393
- // Compute once and reuse in both the native Gemini 3 gate and the streamText fallback path.
394
- const wantsStructuredOutput = analysisSchema || options.output?.format === "json" || options.schema;
395
- // Check for tools from options AND from SDK (MCP tools)
396
- // Need to check early if we should route to native SDK
397
- const gemini3CheckShouldUseTools = !options.disableTools && this.supportsTools() && !wantsStructuredOutput;
398
- const optionTools = options.tools || {};
399
- const sdkTools = gemini3CheckShouldUseTools ? await this.getAllTools() : {};
400
- const combinedToolCount = Object.keys(optionTools).length + Object.keys(sdkTools).length;
401
- const hasTools = gemini3CheckShouldUseTools && combinedToolCount > 0;
402
- if (isGemini3Model(gemini3CheckModelName) && hasTools) {
403
- // Merge SDK tools into options for native SDK path
404
- let mergedOptions = {
405
- ...options,
406
- tools: { ...sdkTools, ...optionTools },
407
- };
408
- // Check for tools + JSON schema conflict (Gemini limitation)
409
- const wantsJsonOutput = options.output?.format === "json" || options.schema;
410
- if (wantsJsonOutput &&
411
- mergedOptions.tools &&
412
- Object.keys(mergedOptions.tools).length > 0 &&
413
- !mergedOptions.disableTools) {
414
- logger.warn("[GoogleAIStudio] Gemini does not support tools and JSON schema output simultaneously. Disabling tools for this request.");
415
- mergedOptions = { ...mergedOptions, disableTools: true, tools: {} };
416
- }
417
- // Only route to native path if tools are still active after conflict check
418
- const hasActiveTools = !mergedOptions.disableTools &&
419
- mergedOptions.tools &&
420
- Object.keys(mergedOptions.tools).length > 0;
421
- if (hasActiveTools) {
422
- logger.info("[GoogleAIStudio] Routing Gemini 3 to native SDK for tool calling", {
423
- model: gemini3CheckModelName,
424
- optionToolCount: Object.keys(optionTools).length,
425
- sdkToolCount: Object.keys(sdkTools).length,
426
- totalToolCount: combinedToolCount,
427
- });
428
- return this.executeNativeGemini3Stream(mergedOptions);
429
- }
430
- // Fall through to standard stream path using merged options (tools disabled for schema)
431
- options = mergedOptions;
432
- }
433
- // Phase 1: if audio input present, bridge to Gemini Live (Studio) using @google/genai
434
- if (options.input?.audio) {
435
- return await this.executeAudioStreamViaGeminiLive(options);
436
- }
437
- this.validateStreamOptions(options);
438
- const startTime = Date.now();
439
- const apiKey = this.getApiKey();
440
- // Ensure environment variable is set for @ai-sdk/google
441
- if (!process.env.GOOGLE_GENERATIVE_AI_API_KEY) {
442
- process.env.GOOGLE_GENERATIVE_AI_API_KEY = apiKey;
443
- }
444
- const model = await this.getAISDKModelWithMiddleware(options);
445
- const timeout = this.getTimeout(options);
446
- const timeoutController = createTimeoutController(timeout, this.providerName, "stream");
447
- try {
448
- // Get tools consistently with generate method (include user-provided RAG tools)
449
- // wantsStructuredOutput already computed before the Gemini 3 native-routing gate
450
- if (wantsStructuredOutput &&
451
- !options.disableTools &&
452
- this.supportsTools()) {
453
- logger.warn("[GoogleAIStudio] Structured output active — disabling tools (Gemini limitation).");
454
- }
455
- const shouldUseTools = !options.disableTools && this.supportsTools() && !wantsStructuredOutput;
456
- const baseTools = shouldUseTools ? await this.getAllTools() : {};
457
- const rawTools = shouldUseTools
458
- ? { ...baseTools, ...(options.tools || {}) }
459
- : {};
460
- // Sanitize tool schemas for Gemini proto compatibility (converts anyOf/oneOf unions to string)
461
- let tools;
462
- if (Object.keys(rawTools).length > 0) {
463
- const sanitized = sanitizeToolsForGemini(rawTools);
464
- if (sanitized.dropped.length > 0) {
465
- logger.warn(`[GoogleAIStudio] Dropped ${sanitized.dropped.length} incompatible tool(s): ${sanitized.dropped.join(", ")}`);
466
- }
467
- tools =
468
- Object.keys(sanitized.tools).length > 0 ? sanitized.tools : undefined;
469
- }
470
- else {
471
- tools = undefined;
472
- }
473
- // Build message array from options with multimodal support
474
- // Using protected helper from BaseProvider to eliminate code duplication
475
- const messages = await this.buildMessagesForStream(options);
476
- const result = await streamText({
477
- model,
478
- messages: messages,
479
- temperature: options.temperature,
480
- maxOutputTokens: options.maxTokens, // No default limit - unlimited unless specified
481
- tools,
482
- stopWhen: stepCountIs(options.maxSteps || DEFAULT_MAX_STEPS),
483
- toolChoice: shouldUseTools ? "auto" : "none",
484
- abortSignal: composeAbortSignals(options.abortSignal, timeoutController?.controller.signal),
485
- experimental_telemetry: this.telemetryHandler.getTelemetryConfig(options),
486
- // Gemini 3: use thinkingLevel via providerOptions
487
- // Gemini 2.5: use thinkingBudget via providerOptions
488
- ...(options.thinkingConfig?.enabled && {
489
- providerOptions: {
490
- google: {
491
- thinkingConfig: {
492
- ...(options.thinkingConfig.thinkingLevel && {
493
- thinkingLevel: options.thinkingConfig.thinkingLevel,
494
- }),
495
- ...(options.thinkingConfig.budgetTokens &&
496
- !options.thinkingConfig.thinkingLevel && {
497
- thinkingBudget: options.thinkingConfig.budgetTokens,
498
- }),
499
- includeThoughts: true,
500
- },
501
- },
502
- },
503
- }),
504
- onStepFinish: ({ toolCalls, toolResults }) => {
505
- this.handleToolExecutionStorage(toolCalls, toolResults, options, new Date()).catch((error) => {
506
- logger.warn("[GoogleAiStudioProvider] Failed to store tool executions", {
507
- provider: this.providerName,
508
- error: error instanceof Error ? error.message : String(error),
509
- });
510
- });
511
- },
512
- });
513
- // Defer timeout cleanup until the stream completes or errors.
514
- // Guard against NoOutputGeneratedError becoming an unhandled rejection.
515
- Promise.resolve(result.text)
516
- .catch((err) => {
517
- logger.debug("Stream text promise rejected (expected for empty streams)", {
518
- error: err instanceof Error ? err.message : String(err),
519
- });
520
- })
521
- .finally(() => timeoutController?.cleanup());
522
- // Transform string stream to content object stream using BaseProvider method
523
- const transformedStream = this.createTextStream(result);
524
- // Create analytics promise that resolves after stream completion
525
- const analyticsPromise = streamAnalyticsCollector.createAnalytics(this.providerName, this.modelName, toAnalyticsStreamResult(result), Date.now() - startTime, {
526
- requestId: `google-ai-stream-${Date.now()}`,
527
- streamingMode: true,
528
- });
529
- return {
530
- stream: transformedStream,
531
- provider: this.providerName,
532
- model: this.modelName,
533
- analytics: analyticsPromise,
534
- metadata: {
535
- startTime,
536
- streamId: `google-ai-${Date.now()}`,
537
- },
538
- };
539
- }
540
- catch (error) {
541
- timeoutController?.cleanup();
542
- throw this.handleProviderError(error);
543
- }
544
- }
545
- /**
546
- * Execute stream using native @google/genai SDK for Gemini 3 models
547
- * This bypasses @ai-sdk/google to properly handle thought_signature
548
- */
549
- async executeNativeGemini3Stream(options) {
550
- const modelName = options.model || this.modelName;
551
- return withClientSpan({
552
- name: "neurolink.provider.stream",
553
- tracer: tracers.provider,
554
- attributes: {
555
- [ATTR.GEN_AI_SYSTEM]: "google-ai",
556
- [ATTR.GEN_AI_MODEL]: modelName,
557
- [ATTR.GEN_AI_OPERATION]: "stream",
558
- [ATTR.NL_PROVIDER]: this.providerName,
559
- },
560
- }, async (span) => {
561
- const startTime = Date.now();
562
- const timeout = this.getTimeout(options);
563
- const timeoutController = createTimeoutController(timeout, this.providerName, "stream");
564
- try {
565
- const apiKey = this.getApiKey();
566
- const client = await createGoogleGenAIClient(apiKey);
567
- logger.debug("[GoogleAIStudio] Using native @google/genai for Gemini 3", {
568
- model: modelName,
569
- hasTools: !!options.tools && Object.keys(options.tools).length > 0,
570
- });
571
- // Build contents from input
572
- const currentContents = [{ role: "user", parts: [{ text: options.input.text }] }];
573
- // Convert tools
574
- let toolsConfig;
575
- let executeMap = new Map();
576
- if (options.tools &&
577
- Object.keys(options.tools).length > 0 &&
578
- !options.disableTools) {
579
- const result = buildNativeToolDeclarations(options.tools);
580
- toolsConfig = result.toolsConfig;
581
- executeMap = result.executeMap;
582
- logger.debug("[GoogleAIStudio] Converted tools for native SDK", {
583
- toolCount: toolsConfig[0].functionDeclarations.length,
584
- toolNames: toolsConfig[0].functionDeclarations.map((t) => t.name),
585
- });
586
- }
587
- const config = buildNativeConfig(options, toolsConfig);
588
- const maxSteps = computeMaxSteps(options.maxSteps);
589
- // Compose abort signal from user signal + timeout
590
- const composedSignal = composeAbortSignals(options.abortSignal, timeoutController?.controller.signal);
591
- // Create a push-based text channel so the caller receives tokens as
592
- // they arrive from the network rather than after full buffering.
593
- const channel = createTextChannel();
594
- // Shared mutable state updated by the background agentic loop.
595
- const allToolCalls = [];
596
- // analyticsResolvers lets the background loop settle the analytics
597
- // promise once token counts are known (after the loop completes).
598
- let analyticsResolve;
599
- let analyticsReject;
600
- const analyticsPromise = new Promise((res, rej) => {
601
- analyticsResolve = res;
602
- analyticsReject = rej;
603
- });
604
- // Shared metadata object mutated by the background loop so the
605
- // returned object reflects the final values after stream completion.
606
- const metadata = {
607
- streamId: `native-${Date.now()}`,
608
- startTime,
609
- responseTime: 0,
610
- totalToolExecutions: 0,
611
- };
612
- // Run the agentic loop in the background without awaiting it here,
613
- // so we can return the StreamResult (with channel.iterable) immediately.
614
- const loopPromise = (async () => {
615
- let lastStepText = "";
616
- let totalInputTokens = 0;
617
- let totalOutputTokens = 0;
618
- let step = 0;
619
- let completedWithFinalAnswer = false;
620
- const failedTools = new Map();
621
- try {
622
- // Agentic loop for tool calling
623
- while (step < maxSteps) {
624
- if (composedSignal?.aborted) {
625
- throw composedSignal.reason instanceof Error
626
- ? composedSignal.reason
627
- : new Error("Request aborted");
628
- }
629
- step++;
630
- logger.debug(`[GoogleAIStudio] Native SDK step ${step}/${maxSteps}`);
631
- try {
632
- const rawStream = await client.models.generateContentStream({
633
- model: modelName,
634
- contents: currentContents,
635
- config,
636
- ...(composedSignal
637
- ? { httpOptions: { signal: composedSignal } }
638
- : {}),
639
- });
640
- // For every step, use incremental collection so text parts
641
- // are pushed to the channel as they arrive. For intermediate
642
- // steps (those that produce function calls) we still need the
643
- // complete rawResponseParts for pushModelResponseToHistory,
644
- // which collectStreamChunksIncremental provides at stream end.
645
- const chunkResult = await collectStreamChunksIncremental(rawStream, channel);
646
- totalInputTokens += chunkResult.inputTokens;
647
- totalOutputTokens += chunkResult.outputTokens;
648
- const stepText = extractTextFromParts(chunkResult.rawResponseParts);
649
- // If no function calls, this was the final step — channel
650
- // already received all text parts incrementally.
651
- if (chunkResult.stepFunctionCalls.length === 0) {
652
- completedWithFinalAnswer = true;
653
- break;
654
- }
655
- lastStepText = stepText;
656
- // Record tool call events on the span
657
- for (const fc of chunkResult.stepFunctionCalls) {
658
- span.addEvent("gen_ai.tool_call", {
659
- "tool.name": fc.name,
660
- "tool.step": step,
661
- });
662
- }
663
- logger.debug(`[GoogleAIStudio] Executing ${chunkResult.stepFunctionCalls.length} function calls`);
664
- // Add model response with ALL parts (including thoughtSignature) to history
665
- pushModelResponseToHistory(currentContents, chunkResult.rawResponseParts, chunkResult.stepFunctionCalls);
666
- const functionResponses = await executeNativeToolCalls("[GoogleAIStudio]", chunkResult.stepFunctionCalls, executeMap, failedTools, allToolCalls, { abortSignal: composedSignal });
667
- // Add function responses to history — the @google/genai SDK
668
- // only accepts "user" and "model" as valid roles in contents.
669
- // Function/tool responses must use role: "user" (matching the
670
- // SDK's own automaticFunctionCalling implementation).
671
- currentContents.push({
672
- role: "user",
673
- parts: functionResponses,
674
- });
675
- }
676
- catch (error) {
677
- logger.error("[GoogleAIStudio] Native SDK error", error);
678
- throw this.handleProviderError(error);
679
- }
680
- }
681
- // Handle max-steps termination: if the model was still calling
682
- // tools when we hit the limit, push a synthetic final message.
683
- const hitStepLimitWithoutFinalAnswer = step >= maxSteps && !completedWithFinalAnswer;
684
- if (hitStepLimitWithoutFinalAnswer) {
685
- const fallback = handleMaxStepsTermination("[GoogleAIStudio]", step, maxSteps, "", // finalText is empty — model didn't stop on its own
686
- lastStepText);
687
- if (fallback) {
688
- channel.push(fallback);
689
- }
690
- }
691
- const responseTime = Date.now() - startTime;
692
- // Update shared metadata so the returned object reflects final values.
693
- metadata.responseTime = responseTime;
694
- metadata.totalToolExecutions = allToolCalls.length;
695
- // Set token usage and finish reason on the span
696
- span.setAttribute(ATTR.GEN_AI_INPUT_TOKENS, totalInputTokens);
697
- span.setAttribute(ATTR.GEN_AI_OUTPUT_TOKENS, totalOutputTokens);
698
- span.setAttribute(ATTR.GEN_AI_FINISH_REASON, hitStepLimitWithoutFinalAnswer ? "max_steps" : "stop");
699
- analyticsResolve({
700
- provider: this.providerName,
701
- model: modelName,
702
- tokenUsage: {
703
- input: totalInputTokens,
704
- output: totalOutputTokens,
705
- total: totalInputTokens + totalOutputTokens,
706
- },
707
- requestDuration: responseTime,
708
- timestamp: new Date().toISOString(),
709
- });
710
- channel.close();
711
- }
712
- catch (err) {
713
- channel.error(err);
714
- analyticsReject(err);
715
- }
716
- finally {
717
- timeoutController?.cleanup();
718
- }
719
- })();
720
- // Suppress unhandled-rejection warnings on loopPromise — errors are
721
- // forwarded to the channel and will surface when the caller iterates.
722
- loopPromise.catch(() => undefined);
723
- return {
724
- stream: channel.iterable,
725
- provider: this.providerName,
726
- model: modelName,
727
- toolCalls: allToolCalls,
728
- analytics: analyticsPromise,
729
- metadata,
730
- };
731
- }
732
- finally {
733
- // Timeout controller cleanup is managed inside the background loop
734
- }
735
- });
736
- }
737
- /**
738
- * Execute generate using native @google/genai SDK for Gemini 3 models
739
- * This bypasses @ai-sdk/google to properly handle thought_signature
740
- */
741
- async executeNativeGemini3Generate(options) {
742
- const modelName = options.model || this.modelName;
743
- return withClientSpan({
744
- name: "neurolink.provider.generate",
745
- tracer: tracers.provider,
746
- attributes: {
747
- [ATTR.GEN_AI_SYSTEM]: "google-ai",
748
- [ATTR.GEN_AI_MODEL]: modelName,
749
- [ATTR.GEN_AI_OPERATION]: "generate",
750
- [ATTR.NL_PROVIDER]: this.providerName,
751
- },
752
- }, async (span) => {
753
- const startTime = Date.now();
754
- const timeout = this.getTimeout(options);
755
- const timeoutController = createTimeoutController(timeout, this.providerName, "generate");
756
- try {
757
- const apiKey = this.getApiKey();
758
- const client = await createGoogleGenAIClient(apiKey);
759
- logger.debug("[GoogleAIStudio] Using native @google/genai for Gemini 3 generate", {
760
- model: modelName,
761
- hasTools: !!options.tools && Object.keys(options.tools).length > 0,
762
- });
763
- // Build contents from input
764
- // Prefer input.text over prompt — processCSVFilesForNativeSDK enriches
765
- // input.text with inlined CSV data, so using prompt first would discard it.
766
- const promptText = options.input?.text || options.prompt || "";
767
- const currentContents = [{ role: "user", parts: [{ text: promptText }] }];
768
- // Convert tools (merge SDK tools with options.tools)
769
- let toolsConfig;
770
- let executeMap = new Map();
771
- const shouldUseTools = !options.disableTools;
772
- if (shouldUseTools) {
773
- const sdkTools = await this.getAllTools();
774
- const mergedTools = { ...sdkTools, ...(options.tools || {}) };
775
- if (Object.keys(mergedTools).length > 0) {
776
- const result = buildNativeToolDeclarations(mergedTools);
777
- toolsConfig = result.toolsConfig;
778
- executeMap = result.executeMap;
779
- logger.debug("[GoogleAIStudio] Converted tools for native SDK generate", {
780
- toolCount: toolsConfig[0].functionDeclarations.length,
781
- toolNames: toolsConfig[0].functionDeclarations.map((t) => t.name),
782
- });
783
- }
784
- }
785
- const config = buildNativeConfig(options, toolsConfig);
786
- const composedSignal = composeAbortSignals(options.abortSignal, timeoutController?.controller.signal);
787
- const maxSteps = computeMaxSteps(options.maxSteps);
788
- let finalText = "";
789
- let lastStepText = "";
790
- let totalInputTokens = 0;
791
- let totalOutputTokens = 0;
792
- const allToolCalls = [];
793
- const toolExecutions = [];
794
- let step = 0;
795
- const failedTools = new Map();
796
- // Agentic loop for tool calling
797
- while (step < maxSteps) {
798
- if (composedSignal?.aborted) {
799
- throw composedSignal.reason instanceof Error
800
- ? composedSignal.reason
801
- : new Error("Request aborted");
802
- }
803
- step++;
804
- logger.debug(`[GoogleAIStudio] Native SDK generate step ${step}/${maxSteps}`);
805
- try {
806
- const stream = await client.models.generateContentStream({
807
- model: modelName,
808
- contents: currentContents,
809
- config,
810
- ...(composedSignal
811
- ? { httpOptions: { signal: composedSignal } }
812
- : {}),
813
- });
814
- const chunkResult = await collectStreamChunks(stream);
815
- totalInputTokens += chunkResult.inputTokens;
816
- totalOutputTokens += chunkResult.outputTokens;
817
- const stepText = extractTextFromParts(chunkResult.rawResponseParts);
818
- // If no function calls, we're done
819
- if (chunkResult.stepFunctionCalls.length === 0) {
820
- finalText = stepText;
821
- break;
822
- }
823
- lastStepText = stepText;
824
- // Record tool call events on the span
825
- for (const fc of chunkResult.stepFunctionCalls) {
826
- span.addEvent("gen_ai.tool_call", {
827
- "tool.name": fc.name,
828
- "tool.step": step,
829
- });
830
- }
831
- logger.debug(`[GoogleAIStudio] Executing ${chunkResult.stepFunctionCalls.length} function calls in generate`);
832
- // Add model response with ALL parts (including thoughtSignature) to history
833
- // This is critical for Gemini 3 - it requires thought signatures in subsequent turns
834
- pushModelResponseToHistory(currentContents, chunkResult.rawResponseParts, chunkResult.stepFunctionCalls);
835
- const functionResponses = await executeNativeToolCalls("[GoogleAIStudio]", chunkResult.stepFunctionCalls, executeMap, failedTools, allToolCalls, { toolExecutions, abortSignal: composedSignal });
836
- // Add function responses to history — the @google/genai SDK
837
- // only accepts "user" and "model" as valid roles in contents.
838
- // Function/tool responses must use role: "user" (matching the
839
- // SDK's own automaticFunctionCalling implementation).
840
- currentContents.push({
841
- role: "user",
842
- parts: functionResponses,
843
- });
844
- }
845
- catch (error) {
846
- logger.error("[GoogleAIStudio] Native SDK generate error", error);
847
- throw this.handleProviderError(error);
848
- }
849
- }
850
- finalText = handleMaxStepsTermination("[GoogleAIStudio]", step, maxSteps, finalText, lastStepText);
851
- const responseTime = Date.now() - startTime;
852
- // Set token usage and finish reason on the span
853
- span.setAttribute(ATTR.GEN_AI_INPUT_TOKENS, totalInputTokens);
854
- span.setAttribute(ATTR.GEN_AI_OUTPUT_TOKENS, totalOutputTokens);
855
- span.setAttribute(ATTR.GEN_AI_FINISH_REASON, step >= maxSteps ? "max_steps" : "stop");
856
- // Build EnhancedGenerateResult
857
- return {
858
- content: finalText,
859
- provider: this.providerName,
860
- model: modelName,
861
- usage: {
862
- input: totalInputTokens,
863
- output: totalOutputTokens,
864
- total: totalInputTokens + totalOutputTokens,
865
- },
866
- responseTime,
867
- toolsUsed: allToolCalls.map((tc) => tc.toolName),
868
- toolExecutions: toolExecutions,
869
- enhancedWithTools: allToolCalls.length > 0,
870
- };
871
- }
872
- finally {
873
- timeoutController?.cleanup();
874
- }
875
- });
876
- }
877
- /**
878
- * Override generate to route Gemini 3 models with tools to native SDK
879
- */
880
- async generate(optionsOrPrompt) {
881
- // Normalize options
882
- const options = typeof optionsOrPrompt === "string"
883
- ? { prompt: optionsOrPrompt }
884
- : optionsOrPrompt;
885
- const modelName = options.model || this.modelName;
886
- // Check if we should use native SDK for Gemini 3 with tools
887
- const shouldUseTools = !options.disableTools && this.supportsTools();
888
- const sdkTools = shouldUseTools ? await this.getAllTools() : {};
889
- const hasTools = shouldUseTools &&
890
- (Object.keys(sdkTools).length > 0 ||
891
- (options.tools && Object.keys(options.tools).length > 0));
892
- if (isGemini3Model(modelName) && hasTools) {
893
- // Merge SDK tools into options for native SDK path
894
- let mergedOptions = {
895
- ...options,
896
- tools: { ...sdkTools, ...(options.tools || {}) },
897
- };
898
- // Check for tools + JSON schema conflict (Gemini limitation)
899
- const wantsJsonOutput = options.output?.format === "json" || options.schema;
900
- if (wantsJsonOutput &&
901
- mergedOptions.tools &&
902
- Object.keys(mergedOptions.tools).length > 0 &&
903
- !mergedOptions.disableTools) {
904
- logger.warn("[GoogleAIStudio] Gemini does not support tools and JSON schema output simultaneously. Disabling tools for this request.");
905
- mergedOptions = { ...mergedOptions, disableTools: true, tools: {} };
906
- }
907
- // Only route to native path if tools are still active after conflict check
908
- const hasActiveTools = !mergedOptions.disableTools &&
909
- mergedOptions.tools &&
910
- Object.keys(mergedOptions.tools).length > 0;
911
- if (hasActiveTools) {
912
- logger.info("[GoogleAIStudio] Routing Gemini 3 generate to native SDK for tool calling", {
913
- model: modelName,
914
- sdkToolCount: Object.keys(sdkTools).length,
915
- optionToolCount: Object.keys(options.tools || {}).length,
916
- totalToolCount: Object.keys(sdkTools).length +
917
- Object.keys(options.tools || {}).length,
918
- });
919
- return this.executeNativeGemini3Generate(mergedOptions);
920
- }
921
- // Fall through to standard generate path using merged options (tools disabled for schema)
922
- return super.generate(mergedOptions);
923
- }
924
- // Fall back to BaseProvider implementation
925
- return super.generate(options);
926
- }
927
- // ===================
928
- // HELPER METHODS
929
- // ===================
930
- async executeAudioStreamViaGeminiLive(options) {
931
- const startTime = Date.now();
932
- const apiKey = this.getApiKey();
933
- // Dynamic import to avoid hard dependency unless audio streaming is used
934
- let client;
935
- try {
936
- client = await createGoogleGenAIClient(apiKey);
937
- }
938
- catch {
939
- throw new AuthenticationError("Missing '@google/genai'. Install with: pnpm add @google/genai", this.providerName);
940
- }
941
- const model = this.modelName ||
942
- process.env.GOOGLE_VOICE_AI_MODEL ||
943
- "gemini-2.5-flash-preview-native-audio-dialog";
944
- const queue = [];
945
- let resolveNext = null;
946
- let done = false;
947
- const push = (item) => {
948
- if (done) {
949
- return;
950
- }
951
- if (item.type === "audio") {
952
- if (resolveNext) {
953
- const fn = resolveNext;
954
- resolveNext = null;
955
- fn({ value: { type: "audio", audio: item.audio }, done: false });
956
- return;
957
- }
958
- }
959
- queue.push(item);
960
- };
961
- const session = await client.live.connect({
962
- model,
963
- callbacks: {
964
- onopen: () => {
965
- // no-op
966
- },
967
- onmessage: async (message) => {
968
- try {
969
- const audio = message?.serverContent?.modelTurn?.parts?.[0]?.inlineData;
970
- if (audio?.data) {
971
- const buf = Buffer.from(String(audio.data), "base64");
972
- const chunk = {
973
- data: buf,
974
- sampleRateHz: 24000,
975
- channels: 1,
976
- encoding: "PCM16LE",
977
- };
978
- push({ type: "audio", audio: chunk });
979
- }
980
- if (message?.serverContent?.interrupted) {
981
- // allow consumer to handle; no special action required here
982
- }
983
- }
984
- catch (e) {
985
- push({ type: "error", error: e });
986
- }
987
- },
988
- onerror: (e) => {
989
- push({ type: "error", error: e });
990
- },
991
- onclose: (_e) => {
992
- push({ type: "end" });
993
- },
994
- },
995
- config: {
996
- responseModalities: ["AUDIO"],
997
- speechConfig: {
998
- voiceConfig: { prebuiltVoiceConfig: { voiceName: "Orus" } },
999
- },
1000
- },
1001
- });
1002
- // Feed upstream audio frames concurrently
1003
- (async () => {
1004
- try {
1005
- const spec = options.input?.audio;
1006
- if (!spec) {
1007
- logger.debug("[GeminiLive] No audio spec found on input; skipping upstream send");
1008
- return;
1009
- }
1010
- for await (const frame of spec.frames) {
1011
- // Zero-length frame acts as a 'flush' control signal
1012
- if (!frame || frame.byteLength === 0) {
1013
- try {
1014
- if (session.sendInput) {
1015
- await session.sendInput({ event: "flush" });
1016
- }
1017
- else if (session.sendRealtimeInput) {
1018
- await session.sendRealtimeInput({ event: "flush" });
1019
- }
1020
- }
1021
- catch (err) {
1022
- logger.debug("[GeminiLive] flush control failed (non-fatal)", {
1023
- error: err instanceof Error ? err.message : String(err),
1024
- });
1025
- }
1026
- continue;
1027
- }
1028
- // Convert PCM16LE buffer to base64 and wrap in genai Blob-like object
1029
- const base64 = frame.toString("base64");
1030
- const mimeType = `audio/pcm;rate=${spec.sampleRateHz || 16000}`;
1031
- await session.sendRealtimeInput?.({
1032
- media: { data: base64, mimeType },
1033
- });
1034
- }
1035
- // Best-effort flush signal if supported
1036
- try {
1037
- if (session.sendInput) {
1038
- await session.sendInput({ event: "flush" });
1039
- }
1040
- else if (session.sendRealtimeInput) {
1041
- await session.sendRealtimeInput({ event: "flush" });
1042
- }
1043
- }
1044
- catch (err) {
1045
- logger.debug("[GeminiLive] final flush failed (non-fatal)", {
1046
- error: err instanceof Error ? err.message : String(err),
1047
- });
1048
- }
1049
- }
1050
- catch (e) {
1051
- push({ type: "error", error: e });
1052
- }
1053
- })().catch(() => {
1054
- // ignore
1055
- });
1056
- // AsyncIterable for stream events
1057
- const asyncIterable = {
1058
- [Symbol.asyncIterator]() {
1059
- return {
1060
- async next() {
1061
- if (queue.length > 0) {
1062
- const item = queue.shift();
1063
- if (!item) {
1064
- return {
1065
- value: undefined,
1066
- done: true,
1067
- };
1068
- }
1069
- if (item.type === "audio") {
1070
- return {
1071
- value: { type: "audio", audio: item.audio },
1072
- done: false,
1073
- };
1074
- }
1075
- if (item.type === "end") {
1076
- done = true;
1077
- return {
1078
- value: undefined,
1079
- done: true,
1080
- };
1081
- }
1082
- if (item.type === "error") {
1083
- done = true;
1084
- throw item.error instanceof Error
1085
- ? item.error
1086
- : new Error(String(item.error));
1087
- }
1088
- }
1089
- if (done) {
1090
- return {
1091
- value: undefined,
1092
- done: true,
1093
- };
1094
- }
1095
- return await new Promise((resolve) => {
1096
- resolveNext = resolve;
1097
- });
1098
- },
1099
- };
1100
- },
1101
- };
1102
- return {
1103
- stream: asyncIterable,
1104
- provider: this.providerName,
1105
- model: model,
1106
- metadata: {
1107
- startTime,
1108
- streamId: `google-ai-audio-${Date.now()}`,
1109
- },
1110
- };
1111
- }
1112
- getDefaultEmbeddingModel() {
1113
- return (process.env.GOOGLE_AI_EMBEDDING_MODEL ||
1114
- process.env.GOOGLE_EMBEDDING_MODEL ||
1115
- "gemini-embedding-001");
1116
- }
1117
- /**
1118
- * Generate embeddings for text using Google AI Studio embedding models
1119
- * @param text - The text to embed
1120
- * @param modelName - The embedding model to use (default: gemini-embedding-001)
1121
- * @returns Promise resolving to the embedding vector
1122
- */
1123
- async embed(text, modelName) {
1124
- const embeddingModelName = modelName || this.getDefaultEmbeddingModel() || "gemini-embedding-001";
1125
- logger.debug("Generating embedding", {
1126
- provider: this.providerName,
1127
- model: embeddingModelName,
1128
- textLength: text.length,
1129
- });
1130
- try {
1131
- const apiKey = this.getApiKey();
1132
- const google = createGoogleGenerativeAI({ apiKey });
1133
- const embeddingModel = google.textEmbeddingModel(embeddingModelName);
1134
- const result = await embed({
1135
- model: embeddingModel,
1136
- value: text,
1137
- });
1138
- logger.debug("Embedding generated successfully", {
1139
- provider: this.providerName,
1140
- model: embeddingModelName,
1141
- embeddingDimension: result.embedding.length,
1142
- });
1143
- return result.embedding;
1144
- }
1145
- catch (error) {
1146
- logger.error("Embedding generation failed", {
1147
- error: error instanceof Error ? error.message : String(error),
1148
- model: embeddingModelName,
1149
- textLength: text.length,
1150
- });
1151
- throw this.handleProviderError(error);
1152
- }
1153
- }
1154
- /**
1155
- * Generate embeddings for multiple texts in a single batch
1156
- * @param texts - The texts to embed
1157
- * @param modelName - The embedding model to use (default: gemini-embedding-001)
1158
- * @returns Promise resolving to an array of embedding vectors
1159
- */
1160
- async embedMany(texts, modelName) {
1161
- const embeddingModelName = modelName || this.getDefaultEmbeddingModel() || "gemini-embedding-001";
1162
- logger.debug("Generating batch embeddings", {
1163
- provider: this.providerName,
1164
- model: embeddingModelName,
1165
- count: texts.length,
1166
- });
1167
- try {
1168
- const apiKey = this.getApiKey();
1169
- const google = createGoogleGenerativeAI({ apiKey });
1170
- const embeddingModel = google.textEmbeddingModel(embeddingModelName);
1171
- const result = await embedMany({
1172
- model: embeddingModel,
1173
- values: texts,
1174
- });
1175
- logger.debug("Batch embeddings generated successfully", {
1176
- provider: this.providerName,
1177
- model: embeddingModelName,
1178
- count: result.embeddings.length,
1179
- embeddingDimension: result.embeddings[0]?.length,
1180
- });
1181
- return result.embeddings;
1182
- }
1183
- catch (error) {
1184
- logger.error("Batch embedding generation failed", {
1185
- error: error instanceof Error ? error.message : String(error),
1186
- model: embeddingModelName,
1187
- count: texts.length,
1188
- });
1189
- throw this.handleProviderError(error);
1190
- }
1191
- }
1192
- getApiKey() {
1193
- const apiKey = process.env.GOOGLE_AI_API_KEY || process.env.GOOGLE_GENERATIVE_AI_API_KEY;
1194
- if (!apiKey) {
1195
- throw new AuthenticationError("GOOGLE_AI_API_KEY or GOOGLE_GENERATIVE_AI_API_KEY environment variable is not set", this.providerName);
1196
- }
1197
- return apiKey;
1198
- }
1199
- }
1200
- export default GoogleAIStudioProvider;