@juspay/neurolink 9.31.2 → 9.32.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/dist/auth/AuthProviderFactory.d.ts +71 -0
- package/dist/auth/AuthProviderFactory.js +111 -0
- package/dist/auth/AuthProviderRegistry.d.ts +33 -0
- package/dist/auth/AuthProviderRegistry.js +190 -0
- package/dist/auth/RequestContext.d.ts +23 -0
- package/dist/auth/RequestContext.js +78 -0
- package/dist/auth/anthropicOAuth.js +1 -1
- package/dist/auth/authContext.d.ts +198 -0
- package/dist/auth/authContext.js +314 -0
- package/dist/auth/errors.d.ts +63 -0
- package/dist/auth/errors.js +39 -0
- package/dist/auth/index.d.ts +20 -8
- package/dist/auth/index.js +35 -7
- package/dist/auth/middleware/AuthMiddleware.d.ts +181 -0
- package/dist/auth/middleware/AuthMiddleware.js +519 -0
- package/dist/auth/middleware/rateLimitByUser.d.ts +282 -0
- package/dist/auth/middleware/rateLimitByUser.js +554 -0
- package/dist/auth/providers/BaseAuthProvider.d.ts +259 -0
- package/dist/auth/providers/BaseAuthProvider.js +723 -0
- package/dist/auth/providers/CognitoProvider.d.ts +61 -0
- package/dist/auth/providers/CognitoProvider.js +304 -0
- package/dist/auth/providers/KeycloakProvider.d.ts +61 -0
- package/dist/auth/providers/KeycloakProvider.js +393 -0
- package/dist/auth/providers/auth0.d.ts +59 -0
- package/dist/auth/providers/auth0.js +274 -0
- package/dist/auth/providers/betterAuth.d.ts +51 -0
- package/dist/auth/providers/betterAuth.js +182 -0
- package/dist/auth/providers/clerk.d.ts +65 -0
- package/dist/auth/providers/clerk.js +317 -0
- package/dist/auth/providers/custom.d.ts +64 -0
- package/dist/auth/providers/custom.js +112 -0
- package/dist/auth/providers/firebase.d.ts +63 -0
- package/dist/auth/providers/firebase.js +226 -0
- package/dist/auth/providers/jwt.d.ts +68 -0
- package/dist/auth/providers/jwt.js +212 -0
- package/dist/auth/providers/oauth2.d.ts +73 -0
- package/dist/auth/providers/oauth2.js +303 -0
- package/dist/auth/providers/supabase.d.ts +63 -0
- package/dist/auth/providers/supabase.js +259 -0
- package/dist/auth/providers/workos.d.ts +61 -0
- package/dist/auth/providers/workos.js +284 -0
- package/dist/auth/serverBridge.d.ts +14 -0
- package/dist/auth/serverBridge.js +25 -0
- package/dist/auth/sessionManager.d.ts +142 -0
- package/dist/auth/sessionManager.js +437 -0
- package/dist/cli/commands/authProviders.d.ts +43 -0
- package/dist/cli/commands/authProviders.js +399 -0
- package/dist/cli/commands/proxy.js +18 -5
- package/dist/cli/factories/authCommandFactory.d.ts +23 -5
- package/dist/cli/factories/authCommandFactory.js +108 -5
- package/dist/cli/parser.js +1 -1
- package/dist/client/aiSdkAdapter.js +1 -1
- package/dist/client/index.js +138 -477
- package/dist/core/factory.js +0 -1
- package/dist/core/infrastructure/baseRegistry.d.ts +3 -1
- package/dist/core/infrastructure/baseRegistry.js +5 -1
- package/dist/core/redisConversationMemoryManager.js +1 -1
- package/dist/features/ppt/slideGenerator.js +0 -1
- package/dist/features/ppt/utils.js +0 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.js +25 -0
- package/dist/lib/auth/AuthProviderFactory.d.ts +71 -0
- package/dist/lib/auth/AuthProviderFactory.js +112 -0
- package/dist/lib/auth/AuthProviderRegistry.d.ts +33 -0
- package/dist/lib/auth/AuthProviderRegistry.js +191 -0
- package/dist/lib/auth/RequestContext.d.ts +23 -0
- package/dist/lib/auth/RequestContext.js +79 -0
- package/dist/lib/auth/authContext.d.ts +198 -0
- package/dist/lib/auth/authContext.js +315 -0
- package/dist/lib/auth/errors.d.ts +63 -0
- package/dist/lib/auth/errors.js +40 -0
- package/dist/lib/auth/index.d.ts +20 -8
- package/dist/lib/auth/index.js +35 -7
- package/dist/lib/auth/middleware/AuthMiddleware.d.ts +181 -0
- package/dist/lib/auth/middleware/AuthMiddleware.js +520 -0
- package/dist/lib/auth/middleware/rateLimitByUser.d.ts +282 -0
- package/dist/lib/auth/middleware/rateLimitByUser.js +555 -0
- package/dist/lib/auth/providers/BaseAuthProvider.d.ts +259 -0
- package/dist/lib/auth/providers/BaseAuthProvider.js +724 -0
- package/dist/lib/auth/providers/CognitoProvider.d.ts +61 -0
- package/dist/lib/auth/providers/CognitoProvider.js +305 -0
- package/dist/lib/auth/providers/KeycloakProvider.d.ts +61 -0
- package/dist/lib/auth/providers/KeycloakProvider.js +394 -0
- package/dist/lib/auth/providers/auth0.d.ts +59 -0
- package/dist/lib/auth/providers/auth0.js +275 -0
- package/dist/lib/auth/providers/betterAuth.d.ts +51 -0
- package/dist/lib/auth/providers/betterAuth.js +183 -0
- package/dist/lib/auth/providers/clerk.d.ts +65 -0
- package/dist/lib/auth/providers/clerk.js +318 -0
- package/dist/lib/auth/providers/custom.d.ts +64 -0
- package/dist/lib/auth/providers/custom.js +113 -0
- package/dist/lib/auth/providers/firebase.d.ts +63 -0
- package/dist/lib/auth/providers/firebase.js +227 -0
- package/dist/lib/auth/providers/jwt.d.ts +68 -0
- package/dist/lib/auth/providers/jwt.js +213 -0
- package/dist/lib/auth/providers/oauth2.d.ts +73 -0
- package/dist/lib/auth/providers/oauth2.js +304 -0
- package/dist/lib/auth/providers/supabase.d.ts +63 -0
- package/dist/lib/auth/providers/supabase.js +260 -0
- package/dist/lib/auth/providers/workos.d.ts +61 -0
- package/dist/lib/auth/providers/workos.js +285 -0
- package/dist/lib/auth/serverBridge.d.ts +14 -0
- package/dist/lib/auth/serverBridge.js +26 -0
- package/dist/lib/auth/sessionManager.d.ts +142 -0
- package/dist/lib/auth/sessionManager.js +438 -0
- package/dist/lib/core/infrastructure/baseRegistry.d.ts +3 -1
- package/dist/lib/core/infrastructure/baseRegistry.js +5 -1
- package/dist/lib/index.d.ts +1 -0
- package/dist/lib/index.js +25 -0
- package/dist/lib/mcp/toolRegistry.js +11 -1
- package/dist/lib/neurolink.d.ts +42 -1
- package/dist/lib/neurolink.js +218 -0
- package/dist/lib/rag/ChunkerRegistry.js +2 -2
- package/dist/lib/rag/metadata/MetadataExtractorRegistry.js +2 -2
- package/dist/lib/rag/reranker/RerankerRegistry.js +2 -2
- package/dist/lib/server/routes/agentRoutes.js +20 -2
- package/dist/lib/server/routes/claudeProxyRoutes.js +45 -9
- package/dist/lib/types/authTypes.d.ts +937 -1
- package/dist/lib/types/authTypes.js +2 -1
- package/dist/lib/types/configTypes.d.ts +46 -0
- package/dist/lib/types/generateTypes.d.ts +6 -0
- package/dist/lib/types/index.d.ts +1 -0
- package/dist/lib/types/streamTypes.d.ts +6 -0
- package/dist/mcp/elicitationProtocol.js +1 -1
- package/dist/mcp/servers/agent/directToolsServer.js +0 -1
- package/dist/mcp/toolRegistry.js +11 -1
- package/dist/neurolink.d.ts +42 -1
- package/dist/neurolink.js +218 -0
- package/dist/providers/azureOpenai.js +1 -1
- package/dist/providers/huggingFace.js +0 -1
- package/dist/providers/openaiCompatible.js +0 -1
- package/dist/rag/ChunkerRegistry.js +2 -2
- package/dist/rag/metadata/MetadataExtractorRegistry.js +2 -2
- package/dist/rag/reranker/RerankerRegistry.js +2 -2
- package/dist/sdk/toolRegistration.js +0 -1
- package/dist/server/openapi/generator.js +1 -1
- package/dist/server/routes/agentRoutes.js +20 -2
- package/dist/server/routes/claudeProxyRoutes.js +45 -9
- package/dist/types/authTypes.d.ts +937 -1
- package/dist/types/authTypes.js +2 -1
- package/dist/types/configTypes.d.ts +46 -0
- package/dist/types/configTypes.js +0 -5
- package/dist/types/generateTypes.d.ts +6 -0
- package/dist/types/index.d.ts +1 -0
- package/dist/types/modelTypes.js +0 -1
- package/dist/types/streamTypes.d.ts +6 -0
- package/dist/types/tools.js +0 -1
- package/dist/types/typeAliases.js +0 -1
- package/dist/types/utilities.js +1 -1
- package/dist/types/workflowTypes.js +0 -1
- package/dist/utils/providerRetry.js +0 -1
- package/dist/utils/providerUtils.js +0 -1
- package/package.json +3 -2
- package/dist/client/adapters/providerImageAdapter.js +0 -588
- package/dist/client/adapters/tts/googleTTSHandler.js +0 -344
- package/dist/client/adapters/video/directorPipeline.js +0 -516
- package/dist/client/adapters/video/ffmpegAdapter.js +0 -206
- package/dist/client/adapters/video/frameExtractor.js +0 -143
- package/dist/client/adapters/video/vertexVideoHandler.js +0 -763
- package/dist/client/adapters/video/videoAnalyzer.js +0 -238
- package/dist/client/adapters/video/videoMerger.js +0 -171
- package/dist/client/agent/directTools.js +0 -840
- package/dist/client/auth/anthropicOAuth.js +0 -974
- package/dist/client/auth/tokenStore.js +0 -799
- package/dist/client/client/aiSdkAdapter.js +0 -487
- package/dist/client/client/auth.js +0 -473
- package/dist/client/client/errors.js +0 -552
- package/dist/client/client/httpClient.js +0 -837
- package/dist/client/client/index.js +0 -172
- package/dist/client/client/interceptors.js +0 -601
- package/dist/client/client/sseClient.js +0 -545
- package/dist/client/client/streamingClient.js +0 -917
- package/dist/client/client/wsClient.js +0 -369
- package/dist/client/config/configManager.js +0 -303
- package/dist/client/config/conversationMemory.js +0 -86
- package/dist/client/config/taskClassificationConfig.js +0 -148
- package/dist/client/constants/contextWindows.js +0 -295
- package/dist/client/constants/enums.js +0 -853
- package/dist/client/constants/index.js +0 -207
- package/dist/client/constants/performance.js +0 -389
- package/dist/client/constants/retry.js +0 -266
- package/dist/client/constants/timeouts.js +0 -182
- package/dist/client/constants/tokens.js +0 -380
- package/dist/client/constants/videoErrors.js +0 -46
- package/dist/client/context/budgetChecker.js +0 -98
- package/dist/client/context/contextCompactor.js +0 -205
- package/dist/client/context/emergencyTruncation.js +0 -88
- package/dist/client/context/errorDetection.js +0 -171
- package/dist/client/context/errors.js +0 -21
- package/dist/client/context/fileTokenBudget.js +0 -127
- package/dist/client/context/prompts/summarizationPrompt.js +0 -117
- package/dist/client/context/stages/fileReadDeduplicator.js +0 -66
- package/dist/client/context/stages/slidingWindowTruncator.js +0 -190
- package/dist/client/context/stages/structuredSummarizer.js +0 -99
- package/dist/client/context/stages/toolOutputPruner.js +0 -52
- package/dist/client/context/summarizationEngine.js +0 -136
- package/dist/client/context/toolOutputLimits.js +0 -78
- package/dist/client/context/toolPairRepair.js +0 -66
- package/dist/client/core/analytics.js +0 -88
- package/dist/client/core/baseProvider.js +0 -1385
- package/dist/client/core/constants.js +0 -140
- package/dist/client/core/conversationMemoryFactory.js +0 -141
- package/dist/client/core/conversationMemoryInitializer.js +0 -128
- package/dist/client/core/conversationMemoryManager.js +0 -344
- package/dist/client/core/dynamicModels.js +0 -358
- package/dist/client/core/evaluation.js +0 -309
- package/dist/client/core/evaluationProviders.js +0 -248
- package/dist/client/core/factory.js +0 -412
- package/dist/client/core/infrastructure/baseError.js +0 -22
- package/dist/client/core/infrastructure/baseFactory.js +0 -54
- package/dist/client/core/infrastructure/baseRegistry.js +0 -49
- package/dist/client/core/infrastructure/index.js +0 -5
- package/dist/client/core/infrastructure/retry.js +0 -20
- package/dist/client/core/infrastructure/typedEventEmitter.js +0 -23
- package/dist/client/core/modelConfiguration.js +0 -851
- package/dist/client/core/modules/GenerationHandler.js +0 -588
- package/dist/client/core/modules/MessageBuilder.js +0 -273
- package/dist/client/core/modules/StreamHandler.js +0 -185
- package/dist/client/core/modules/TelemetryHandler.js +0 -203
- package/dist/client/core/modules/ToolsManager.js +0 -499
- package/dist/client/core/modules/Utilities.js +0 -331
- package/dist/client/core/redisConversationMemoryManager.js +0 -1435
- package/dist/client/core/streamAnalytics.js +0 -131
- package/dist/client/evaluation/contextBuilder.js +0 -134
- package/dist/client/evaluation/index.js +0 -61
- package/dist/client/evaluation/prompts.js +0 -73
- package/dist/client/evaluation/ragasEvaluator.js +0 -110
- package/dist/client/evaluation/retryManager.js +0 -78
- package/dist/client/evaluation/scoring.js +0 -61
- package/dist/client/factories/providerFactory.js +0 -166
- package/dist/client/factories/providerRegistry.js +0 -166
- package/dist/client/features/ppt/constants.js +0 -896
- package/dist/client/features/ppt/contentPlanner.js +0 -529
- package/dist/client/features/ppt/presentationOrchestrator.js +0 -236
- package/dist/client/features/ppt/slideGenerator.js +0 -532
- package/dist/client/features/ppt/slideRenderers.js +0 -2383
- package/dist/client/features/ppt/slideTypeInference.js +0 -405
- package/dist/client/features/ppt/types.js +0 -13
- package/dist/client/features/ppt/utils.js +0 -443
- package/dist/client/files/fileReferenceRegistry.js +0 -1543
- package/dist/client/files/fileTools.js +0 -450
- package/dist/client/files/streamingReader.js +0 -321
- package/dist/client/files/types.js +0 -23
- package/dist/client/hitl/hitlErrors.js +0 -54
- package/dist/client/hitl/hitlManager.js +0 -460
- package/dist/client/mcp/agentExposure.js +0 -356
- package/dist/client/mcp/auth/index.js +0 -11
- package/dist/client/mcp/auth/oauthClientProvider.js +0 -325
- package/dist/client/mcp/auth/tokenStorage.js +0 -134
- package/dist/client/mcp/batching/index.js +0 -10
- package/dist/client/mcp/batching/requestBatcher.js +0 -441
- package/dist/client/mcp/caching/index.js +0 -10
- package/dist/client/mcp/caching/toolCache.js +0 -433
- package/dist/client/mcp/elicitation/elicitationManager.js +0 -376
- package/dist/client/mcp/elicitation/index.js +0 -11
- package/dist/client/mcp/elicitation/types.js +0 -10
- package/dist/client/mcp/elicitationProtocol.js +0 -375
- package/dist/client/mcp/enhancedToolDiscovery.js +0 -481
- package/dist/client/mcp/externalServerManager.js +0 -1478
- package/dist/client/mcp/factory.js +0 -161
- package/dist/client/mcp/flexibleToolValidator.js +0 -161
- package/dist/client/mcp/httpRateLimiter.js +0 -391
- package/dist/client/mcp/httpRetryHandler.js +0 -178
- package/dist/client/mcp/index.js +0 -74
- package/dist/client/mcp/mcpCircuitBreaker.js +0 -427
- package/dist/client/mcp/mcpClientFactory.js +0 -708
- package/dist/client/mcp/mcpRegistryClient.js +0 -488
- package/dist/client/mcp/mcpServerBase.js +0 -373
- package/dist/client/mcp/multiServerManager.js +0 -579
- package/dist/client/mcp/registry.js +0 -158
- package/dist/client/mcp/routing/index.js +0 -10
- package/dist/client/mcp/routing/toolRouter.js +0 -416
- package/dist/client/mcp/serverCapabilities.js +0 -502
- package/dist/client/mcp/servers/agent/directToolsServer.js +0 -150
- package/dist/client/mcp/toolAnnotations.js +0 -239
- package/dist/client/mcp/toolConverter.js +0 -258
- package/dist/client/mcp/toolDiscoveryService.js +0 -798
- package/dist/client/mcp/toolIntegration.js +0 -334
- package/dist/client/mcp/toolRegistry.js +0 -719
- package/dist/client/memory/hippocampusInitializer.js +0 -19
- package/dist/client/memory/memoryRetrievalTools.js +0 -166
- package/dist/client/middleware/builtin/analytics.js +0 -132
- package/dist/client/middleware/builtin/autoEvaluation.js +0 -203
- package/dist/client/middleware/builtin/guardrails.js +0 -109
- package/dist/client/middleware/builtin/lifecycle.js +0 -168
- package/dist/client/middleware/factory.js +0 -327
- package/dist/client/middleware/registry.js +0 -295
- package/dist/client/middleware/utils/guardrailsUtils.js +0 -396
- package/dist/client/models/anthropicModels.js +0 -527
- package/dist/client/neurolink.js +0 -8015
- package/dist/client/observability/exporterRegistry.js +0 -413
- package/dist/client/observability/exporters/arizeExporter.js +0 -138
- package/dist/client/observability/exporters/baseExporter.js +0 -190
- package/dist/client/observability/exporters/braintrustExporter.js +0 -154
- package/dist/client/observability/exporters/datadogExporter.js +0 -196
- package/dist/client/observability/exporters/laminarExporter.js +0 -302
- package/dist/client/observability/exporters/langfuseExporter.js +0 -209
- package/dist/client/observability/exporters/langsmithExporter.js +0 -143
- package/dist/client/observability/exporters/otelExporter.js +0 -164
- package/dist/client/observability/exporters/posthogExporter.js +0 -287
- package/dist/client/observability/exporters/sentryExporter.js +0 -165
- package/dist/client/observability/index.js +0 -31
- package/dist/client/observability/metricsAggregator.js +0 -556
- package/dist/client/observability/otelBridge.js +0 -131
- package/dist/client/observability/retryPolicy.js +0 -383
- package/dist/client/observability/sampling/samplers.js +0 -216
- package/dist/client/observability/spanProcessor.js +0 -303
- package/dist/client/observability/tokenTracker.js +0 -413
- package/dist/client/observability/types/exporterTypes.js +0 -5
- package/dist/client/observability/types/index.js +0 -4
- package/dist/client/observability/types/spanTypes.js +0 -92
- package/dist/client/observability/utils/safeMetadata.js +0 -25
- package/dist/client/observability/utils/spanSerializer.js +0 -292
- package/dist/client/processors/archive/ArchiveProcessor.js +0 -1308
- package/dist/client/processors/base/BaseFileProcessor.js +0 -614
- package/dist/client/processors/base/types.js +0 -82
- package/dist/client/processors/config/fileTypes.js +0 -520
- package/dist/client/processors/config/index.js +0 -92
- package/dist/client/processors/config/languageMap.js +0 -410
- package/dist/client/processors/config/mimeTypes.js +0 -363
- package/dist/client/processors/config/sizeLimits.js +0 -258
- package/dist/client/processors/document/ExcelProcessor.js +0 -590
- package/dist/client/processors/document/OpenDocumentProcessor.js +0 -212
- package/dist/client/processors/document/PptxProcessor.js +0 -157
- package/dist/client/processors/document/RtfProcessor.js +0 -361
- package/dist/client/processors/document/WordProcessor.js +0 -353
- package/dist/client/processors/errors/FileErrorCode.js +0 -255
- package/dist/client/processors/errors/errorHelpers.js +0 -386
- package/dist/client/processors/errors/errorSerializer.js +0 -507
- package/dist/client/processors/errors/index.js +0 -49
- package/dist/client/processors/markup/SvgProcessor.js +0 -240
- package/dist/client/processors/media/AudioProcessor.js +0 -707
- package/dist/client/processors/media/VideoProcessor.js +0 -1045
- package/dist/client/providers/amazonBedrock.js +0 -1512
- package/dist/client/providers/amazonSagemaker.js +0 -162
- package/dist/client/providers/anthropic.js +0 -831
- package/dist/client/providers/azureOpenai.js +0 -143
- package/dist/client/providers/googleAiStudio.js +0 -1200
- package/dist/client/providers/googleNativeGemini3.js +0 -543
- package/dist/client/providers/googleVertex.js +0 -2936
- package/dist/client/providers/huggingFace.js +0 -315
- package/dist/client/providers/litellm.js +0 -488
- package/dist/client/providers/mistral.js +0 -157
- package/dist/client/providers/ollama.js +0 -1579
- package/dist/client/providers/openAI.js +0 -627
- package/dist/client/providers/openRouter.js +0 -543
- package/dist/client/providers/openaiCompatible.js +0 -290
- package/dist/client/providers/providerTypeUtils.js +0 -46
- package/dist/client/providers/sagemaker/adaptive-semaphore.js +0 -215
- package/dist/client/providers/sagemaker/client.js +0 -472
- package/dist/client/providers/sagemaker/config.js +0 -317
- package/dist/client/providers/sagemaker/detection.js +0 -606
- package/dist/client/providers/sagemaker/error-constants.js +0 -227
- package/dist/client/providers/sagemaker/errors.js +0 -299
- package/dist/client/providers/sagemaker/language-model.js +0 -775
- package/dist/client/providers/sagemaker/parsers.js +0 -634
- package/dist/client/providers/sagemaker/streaming.js +0 -331
- package/dist/client/providers/sagemaker/structured-parser.js +0 -625
- package/dist/client/proxy/accountQuota.js +0 -162
- package/dist/client/proxy/claudeFormat.js +0 -595
- package/dist/client/proxy/modelRouter.js +0 -29
- package/dist/client/proxy/oauthFetch.js +0 -367
- package/dist/client/proxy/proxyFetch.js +0 -586
- package/dist/client/proxy/requestLogger.js +0 -207
- package/dist/client/proxy/tokenRefresh.js +0 -124
- package/dist/client/proxy/usageStats.js +0 -74
- package/dist/client/proxy/utils/noProxyUtils.js +0 -149
- package/dist/client/rag/ChunkerFactory.js +0 -320
- package/dist/client/rag/ChunkerRegistry.js +0 -421
- package/dist/client/rag/chunkers/BaseChunker.js +0 -143
- package/dist/client/rag/chunkers/CharacterChunker.js +0 -28
- package/dist/client/rag/chunkers/HTMLChunker.js +0 -38
- package/dist/client/rag/chunkers/JSONChunker.js +0 -68
- package/dist/client/rag/chunkers/LaTeXChunker.js +0 -63
- package/dist/client/rag/chunkers/MarkdownChunker.js +0 -306
- package/dist/client/rag/chunkers/RecursiveChunker.js +0 -139
- package/dist/client/rag/chunkers/SemanticMarkdownChunker.js +0 -138
- package/dist/client/rag/chunkers/SentenceChunker.js +0 -66
- package/dist/client/rag/chunkers/TokenChunker.js +0 -61
- package/dist/client/rag/chunkers/index.js +0 -15
- package/dist/client/rag/chunking/characterChunker.js +0 -142
- package/dist/client/rag/chunking/chunkerRegistry.js +0 -194
- package/dist/client/rag/chunking/htmlChunker.js +0 -247
- package/dist/client/rag/chunking/index.js +0 -17
- package/dist/client/rag/chunking/jsonChunker.js +0 -281
- package/dist/client/rag/chunking/latexChunker.js +0 -251
- package/dist/client/rag/chunking/markdownChunker.js +0 -373
- package/dist/client/rag/chunking/recursiveChunker.js +0 -148
- package/dist/client/rag/chunking/semanticChunker.js +0 -306
- package/dist/client/rag/chunking/sentenceChunker.js +0 -230
- package/dist/client/rag/chunking/tokenChunker.js +0 -183
- package/dist/client/rag/document/MDocument.js +0 -392
- package/dist/client/rag/document/index.js +0 -5
- package/dist/client/rag/document/loaders.js +0 -500
- package/dist/client/rag/errors/RAGError.js +0 -274
- package/dist/client/rag/errors/index.js +0 -6
- package/dist/client/rag/graphRag/graphRAG.js +0 -401
- package/dist/client/rag/graphRag/index.js +0 -4
- package/dist/client/rag/index.js +0 -141
- package/dist/client/rag/metadata/MetadataExtractorFactory.js +0 -418
- package/dist/client/rag/metadata/MetadataExtractorRegistry.js +0 -362
- package/dist/client/rag/metadata/index.js +0 -9
- package/dist/client/rag/metadata/metadataExtractor.js +0 -280
- package/dist/client/rag/pipeline/RAGPipeline.js +0 -436
- package/dist/client/rag/pipeline/contextAssembly.js +0 -341
- package/dist/client/rag/pipeline/index.js +0 -5
- package/dist/client/rag/ragIntegration.js +0 -321
- package/dist/client/rag/reranker/RerankerFactory.js +0 -430
- package/dist/client/rag/reranker/RerankerRegistry.js +0 -402
- package/dist/client/rag/reranker/index.js +0 -9
- package/dist/client/rag/reranker/reranker.js +0 -277
- package/dist/client/rag/resilience/CircuitBreaker.js +0 -431
- package/dist/client/rag/resilience/RetryHandler.js +0 -304
- package/dist/client/rag/resilience/index.js +0 -7
- package/dist/client/rag/retrieval/hybridSearch.js +0 -335
- package/dist/client/rag/retrieval/index.js +0 -5
- package/dist/client/rag/retrieval/vectorQueryTool.js +0 -307
- package/dist/client/rag/types.js +0 -8
- package/dist/client/sdk/toolRegistration.js +0 -377
- package/dist/client/server/abstract/baseServerAdapter.js +0 -575
- package/dist/client/server/adapters/expressAdapter.js +0 -486
- package/dist/client/server/adapters/fastifyAdapter.js +0 -472
- package/dist/client/server/adapters/honoAdapter.js +0 -632
- package/dist/client/server/adapters/koaAdapter.js +0 -510
- package/dist/client/server/errors.js +0 -486
- package/dist/client/server/factory/serverAdapterFactory.js +0 -160
- package/dist/client/server/index.js +0 -108
- package/dist/client/server/middleware/abortSignal.js +0 -111
- package/dist/client/server/middleware/auth.js +0 -388
- package/dist/client/server/middleware/cache.js +0 -359
- package/dist/client/server/middleware/common.js +0 -281
- package/dist/client/server/middleware/deprecation.js +0 -190
- package/dist/client/server/middleware/mcpBodyAttachment.js +0 -63
- package/dist/client/server/middleware/rateLimit.js +0 -227
- package/dist/client/server/middleware/validation.js +0 -388
- package/dist/client/server/openapi/generator.js +0 -398
- package/dist/client/server/openapi/index.js +0 -36
- package/dist/client/server/openapi/schemas.js +0 -695
- package/dist/client/server/openapi/templates.js +0 -374
- package/dist/client/server/routes/agentRoutes.js +0 -171
- package/dist/client/server/routes/claudeProxyRoutes.js +0 -1600
- package/dist/client/server/routes/healthRoutes.js +0 -187
- package/dist/client/server/routes/index.js +0 -57
- package/dist/client/server/routes/mcpRoutes.js +0 -342
- package/dist/client/server/routes/memoryRoutes.js +0 -350
- package/dist/client/server/routes/openApiRoutes.js +0 -126
- package/dist/client/server/routes/toolRoutes.js +0 -199
- package/dist/client/server/streaming/dataStream.js +0 -486
- package/dist/client/server/streaming/index.js +0 -11
- package/dist/client/server/types.js +0 -67
- package/dist/client/server/utils/redaction.js +0 -334
- package/dist/client/server/utils/validation.js +0 -243
- package/dist/client/server/websocket/WebSocketHandler.js +0 -383
- package/dist/client/server/websocket/index.js +0 -4
- package/dist/client/services/server/ai/observability/instrumentation.js +0 -808
- package/dist/client/telemetry/attributes.js +0 -100
- package/dist/client/telemetry/index.js +0 -26
- package/dist/client/telemetry/telemetryService.js +0 -308
- package/dist/client/telemetry/tracers.js +0 -17
- package/dist/client/telemetry/withSpan.js +0 -34
- package/dist/client/types/actionTypes.js +0 -6
- package/dist/client/types/analytics.js +0 -5
- package/dist/client/types/authTypes.js +0 -8
- package/dist/client/types/circuitBreakerErrors.js +0 -34
- package/dist/client/types/cli.js +0 -21
- package/dist/client/types/clientTypes.js +0 -10
- package/dist/client/types/common.js +0 -51
- package/dist/client/types/configTypes.js +0 -49
- package/dist/client/types/content.js +0 -19
- package/dist/client/types/contextTypes.js +0 -400
- package/dist/client/types/conversation.js +0 -47
- package/dist/client/types/conversationMemoryInterface.js +0 -6
- package/dist/client/types/domainTypes.js +0 -5
- package/dist/client/types/errors.js +0 -167
- package/dist/client/types/evaluation.js +0 -5
- package/dist/client/types/evaluationProviders.js +0 -5
- package/dist/client/types/evaluationTypes.js +0 -1
- package/dist/client/types/externalMcp.js +0 -6
- package/dist/client/types/fileReferenceTypes.js +0 -8
- package/dist/client/types/fileTypes.js +0 -4
- package/dist/client/types/generateTypes.js +0 -1
- package/dist/client/types/guardrails.js +0 -1
- package/dist/client/types/hitlTypes.js +0 -8
- package/dist/client/types/index.js +0 -57
- package/dist/client/types/mcpTypes.js +0 -5
- package/dist/client/types/middlewareTypes.js +0 -1
- package/dist/client/types/modelTypes.js +0 -30
- package/dist/client/types/multimodal.js +0 -135
- package/dist/client/types/observability.js +0 -6
- package/dist/client/types/pptTypes.js +0 -82
- package/dist/client/types/providers.js +0 -111
- package/dist/client/types/proxyTypes.js +0 -16
- package/dist/client/types/ragTypes.js +0 -7
- package/dist/client/types/sdkTypes.js +0 -8
- package/dist/client/types/serviceTypes.js +0 -5
- package/dist/client/types/streamTypes.js +0 -1
- package/dist/client/types/subscriptionTypes.js +0 -9
- package/dist/client/types/taskClassificationTypes.js +0 -5
- package/dist/client/types/tools.js +0 -24
- package/dist/client/types/ttsTypes.js +0 -57
- package/dist/client/types/typeAliases.js +0 -48
- package/dist/client/types/utilities.js +0 -4
- package/dist/client/types/workflowTypes.js +0 -30
- package/dist/client/utils/async/withTimeout.js +0 -98
- package/dist/client/utils/asyncMutex.js +0 -60
- package/dist/client/utils/conversationMemory.js +0 -431
- package/dist/client/utils/csvProcessor.js +0 -846
- package/dist/client/utils/errorHandling.js +0 -936
- package/dist/client/utils/evaluationUtils.js +0 -131
- package/dist/client/utils/factoryProcessing.js +0 -589
- package/dist/client/utils/fileDetector.js +0 -2161
- package/dist/client/utils/imageCache.js +0 -376
- package/dist/client/utils/imageProcessor.js +0 -704
- package/dist/client/utils/logger.js +0 -491
- package/dist/client/utils/mcpDefaults.js +0 -134
- package/dist/client/utils/messageBuilder.js +0 -1653
- package/dist/client/utils/modelAliasResolver.js +0 -54
- package/dist/client/utils/modelDetection.js +0 -80
- package/dist/client/utils/modelRouter.js +0 -292
- package/dist/client/utils/multimodalOptionsBuilder.js +0 -65
- package/dist/client/utils/observabilityHelpers.js +0 -47
- package/dist/client/utils/parameterValidation.js +0 -966
- package/dist/client/utils/pdfProcessor.js +0 -410
- package/dist/client/utils/performance.js +0 -222
- package/dist/client/utils/pricing.js +0 -340
- package/dist/client/utils/promptRedaction.js +0 -62
- package/dist/client/utils/providerConfig.js +0 -1009
- package/dist/client/utils/providerHealth.js +0 -1237
- package/dist/client/utils/providerRetry.js +0 -112
- package/dist/client/utils/providerUtils.js +0 -434
- package/dist/client/utils/rateLimiter.js +0 -200
- package/dist/client/utils/redis.js +0 -368
- package/dist/client/utils/retryHandler.js +0 -269
- package/dist/client/utils/retryability.js +0 -22
- package/dist/client/utils/sanitizers/svg.js +0 -481
- package/dist/client/utils/schemaConversion.js +0 -255
- package/dist/client/utils/taskClassificationUtils.js +0 -149
- package/dist/client/utils/taskClassifier.js +0 -94
- package/dist/client/utils/thinkingConfig.js +0 -104
- package/dist/client/utils/timeout.js +0 -359
- package/dist/client/utils/tokenEstimation.js +0 -142
- package/dist/client/utils/tokenLimits.js +0 -125
- package/dist/client/utils/tokenUtils.js +0 -239
- package/dist/client/utils/toolUtils.js +0 -75
- package/dist/client/utils/transformationUtils.js +0 -554
- package/dist/client/utils/ttsProcessor.js +0 -286
- package/dist/client/utils/typeUtils.js +0 -97
- package/dist/client/utils/videoAnalysisProcessor.js +0 -67
- package/dist/client/workflow/config.js +0 -398
- package/dist/client/workflow/core/ensembleExecutor.js +0 -407
- package/dist/client/workflow/core/judgeScorer.js +0 -544
- package/dist/client/workflow/core/responseConditioner.js +0 -225
- package/dist/client/workflow/core/types/conditionerTypes.js +0 -7
- package/dist/client/workflow/core/types/ensembleTypes.js +0 -7
- package/dist/client/workflow/core/types/index.js +0 -7
- package/dist/client/workflow/core/types/judgeTypes.js +0 -7
- package/dist/client/workflow/core/types/layerTypes.js +0 -7
- package/dist/client/workflow/core/types/registryTypes.js +0 -7
- package/dist/client/workflow/core/workflowRegistry.js +0 -304
- package/dist/client/workflow/core/workflowRunner.js +0 -586
- package/dist/client/workflow/index.js +0 -50
- package/dist/client/workflow/types.js +0 -9
- package/dist/client/workflow/utils/types/index.js +0 -7
- package/dist/client/workflow/utils/workflowMetrics.js +0 -311
- package/dist/client/workflow/utils/workflowValidation.js +0 -420
- package/dist/client/workflow/workflows/adaptiveWorkflow.js +0 -366
- package/dist/client/workflow/workflows/consensusWorkflow.js +0 -192
- package/dist/client/workflow/workflows/fallbackWorkflow.js +0 -225
- package/dist/client/workflow/workflows/multiJudgeWorkflow.js +0 -351
- /package/dist/client/{client/reactHooks.js → reactHooks.js} +0 -0
|
@@ -1,1385 +0,0 @@
|
|
|
1
|
-
import { context, SpanKind, SpanStatusCode, trace } from "@opentelemetry/api";
|
|
2
|
-
import { generateText } from "ai";
|
|
3
|
-
import { directAgentTools } from "../agent/directTools.js";
|
|
4
|
-
import { IMAGE_GENERATION_MODELS } from "../core/constants.js";
|
|
5
|
-
import { MiddlewareFactory } from "../middleware/factory.js";
|
|
6
|
-
import { SpanStatus, SpanType } from "../observability/types/spanTypes.js";
|
|
7
|
-
import { SpanSerializer } from "../observability/utils/spanSerializer.js";
|
|
8
|
-
import { ATTR, tracers } from "../telemetry/index.js";
|
|
9
|
-
import { calculateCost } from "../utils/pricing.js";
|
|
10
|
-
import { isAbortError } from "../utils/errorHandling.js";
|
|
11
|
-
import { logger } from "../utils/logger.js";
|
|
12
|
-
import { composeAbortSignals, createTimeoutController, TimeoutError, } from "../utils/timeout.js";
|
|
13
|
-
import { shouldDisableBuiltinTools } from "../utils/toolUtils.js";
|
|
14
|
-
import { getKeyCount, getKeysAsString } from "../utils/transformationUtils.js";
|
|
15
|
-
import { TTSProcessor } from "../utils/ttsProcessor.js";
|
|
16
|
-
import { executeVideoAnalysis, hasVideoFrames, } from "../utils/videoAnalysisProcessor.js";
|
|
17
|
-
import { GenerationHandler } from "./modules/GenerationHandler.js";
|
|
18
|
-
// Import modules for composition
|
|
19
|
-
import { MessageBuilder } from "./modules/MessageBuilder.js";
|
|
20
|
-
import { StreamHandler } from "./modules/StreamHandler.js";
|
|
21
|
-
import { TelemetryHandler } from "./modules/TelemetryHandler.js";
|
|
22
|
-
import { ToolsManager } from "./modules/ToolsManager.js";
|
|
23
|
-
import { Utilities } from "./modules/Utilities.js";
|
|
24
|
-
/**
|
|
25
|
-
* Abstract base class for all AI providers
|
|
26
|
-
* Tools are integrated as first-class citizens - always available by default
|
|
27
|
-
*/
|
|
28
|
-
export class BaseProvider {
|
|
29
|
-
modelName;
|
|
30
|
-
providerName;
|
|
31
|
-
defaultTimeout = 30000; // 30 seconds
|
|
32
|
-
middlewareOptions; // TODO: Implement global level middlewares that can be used
|
|
33
|
-
// Tools are conditionally included based on centralized configuration
|
|
34
|
-
directTools = shouldDisableBuiltinTools()
|
|
35
|
-
? {}
|
|
36
|
-
: directAgentTools;
|
|
37
|
-
mcpTools; // MCP tools loaded dynamically when available
|
|
38
|
-
customTools; // Custom tools from registerTool()
|
|
39
|
-
toolExecutor; // Tool executor from setupToolExecutor
|
|
40
|
-
sessionId;
|
|
41
|
-
userId;
|
|
42
|
-
neurolink; // Reference to actual NeuroLink instance for MCP tools
|
|
43
|
-
/** @internal Trace context propagated from NeuroLink SDK for span hierarchy */
|
|
44
|
-
_traceContext = null;
|
|
45
|
-
setTraceContext(ctx) {
|
|
46
|
-
this._traceContext = ctx;
|
|
47
|
-
}
|
|
48
|
-
// Composition modules - Single Responsibility Principle
|
|
49
|
-
messageBuilder;
|
|
50
|
-
streamHandler;
|
|
51
|
-
generationHandler;
|
|
52
|
-
telemetryHandler;
|
|
53
|
-
utilities;
|
|
54
|
-
toolsManager;
|
|
55
|
-
constructor(modelName, providerName, neurolink, middleware) {
|
|
56
|
-
this.modelName = modelName || this.getDefaultModel();
|
|
57
|
-
this.providerName = providerName || this.getProviderName();
|
|
58
|
-
this.neurolink = neurolink;
|
|
59
|
-
this.middlewareOptions = middleware;
|
|
60
|
-
// Initialize composition modules
|
|
61
|
-
this.messageBuilder = new MessageBuilder(this.providerName, this.modelName);
|
|
62
|
-
this.streamHandler = new StreamHandler(this.providerName, this.modelName);
|
|
63
|
-
this.telemetryHandler = new TelemetryHandler(this.providerName, this.modelName, this.neurolink);
|
|
64
|
-
this.generationHandler = new GenerationHandler(this.providerName, this.modelName, () => this.supportsTools(), (options, type) => this.telemetryHandler.getTelemetryConfig(options, type), (toolCalls, toolResults, options, timestamp) => this.handleToolExecutionStorage(toolCalls, toolResults, options, timestamp));
|
|
65
|
-
this.utilities = new Utilities(this.providerName, this.modelName, this.defaultTimeout, this.middlewareOptions);
|
|
66
|
-
this.toolsManager = new ToolsManager(this.providerName, this.directTools, this.neurolink, {
|
|
67
|
-
isZodSchema: (schema) => this.isZodSchema(schema),
|
|
68
|
-
convertToolResult: (result) => this.convertToolResult(result),
|
|
69
|
-
createPermissiveZodSchema: () => this.createPermissiveZodSchema(),
|
|
70
|
-
fixSchemaForOpenAIStrictMode: (schema) => this.fixSchemaForOpenAIStrictMode(schema),
|
|
71
|
-
});
|
|
72
|
-
}
|
|
73
|
-
/**
|
|
74
|
-
* Check if this provider supports tool/function calling
|
|
75
|
-
* Override in subclasses to disable tools for specific providers or models
|
|
76
|
-
* @returns true by default, providers can override to return false
|
|
77
|
-
*/
|
|
78
|
-
supportsTools() {
|
|
79
|
-
return true;
|
|
80
|
-
}
|
|
81
|
-
// ===================
|
|
82
|
-
// PUBLIC API METHODS
|
|
83
|
-
// ===================
|
|
84
|
-
/**
|
|
85
|
-
* Primary streaming method - implements AIProvider interface
|
|
86
|
-
* When tools are involved, falls back to generate() with synthetic streaming
|
|
87
|
-
*/
|
|
88
|
-
async stream(optionsOrPrompt, analysisSchema) {
|
|
89
|
-
let options = this.normalizeStreamOptions(optionsOrPrompt);
|
|
90
|
-
// Observability: create metrics span for provider.stream
|
|
91
|
-
const metricsSpan = SpanSerializer.createSpan(SpanType.MODEL_GENERATION, "provider.stream", {
|
|
92
|
-
"ai.provider": this.providerName || "unknown",
|
|
93
|
-
"ai.model": this.modelName || options.model || "unknown",
|
|
94
|
-
"ai.temperature": options.temperature,
|
|
95
|
-
"ai.max_tokens": options.maxTokens,
|
|
96
|
-
}, this._traceContext?.parentSpanId, this._traceContext?.traceId);
|
|
97
|
-
let metricsSpanRecorded = false;
|
|
98
|
-
// OTEL span for provider-level stream tracing
|
|
99
|
-
const otelStreamSpan = tracers.provider.startSpan("neurolink.provider.stream", {
|
|
100
|
-
kind: SpanKind.CLIENT,
|
|
101
|
-
attributes: {
|
|
102
|
-
[ATTR.GEN_AI_SYSTEM]: this.providerName || "unknown",
|
|
103
|
-
[ATTR.GEN_AI_MODEL]: this.modelName || options.model || "unknown",
|
|
104
|
-
[ATTR.GEN_AI_OPERATION]: "stream",
|
|
105
|
-
[ATTR.NL_PROVIDER]: this.providerName || "unknown",
|
|
106
|
-
},
|
|
107
|
-
});
|
|
108
|
-
try {
|
|
109
|
-
logger.info(`Starting stream`, {
|
|
110
|
-
provider: this.providerName,
|
|
111
|
-
hasTools: !options.disableTools && this.supportsTools(),
|
|
112
|
-
disableTools: !!options.disableTools,
|
|
113
|
-
supportsTools: this.supportsTools(),
|
|
114
|
-
inputLength: options.input?.text?.length || 0,
|
|
115
|
-
maxTokens: options.maxTokens,
|
|
116
|
-
temperature: options.temperature,
|
|
117
|
-
timestamp: Date.now(),
|
|
118
|
-
});
|
|
119
|
-
// ===== EARLY MULTIMODAL DETECTION =====
|
|
120
|
-
const hasFileInput = !!options.input?.files?.length || !!options.input?.videoFiles?.length;
|
|
121
|
-
if (hasFileInput) {
|
|
122
|
-
// ===== VIDEO ANALYSIS DETECTION =====
|
|
123
|
-
// Check if video frames are present and handle with fake streaming
|
|
124
|
-
const messages = await this.buildMessagesForStream(options);
|
|
125
|
-
if (hasVideoFrames(messages)) {
|
|
126
|
-
logger.info(`Video frames detected in stream, using fake streaming for video analysis`, {
|
|
127
|
-
provider: this.providerName,
|
|
128
|
-
model: this.modelName,
|
|
129
|
-
});
|
|
130
|
-
return await this.executeFakeStreaming(options, analysisSchema);
|
|
131
|
-
}
|
|
132
|
-
}
|
|
133
|
-
// CRITICAL: Image generation models don't support real streaming
|
|
134
|
-
// Force fake streaming for image models to ensure image output is yielded
|
|
135
|
-
const isImageModel = IMAGE_GENERATION_MODELS.some((m) => this.modelName.includes(m));
|
|
136
|
-
if (isImageModel) {
|
|
137
|
-
logger.info(`Image model detected, forcing fake streaming`, {
|
|
138
|
-
provider: this.providerName,
|
|
139
|
-
model: this.modelName,
|
|
140
|
-
reason: "Image generation requires fake streaming to yield image output",
|
|
141
|
-
});
|
|
142
|
-
// Skip real streaming, go directly to fake streaming
|
|
143
|
-
return await this.executeFakeStreaming(options, analysisSchema);
|
|
144
|
-
}
|
|
145
|
-
// Central tool merge: Pre-merge base tools (MCP/built-in) with user-provided
|
|
146
|
-
// tools (e.g. RAG tools) into options.tools. This way, every provider's
|
|
147
|
-
// executeStream() can simply use options.tools (or getAllTools() + options.tools)
|
|
148
|
-
// and get the complete tool set without needing per-provider merge logic.
|
|
149
|
-
if (!options.disableTools && this.supportsTools()) {
|
|
150
|
-
const mergedTools = await this.getToolsForStream(options);
|
|
151
|
-
options = { ...options, tools: mergedTools };
|
|
152
|
-
}
|
|
153
|
-
else {
|
|
154
|
-
options = { ...options, tools: {} };
|
|
155
|
-
}
|
|
156
|
-
// CRITICAL FIX: Always prefer real streaming over fake streaming
|
|
157
|
-
// Try real streaming first, use fake streaming only as fallback
|
|
158
|
-
try {
|
|
159
|
-
logger.debug(`Attempting real streaming`, {
|
|
160
|
-
provider: this.providerName,
|
|
161
|
-
timestamp: Date.now(),
|
|
162
|
-
});
|
|
163
|
-
const realStreamResult = await this.executeStream(options, analysisSchema);
|
|
164
|
-
logger.info(`Real streaming succeeded`, {
|
|
165
|
-
provider: this.providerName,
|
|
166
|
-
timestamp: Date.now(),
|
|
167
|
-
});
|
|
168
|
-
// If real streaming succeeds, return it (with tools support via Vercel AI SDK)
|
|
169
|
-
return realStreamResult;
|
|
170
|
-
}
|
|
171
|
-
catch (realStreamError) {
|
|
172
|
-
logger.warn(`Real streaming failed for ${this.providerName}, falling back to fake streaming:`, {
|
|
173
|
-
error: realStreamError instanceof Error
|
|
174
|
-
? realStreamError.message
|
|
175
|
-
: String(realStreamError),
|
|
176
|
-
timestamp: Date.now(),
|
|
177
|
-
});
|
|
178
|
-
// Fallback to fake streaming only if real streaming fails AND tools are enabled
|
|
179
|
-
if (!options.disableTools && this.supportsTools()) {
|
|
180
|
-
return await this.executeFakeStreaming(options, analysisSchema);
|
|
181
|
-
}
|
|
182
|
-
else {
|
|
183
|
-
// If real streaming failed and no tools are enabled, re-throw the original error
|
|
184
|
-
logger.error(`Real streaming failed for ${this.providerName}:`, realStreamError);
|
|
185
|
-
throw this.handleProviderError(realStreamError);
|
|
186
|
-
}
|
|
187
|
-
}
|
|
188
|
-
}
|
|
189
|
-
catch (error) {
|
|
190
|
-
// Observability: record failed stream span
|
|
191
|
-
metricsSpanRecorded = true;
|
|
192
|
-
const _endedStreamSpan = SpanSerializer.endSpan(metricsSpan, SpanStatus.ERROR, error instanceof Error ? error.message : String(error));
|
|
193
|
-
// Note: Do NOT record to getMetricsAggregator() here — neurolink.ts
|
|
194
|
-
// stream:complete listener handles authoritative metrics to avoid double-counting.
|
|
195
|
-
otelStreamSpan.setStatus({
|
|
196
|
-
code: SpanStatusCode.ERROR,
|
|
197
|
-
message: error instanceof Error ? error.message : String(error),
|
|
198
|
-
});
|
|
199
|
-
otelStreamSpan.end();
|
|
200
|
-
throw error;
|
|
201
|
-
}
|
|
202
|
-
finally {
|
|
203
|
-
// Observability: record successful stream span (only if not already ended via error path)
|
|
204
|
-
if (!metricsSpanRecorded) {
|
|
205
|
-
const _endedStreamSpan = SpanSerializer.endSpan(metricsSpan, SpanStatus.OK);
|
|
206
|
-
// Note: Do NOT record to getMetricsAggregator() here — neurolink.ts
|
|
207
|
-
// stream:complete listener handles authoritative metrics to avoid double-counting.
|
|
208
|
-
}
|
|
209
|
-
// End OTEL span on success (only if not already ended via error path)
|
|
210
|
-
if (otelStreamSpan.isRecording()) {
|
|
211
|
-
otelStreamSpan.setStatus({ code: SpanStatusCode.OK });
|
|
212
|
-
otelStreamSpan.end();
|
|
213
|
-
}
|
|
214
|
-
}
|
|
215
|
-
}
|
|
216
|
-
/**
|
|
217
|
-
* Execute fake streaming - extracted method for reusability
|
|
218
|
-
*/
|
|
219
|
-
async executeFakeStreaming(options, analysisSchema) {
|
|
220
|
-
try {
|
|
221
|
-
logger.info(`Starting fake streaming with tools`, {
|
|
222
|
-
provider: this.providerName,
|
|
223
|
-
supportsTools: this.supportsTools(),
|
|
224
|
-
timestamp: Date.now(),
|
|
225
|
-
});
|
|
226
|
-
// Convert stream options to text generation options
|
|
227
|
-
const textOptions = {
|
|
228
|
-
prompt: options.input?.text || "",
|
|
229
|
-
input: options.input,
|
|
230
|
-
systemPrompt: options.systemPrompt,
|
|
231
|
-
temperature: options.temperature,
|
|
232
|
-
maxTokens: options.maxTokens,
|
|
233
|
-
tools: options.tools, // 🔧 FIX: Pass user-provided tools (including RAG tools) to generation pipeline
|
|
234
|
-
disableTools: !!options.disableTools,
|
|
235
|
-
maxSteps: options.maxSteps || 5,
|
|
236
|
-
provider: options.provider,
|
|
237
|
-
model: options.model,
|
|
238
|
-
region: options.region, // Pass region for Vertex AI
|
|
239
|
-
// 🔧 FIX: Include analytics and evaluation options from stream options
|
|
240
|
-
enableAnalytics: options.enableAnalytics,
|
|
241
|
-
enableEvaluation: options.enableEvaluation,
|
|
242
|
-
evaluationDomain: options.evaluationDomain,
|
|
243
|
-
toolUsageContext: options.toolUsageContext,
|
|
244
|
-
context: options.context,
|
|
245
|
-
csvOptions: options.csvOptions,
|
|
246
|
-
// Forward abort, tool filtering, and timeout options to prevent
|
|
247
|
-
// silent bypass when falling back from real streaming to fake streaming
|
|
248
|
-
abortSignal: options.abortSignal,
|
|
249
|
-
toolFilter: options.toolFilter,
|
|
250
|
-
excludeTools: options.excludeTools,
|
|
251
|
-
skipToolPromptInjection: options.skipToolPromptInjection,
|
|
252
|
-
timeout: options.timeout,
|
|
253
|
-
};
|
|
254
|
-
logger.debug(`Calling generate for fake streaming`, {
|
|
255
|
-
provider: this.providerName,
|
|
256
|
-
maxSteps: textOptions.maxSteps,
|
|
257
|
-
disableTools: textOptions.disableTools,
|
|
258
|
-
timestamp: Date.now(),
|
|
259
|
-
});
|
|
260
|
-
const result = await this.generate(textOptions, analysisSchema);
|
|
261
|
-
logger.info(`Generate completed for fake streaming`, {
|
|
262
|
-
provider: this.providerName,
|
|
263
|
-
hasContent: !!result?.content,
|
|
264
|
-
contentLength: result?.content?.length || 0,
|
|
265
|
-
toolsUsed: result?.toolsUsed?.length || 0,
|
|
266
|
-
hasImageOutput: !!result?.imageOutput,
|
|
267
|
-
timestamp: Date.now(),
|
|
268
|
-
});
|
|
269
|
-
// Create a synthetic stream from the generate result that simulates progressive delivery
|
|
270
|
-
return {
|
|
271
|
-
stream: (async function* () {
|
|
272
|
-
if (result?.content) {
|
|
273
|
-
// Split content into words for more natural streaming
|
|
274
|
-
const words = result.content.split(/(\s+)/); // Keep whitespace
|
|
275
|
-
let buffer = "";
|
|
276
|
-
for (let i = 0; i < words.length; i++) {
|
|
277
|
-
buffer += words[i];
|
|
278
|
-
// Yield chunks of roughly 5-10 words or at punctuation
|
|
279
|
-
const shouldYield = i === words.length - 1 || // Last word
|
|
280
|
-
buffer.length > 50 || // Buffer getting long
|
|
281
|
-
/[.!?;,]\s*$/.test(buffer); // End of sentence/clause
|
|
282
|
-
if (shouldYield && buffer.trim()) {
|
|
283
|
-
yield { content: buffer };
|
|
284
|
-
buffer = "";
|
|
285
|
-
// Small delay to simulate streaming (1-10ms)
|
|
286
|
-
await new Promise((resolve) => {
|
|
287
|
-
setTimeout(resolve, Math.random() * 9 + 1);
|
|
288
|
-
});
|
|
289
|
-
}
|
|
290
|
-
}
|
|
291
|
-
// Yield all remaining content
|
|
292
|
-
if (buffer.trim()) {
|
|
293
|
-
yield { content: buffer };
|
|
294
|
-
}
|
|
295
|
-
}
|
|
296
|
-
// 🔧 CRITICAL FIX: Yield image output if present
|
|
297
|
-
if (result?.imageOutput) {
|
|
298
|
-
yield {
|
|
299
|
-
type: "image",
|
|
300
|
-
imageOutput: result.imageOutput,
|
|
301
|
-
};
|
|
302
|
-
}
|
|
303
|
-
})(),
|
|
304
|
-
usage: result?.usage,
|
|
305
|
-
provider: result?.provider,
|
|
306
|
-
model: result?.model,
|
|
307
|
-
toolCalls: result?.toolCalls?.map((call) => ({
|
|
308
|
-
toolName: call.toolName,
|
|
309
|
-
parameters: call.args,
|
|
310
|
-
id: call.toolCallId,
|
|
311
|
-
})),
|
|
312
|
-
toolResults: result?.toolResults
|
|
313
|
-
? result.toolResults.map((tr) => ({
|
|
314
|
-
toolName: tr.toolName || "unknown",
|
|
315
|
-
status: (tr.status === "error"
|
|
316
|
-
? "failure"
|
|
317
|
-
: "success"),
|
|
318
|
-
result: tr.result,
|
|
319
|
-
error: tr.error,
|
|
320
|
-
}))
|
|
321
|
-
: undefined,
|
|
322
|
-
// 🔧 FIX: Include analytics and evaluation from generate result
|
|
323
|
-
analytics: result?.analytics,
|
|
324
|
-
evaluation: result?.evaluation,
|
|
325
|
-
};
|
|
326
|
-
}
|
|
327
|
-
catch (error) {
|
|
328
|
-
logger.error(`Fake streaming fallback failed for ${this.providerName}:`, error);
|
|
329
|
-
throw this.handleProviderError(error);
|
|
330
|
-
}
|
|
331
|
-
}
|
|
332
|
-
/**
|
|
333
|
-
* Apply per-call tool filtering (whitelist/blacklist) to a tools record.
|
|
334
|
-
* If toolFilter is set, only tools whose names are in the list are kept.
|
|
335
|
-
* If excludeTools is set, matching tools are removed. excludeTools is applied after toolFilter.
|
|
336
|
-
*/
|
|
337
|
-
applyToolFiltering(tools, options) {
|
|
338
|
-
if ((!options.toolFilter || options.toolFilter.length === 0) &&
|
|
339
|
-
(!options.excludeTools || options.excludeTools.length === 0)) {
|
|
340
|
-
return tools;
|
|
341
|
-
}
|
|
342
|
-
const beforeCount = Object.keys(tools).length;
|
|
343
|
-
let filtered = { ...tools };
|
|
344
|
-
if (options.toolFilter && options.toolFilter.length > 0) {
|
|
345
|
-
const allowSet = new Set(options.toolFilter);
|
|
346
|
-
const result = {};
|
|
347
|
-
for (const [name, tool] of Object.entries(filtered)) {
|
|
348
|
-
if (allowSet.has(name)) {
|
|
349
|
-
result[name] = tool;
|
|
350
|
-
}
|
|
351
|
-
}
|
|
352
|
-
filtered = result;
|
|
353
|
-
}
|
|
354
|
-
if (options.excludeTools && options.excludeTools.length > 0) {
|
|
355
|
-
const denySet = new Set(options.excludeTools);
|
|
356
|
-
for (const name of Object.keys(filtered)) {
|
|
357
|
-
if (denySet.has(name)) {
|
|
358
|
-
delete filtered[name];
|
|
359
|
-
}
|
|
360
|
-
}
|
|
361
|
-
}
|
|
362
|
-
const afterCount = Object.keys(filtered).length;
|
|
363
|
-
if (beforeCount !== afterCount) {
|
|
364
|
-
logger.debug(`Tool filtering applied`, {
|
|
365
|
-
provider: this.providerName,
|
|
366
|
-
beforeCount,
|
|
367
|
-
afterCount,
|
|
368
|
-
toolFilter: options.toolFilter,
|
|
369
|
-
excludeTools: options.excludeTools,
|
|
370
|
-
});
|
|
371
|
-
}
|
|
372
|
-
return filtered;
|
|
373
|
-
}
|
|
374
|
-
/**
|
|
375
|
-
* Prepare generation context including tools and model
|
|
376
|
-
*/
|
|
377
|
-
async prepareGenerationContext(options) {
|
|
378
|
-
const shouldUseTools = !options.disableTools && this.supportsTools();
|
|
379
|
-
const baseTools = shouldUseTools ? await this.getAllTools() : {};
|
|
380
|
-
let tools = shouldUseTools
|
|
381
|
-
? {
|
|
382
|
-
...baseTools,
|
|
383
|
-
...(options.tools || {}),
|
|
384
|
-
}
|
|
385
|
-
: {};
|
|
386
|
-
// Apply per-call tool filtering (whitelist/blacklist)
|
|
387
|
-
tools = this.applyToolFiltering(tools, options);
|
|
388
|
-
logger.debug(`Final tools prepared for AI`, {
|
|
389
|
-
provider: this.providerName,
|
|
390
|
-
directTools: getKeyCount(baseTools),
|
|
391
|
-
directToolNames: getKeysAsString(baseTools),
|
|
392
|
-
externalTools: getKeyCount(options.tools || {}),
|
|
393
|
-
externalToolNames: getKeysAsString(options.tools || {}),
|
|
394
|
-
totalTools: getKeyCount(tools),
|
|
395
|
-
totalToolNames: getKeysAsString(tools),
|
|
396
|
-
shouldUseTools,
|
|
397
|
-
timestamp: Date.now(),
|
|
398
|
-
});
|
|
399
|
-
const model = await this.getAISDKModelWithMiddleware(options);
|
|
400
|
-
return { tools, model };
|
|
401
|
-
}
|
|
402
|
-
/**
|
|
403
|
-
* Get merged tools for streaming: combines base tools (MCP/built-in) with
|
|
404
|
-
* user-provided tools (e.g., RAG tools passed via options.tools).
|
|
405
|
-
*
|
|
406
|
-
* This is the canonical tool-merge pattern for executeStream() implementations.
|
|
407
|
-
* All providers should call this instead of getAllTools() directly.
|
|
408
|
-
*/
|
|
409
|
-
async getToolsForStream(options) {
|
|
410
|
-
const shouldUseTools = !options.disableTools && this.supportsTools();
|
|
411
|
-
if (!shouldUseTools) {
|
|
412
|
-
return {};
|
|
413
|
-
}
|
|
414
|
-
const baseTools = await this.getAllTools();
|
|
415
|
-
const externalTools = (options.tools || {});
|
|
416
|
-
let merged = { ...baseTools, ...externalTools };
|
|
417
|
-
// Apply per-call tool filtering (whitelist/blacklist)
|
|
418
|
-
merged = this.applyToolFiltering(merged, options);
|
|
419
|
-
logger.debug(`Tools prepared for streaming`, {
|
|
420
|
-
provider: this.providerName,
|
|
421
|
-
baseToolCount: Object.keys(baseTools).length,
|
|
422
|
-
externalToolCount: Object.keys(externalTools).length,
|
|
423
|
-
totalToolCount: Object.keys(merged).length,
|
|
424
|
-
});
|
|
425
|
-
return merged;
|
|
426
|
-
}
|
|
427
|
-
/**
|
|
428
|
-
* Build messages array for generation - delegated to MessageBuilder
|
|
429
|
-
*/
|
|
430
|
-
async buildMessages(options) {
|
|
431
|
-
return this.messageBuilder.buildMessages(options);
|
|
432
|
-
}
|
|
433
|
-
/**
|
|
434
|
-
* Build messages array for streaming operations - delegated to MessageBuilder
|
|
435
|
-
* This is a protected helper method that providers can use to build messages
|
|
436
|
-
* with automatic multimodal detection, eliminating code duplication
|
|
437
|
-
*
|
|
438
|
-
* @param options - Stream options or text generation options
|
|
439
|
-
* @returns Promise resolving to ModelMessage array ready for AI SDK
|
|
440
|
-
*/
|
|
441
|
-
async buildMessagesForStream(options) {
|
|
442
|
-
return this.messageBuilder.buildMessagesForStream(options);
|
|
443
|
-
}
|
|
444
|
-
/**
|
|
445
|
-
* Execute the generation with AI SDK - delegated to GenerationHandler
|
|
446
|
-
*/
|
|
447
|
-
async executeGeneration(model, messages, tools, options) {
|
|
448
|
-
return this.generationHandler.executeGeneration(model, messages, tools, options);
|
|
449
|
-
}
|
|
450
|
-
/**
|
|
451
|
-
* Log generation completion information - delegated to GenerationHandler
|
|
452
|
-
*/
|
|
453
|
-
logGenerationComplete(generateResult) {
|
|
454
|
-
this.generationHandler.logGenerationComplete(generateResult);
|
|
455
|
-
}
|
|
456
|
-
/**
|
|
457
|
-
* Record performance metrics - delegated to TelemetryHandler
|
|
458
|
-
*/
|
|
459
|
-
async recordPerformanceMetrics(usage, responseTime) {
|
|
460
|
-
await this.telemetryHandler.recordPerformanceMetrics(usage, responseTime);
|
|
461
|
-
}
|
|
462
|
-
/**
|
|
463
|
-
* Extract tool information from generation result - delegated to GenerationHandler
|
|
464
|
-
*/
|
|
465
|
-
extractToolInformation(generateResult) {
|
|
466
|
-
return this.generationHandler.extractToolInformation(generateResult);
|
|
467
|
-
}
|
|
468
|
-
/**
|
|
469
|
-
* Format the enhanced result - delegated to GenerationHandler
|
|
470
|
-
*/
|
|
471
|
-
formatEnhancedResult(generateResult, tools, toolsUsed, toolExecutions, options) {
|
|
472
|
-
return this.generationHandler.formatEnhancedResult(generateResult, tools, toolsUsed, toolExecutions, options);
|
|
473
|
-
}
|
|
474
|
-
/**
|
|
475
|
-
* Analyze AI response structure and log detailed debugging information - delegated to GenerationHandler
|
|
476
|
-
*/
|
|
477
|
-
analyzeAIResponse(result) {
|
|
478
|
-
this.generationHandler.analyzeAIResponse(result);
|
|
479
|
-
}
|
|
480
|
-
/**
|
|
481
|
-
* Text generation method - implements AIProvider interface
|
|
482
|
-
* Tools are always available unless explicitly disabled
|
|
483
|
-
*
|
|
484
|
-
* Supports Text-to-Speech (TTS) audio generation in two modes:
|
|
485
|
-
* 1. Direct synthesis (default): TTS synthesizes the input text without AI generation
|
|
486
|
-
* 2. AI response synthesis: TTS synthesizes the AI-generated response after generation
|
|
487
|
-
*
|
|
488
|
-
* When TTS is enabled with useAiResponse=false (default), the method returns early with
|
|
489
|
-
* only the audio result, skipping AI generation entirely for optimal performance.
|
|
490
|
-
*
|
|
491
|
-
* When TTS is enabled with useAiResponse=true, the method performs full AI generation
|
|
492
|
-
* and then synthesizes the AI response to audio.
|
|
493
|
-
*
|
|
494
|
-
* @param optionsOrPrompt - Generation options or prompt string
|
|
495
|
-
* @param _analysisSchema - Optional analysis schema (not used)
|
|
496
|
-
* @returns Enhanced result with optional audio field containing TTSResult
|
|
497
|
-
*
|
|
498
|
-
* IMPLEMENTATION NOTE: Uses streamText() under the hood and accumulates results
|
|
499
|
-
* for consistency and better performance
|
|
500
|
-
*/
|
|
501
|
-
async generate(optionsOrPrompt, _analysisSchema) {
|
|
502
|
-
const options = this.normalizeTextOptions(optionsOrPrompt);
|
|
503
|
-
this.validateOptions(options);
|
|
504
|
-
const startTime = Date.now();
|
|
505
|
-
// Observability: create metrics span for provider.generate
|
|
506
|
-
const metricsSpan = SpanSerializer.createSpan(SpanType.MODEL_GENERATION, "provider.generate", {
|
|
507
|
-
"ai.provider": this.providerName || "unknown",
|
|
508
|
-
"ai.model": this.modelName || options.model || "unknown",
|
|
509
|
-
"ai.temperature": options.temperature,
|
|
510
|
-
"ai.max_tokens": options.maxTokens,
|
|
511
|
-
}, this._traceContext?.parentSpanId, this._traceContext?.traceId);
|
|
512
|
-
// OTEL span for provider-level generate tracing
|
|
513
|
-
// Use startActiveSpan pattern via context.with() so child spans become descendants
|
|
514
|
-
const otelSpan = tracers.provider.startSpan("neurolink.provider.generate", {
|
|
515
|
-
kind: SpanKind.CLIENT,
|
|
516
|
-
attributes: {
|
|
517
|
-
[ATTR.GEN_AI_SYSTEM]: this.providerName || "unknown",
|
|
518
|
-
[ATTR.GEN_AI_MODEL]: this.modelName || options.model || "unknown",
|
|
519
|
-
[ATTR.GEN_AI_OPERATION]: "generate",
|
|
520
|
-
[ATTR.NL_PROVIDER]: this.providerName || "unknown",
|
|
521
|
-
},
|
|
522
|
-
});
|
|
523
|
-
// Set this span as the active context so child spans (GenerationHandler, etc.) become descendants
|
|
524
|
-
const activeCtx = trace.setSpan(context.active(), otelSpan);
|
|
525
|
-
let otelSpanEnded = false;
|
|
526
|
-
return await context.with(activeCtx, async () => {
|
|
527
|
-
try {
|
|
528
|
-
// ===== VIDEO GENERATION MODE =====
|
|
529
|
-
// Generate video from image + prompt using Veo 3.1
|
|
530
|
-
if (options.output?.mode === "video") {
|
|
531
|
-
return await this.handleVideoGeneration(options, startTime);
|
|
532
|
-
}
|
|
533
|
-
// ===== IMAGE GENERATION MODE =====
|
|
534
|
-
// Route to executeImageGeneration for image generation models
|
|
535
|
-
const isImageModel = IMAGE_GENERATION_MODELS.some((m) => this.modelName.includes(m));
|
|
536
|
-
if (isImageModel) {
|
|
537
|
-
logger.info(`Image generation model detected, routing to executeImageGeneration`, {
|
|
538
|
-
provider: this.providerName,
|
|
539
|
-
model: this.modelName,
|
|
540
|
-
});
|
|
541
|
-
const imageResult = await this.executeImageGeneration(options);
|
|
542
|
-
return await this.enhanceResult(imageResult, options, startTime);
|
|
543
|
-
}
|
|
544
|
-
// ===== TTS MODE 1: Direct Input Synthesis (useAiResponse=false) =====
|
|
545
|
-
// Synthesize input text directly without AI generation
|
|
546
|
-
// This is optimal for simple read-aloud scenarios
|
|
547
|
-
if (options.tts?.enabled && !options.tts?.useAiResponse) {
|
|
548
|
-
const textToSynthesize = options.prompt ?? options.input?.text ?? "";
|
|
549
|
-
// Build base result structure - common to both paths
|
|
550
|
-
const baseResult = {
|
|
551
|
-
content: textToSynthesize,
|
|
552
|
-
provider: options.provider ?? this.providerName,
|
|
553
|
-
model: this.modelName,
|
|
554
|
-
usage: { input: 0, output: 0, total: 0 },
|
|
555
|
-
};
|
|
556
|
-
try {
|
|
557
|
-
const ttsResult = await TTSProcessor.synthesize(textToSynthesize, options.provider ?? this.providerName, options.tts);
|
|
558
|
-
baseResult.audio = ttsResult;
|
|
559
|
-
}
|
|
560
|
-
catch (ttsError) {
|
|
561
|
-
logger.error(`TTS synthesis failed in Mode 1 (direct input synthesis):`, ttsError);
|
|
562
|
-
// baseResult remains without audio - graceful degradation
|
|
563
|
-
}
|
|
564
|
-
// Call enhanceResult for consistency - enables analytics/evaluation for TTS-only requests
|
|
565
|
-
return await this.enhanceResult(baseResult, options, startTime);
|
|
566
|
-
}
|
|
567
|
-
// ===== Normal AI Generation Flow =====
|
|
568
|
-
const { tools, model } = await this.prepareGenerationContext(options);
|
|
569
|
-
const messages = await this.buildMessages(options);
|
|
570
|
-
// ===== VIDEO ANALYSIS FROM MESSAGES CONTENT =====
|
|
571
|
-
// Check if video files are present in messages content array
|
|
572
|
-
// If video analysis is needed, perform it via Gemini, then pass through Claude for formatting
|
|
573
|
-
if (hasVideoFrames(messages)) {
|
|
574
|
-
const videoAnalysisResult = await executeVideoAnalysis(messages, {
|
|
575
|
-
provider: options.provider,
|
|
576
|
-
providerName: this.providerName,
|
|
577
|
-
region: options.region,
|
|
578
|
-
// Don't pass the main conversation model — video analysis uses
|
|
579
|
-
// Google's Gemini API (generateContent) which only supports Gemini models.
|
|
580
|
-
// Let videoAnalysisProcessor use its own default (gemini-2.5-flash).
|
|
581
|
-
});
|
|
582
|
-
// Extract user's original text from messages (excluding image parts)
|
|
583
|
-
const userTextParts = messages
|
|
584
|
-
.filter((m) => m.role === "user")
|
|
585
|
-
.flatMap((m) => Array.isArray(m.content)
|
|
586
|
-
? m.content
|
|
587
|
-
.filter((p) => p.type === "text")
|
|
588
|
-
.map((p) => p.text)
|
|
589
|
-
: [typeof m.content === "string" ? m.content : ""])
|
|
590
|
-
.filter(Boolean);
|
|
591
|
-
const userText = userTextParts.join("\n").trim();
|
|
592
|
-
// Pass Gemini's analysis through Claude for structured JSON formatting
|
|
593
|
-
// The system prompt (from Curator) includes JSON_REPORT_PROMPT_SUFFIX
|
|
594
|
-
// which instructs Claude to output {"summary": "...", "details": "..."}
|
|
595
|
-
let formattedContent = videoAnalysisResult;
|
|
596
|
-
let usage = { input: 0, output: 0, total: 0 };
|
|
597
|
-
if (options.systemPrompt) {
|
|
598
|
-
try {
|
|
599
|
-
const formattingPrompt = userText
|
|
600
|
-
? `The user asked: "${userText}"\n\nHere is the video/image analysis result from the visual analysis system:\n\n${videoAnalysisResult}\n\nBased on this analysis, provide your response.`
|
|
601
|
-
: `Here is a video/image analysis result from the visual analysis system:\n\n${videoAnalysisResult}\n\nBased on this analysis, provide your response.`;
|
|
602
|
-
logger.debug("[VideoAnalysis] Formatting via Claude", {
|
|
603
|
-
userTextLength: userText.length,
|
|
604
|
-
analysisLength: videoAnalysisResult.length,
|
|
605
|
-
});
|
|
606
|
-
const formattedResult = await generateText({
|
|
607
|
-
model,
|
|
608
|
-
system: options.systemPrompt,
|
|
609
|
-
messages: [
|
|
610
|
-
{ role: "user", content: formattingPrompt },
|
|
611
|
-
],
|
|
612
|
-
maxOutputTokens: options.maxTokens || 8192,
|
|
613
|
-
temperature: 0.3,
|
|
614
|
-
abortSignal: options.abortSignal,
|
|
615
|
-
experimental_telemetry: this.telemetryHandler?.getTelemetryConfig(options, "generate"),
|
|
616
|
-
});
|
|
617
|
-
formattedContent = formattedResult.text;
|
|
618
|
-
usage = {
|
|
619
|
-
input: formattedResult.usage?.inputTokens || 0,
|
|
620
|
-
output: formattedResult.usage?.outputTokens || 0,
|
|
621
|
-
total: (formattedResult.usage?.inputTokens || 0) +
|
|
622
|
-
(formattedResult.usage?.outputTokens || 0),
|
|
623
|
-
};
|
|
624
|
-
logger.debug("[VideoAnalysis] Claude formatting complete", {
|
|
625
|
-
formattedLength: formattedContent.length,
|
|
626
|
-
usage,
|
|
627
|
-
});
|
|
628
|
-
}
|
|
629
|
-
catch (error) {
|
|
630
|
-
logger.warn("[VideoAnalysis] Claude formatting failed, using raw Gemini output", {
|
|
631
|
-
error: error instanceof Error ? error.message : String(error),
|
|
632
|
-
});
|
|
633
|
-
// formattedContent remains as raw videoAnalysisResult (graceful degradation)
|
|
634
|
-
}
|
|
635
|
-
}
|
|
636
|
-
const videoResult = {
|
|
637
|
-
content: formattedContent,
|
|
638
|
-
provider: options.provider ?? this.providerName,
|
|
639
|
-
model: this.modelName,
|
|
640
|
-
usage,
|
|
641
|
-
};
|
|
642
|
-
return await this.enhanceResult(videoResult, options, startTime);
|
|
643
|
-
}
|
|
644
|
-
// Compose timeout signal with user-provided abort signal (mirrors stream path)
|
|
645
|
-
const timeoutController = createTimeoutController(options.timeout, this.providerName, "generate");
|
|
646
|
-
const composedSignal = composeAbortSignals(options.abortSignal, timeoutController?.controller.signal);
|
|
647
|
-
const composedOptions = composedSignal
|
|
648
|
-
? { ...options, abortSignal: composedSignal }
|
|
649
|
-
: options;
|
|
650
|
-
let generateResult;
|
|
651
|
-
try {
|
|
652
|
-
generateResult = await this.executeGeneration(model, messages, tools, composedOptions);
|
|
653
|
-
}
|
|
654
|
-
finally {
|
|
655
|
-
timeoutController?.cleanup();
|
|
656
|
-
}
|
|
657
|
-
this.analyzeAIResponse(generateResult);
|
|
658
|
-
this.logGenerationComplete(generateResult);
|
|
659
|
-
const responseTime = Date.now() - startTime;
|
|
660
|
-
await this.recordPerformanceMetrics(generateResult.usage, responseTime);
|
|
661
|
-
const { toolsUsed, toolExecutions } = this.extractToolInformation(generateResult);
|
|
662
|
-
let enhancedResult = this.formatEnhancedResult(generateResult, tools, toolsUsed, toolExecutions, options);
|
|
663
|
-
// ===== TTS MODE 2: AI Response Synthesis (useAiResponse=true) =====
|
|
664
|
-
// Synthesize AI-generated response after generation completes
|
|
665
|
-
if (options.tts?.enabled && options.tts?.useAiResponse) {
|
|
666
|
-
const aiResponse = enhancedResult.content;
|
|
667
|
-
const provider = options.provider ?? this.providerName;
|
|
668
|
-
// Validate AI response and provider before synthesis
|
|
669
|
-
if (aiResponse && provider) {
|
|
670
|
-
try {
|
|
671
|
-
const ttsResult = await TTSProcessor.synthesize(aiResponse, provider, options.tts);
|
|
672
|
-
// Add audio to enhanced result (TTSProcessor already includes latency in metadata)
|
|
673
|
-
enhancedResult = {
|
|
674
|
-
...enhancedResult,
|
|
675
|
-
audio: ttsResult,
|
|
676
|
-
};
|
|
677
|
-
}
|
|
678
|
-
catch (ttsError) {
|
|
679
|
-
// Log TTS error but continue with text-only result
|
|
680
|
-
logger.error(`TTS synthesis failed in Mode 2 (AI response synthesis):`, ttsError);
|
|
681
|
-
// enhancedResult remains unchanged (no audio field added)
|
|
682
|
-
}
|
|
683
|
-
}
|
|
684
|
-
else {
|
|
685
|
-
logger.warn(`TTS synthesis skipped despite being enabled`, {
|
|
686
|
-
provider: this.providerName,
|
|
687
|
-
hasAiResponse: !!aiResponse,
|
|
688
|
-
aiResponseLength: aiResponse?.length ?? 0,
|
|
689
|
-
hasProvider: !!provider,
|
|
690
|
-
ttsConfig: {
|
|
691
|
-
enabled: options.tts?.enabled,
|
|
692
|
-
useAiResponse: options.tts?.useAiResponse,
|
|
693
|
-
},
|
|
694
|
-
reason: !aiResponse
|
|
695
|
-
? "AI response is empty or undefined"
|
|
696
|
-
: "Provider is missing",
|
|
697
|
-
});
|
|
698
|
-
}
|
|
699
|
-
}
|
|
700
|
-
// Observability: record successful generate span with token/cost data
|
|
701
|
-
let enrichedGenerateSpan = { ...metricsSpan };
|
|
702
|
-
if (enhancedResult?.usage) {
|
|
703
|
-
enrichedGenerateSpan = SpanSerializer.enrichWithTokenUsage(enrichedGenerateSpan, {
|
|
704
|
-
promptTokens: enhancedResult.usage.input || 0,
|
|
705
|
-
completionTokens: enhancedResult.usage.output || 0,
|
|
706
|
-
totalTokens: enhancedResult.usage.total || 0,
|
|
707
|
-
});
|
|
708
|
-
const cost = calculateCost(this.providerName, this.modelName, {
|
|
709
|
-
input: enhancedResult.usage.input || 0,
|
|
710
|
-
output: enhancedResult.usage.output || 0,
|
|
711
|
-
total: enhancedResult.usage.total || 0,
|
|
712
|
-
});
|
|
713
|
-
if (cost && cost > 0) {
|
|
714
|
-
enrichedGenerateSpan = SpanSerializer.enrichWithCost(enrichedGenerateSpan, {
|
|
715
|
-
totalCost: cost,
|
|
716
|
-
});
|
|
717
|
-
}
|
|
718
|
-
}
|
|
719
|
-
const _endedGenerateSpan = SpanSerializer.endSpan(enrichedGenerateSpan, SpanStatus.OK);
|
|
720
|
-
// Note: Do NOT record to getMetricsAggregator() here — the neurolink.ts
|
|
721
|
-
// generation:end listener creates an authoritative span with richer context
|
|
722
|
-
// (provider name, model, input/output) and records to both aggregators.
|
|
723
|
-
// Recording here would double-count cost and token metrics.
|
|
724
|
-
return await this.enhanceResult(enhancedResult, options, startTime);
|
|
725
|
-
}
|
|
726
|
-
catch (error) {
|
|
727
|
-
// Observability: record failed generate span
|
|
728
|
-
const _endedGenerateSpan = SpanSerializer.endSpan(metricsSpan, SpanStatus.ERROR, error instanceof Error ? error.message : String(error));
|
|
729
|
-
// Note: Do NOT record to getMetricsAggregator() here — neurolink.ts
|
|
730
|
-
// handles authoritative metrics recording to avoid double-counting.
|
|
731
|
-
otelSpan.setStatus({
|
|
732
|
-
code: SpanStatusCode.ERROR,
|
|
733
|
-
message: error instanceof Error ? error.message : String(error),
|
|
734
|
-
});
|
|
735
|
-
otelSpan.end();
|
|
736
|
-
otelSpanEnded = true;
|
|
737
|
-
// Abort errors are expected when a generation is cancelled — log at info, not error
|
|
738
|
-
if (isAbortError(error)) {
|
|
739
|
-
logger.info(`Generate aborted for ${this.providerName}`, {
|
|
740
|
-
error: error instanceof Error ? error.message : String(error),
|
|
741
|
-
});
|
|
742
|
-
}
|
|
743
|
-
else {
|
|
744
|
-
logger.error(`Generate failed for ${this.providerName}:`, error);
|
|
745
|
-
}
|
|
746
|
-
throw this.handleProviderError(error);
|
|
747
|
-
}
|
|
748
|
-
finally {
|
|
749
|
-
if (!otelSpanEnded) {
|
|
750
|
-
otelSpan.setStatus({ code: SpanStatusCode.OK });
|
|
751
|
-
otelSpan.end();
|
|
752
|
-
}
|
|
753
|
-
}
|
|
754
|
-
}); // end context.with
|
|
755
|
-
}
|
|
756
|
-
/**
|
|
757
|
-
* Alias for generate method - implements AIProvider interface
|
|
758
|
-
*/
|
|
759
|
-
async gen(optionsOrPrompt, analysisSchema) {
|
|
760
|
-
return this.generate(optionsOrPrompt, analysisSchema);
|
|
761
|
-
}
|
|
762
|
-
/**
|
|
763
|
-
* BACKWARD COMPATIBILITY: Legacy generateText method
|
|
764
|
-
* Converts EnhancedGenerateResult to TextGenerationResult format
|
|
765
|
-
* Ensures existing scripts using createAIProvider().generateText() continue to work
|
|
766
|
-
*/
|
|
767
|
-
async generateText(options) {
|
|
768
|
-
// Validate required parameters for backward compatibility - support both prompt and input.text
|
|
769
|
-
const promptText = options.prompt || options.input?.text;
|
|
770
|
-
if (!promptText ||
|
|
771
|
-
typeof promptText !== "string" ||
|
|
772
|
-
promptText.trim() === "") {
|
|
773
|
-
throw new Error("GenerateText options must include prompt or input.text as a non-empty string");
|
|
774
|
-
}
|
|
775
|
-
// Call the main generate method
|
|
776
|
-
const result = await this.generate(options);
|
|
777
|
-
if (!result) {
|
|
778
|
-
throw new Error("Generation failed: No result returned");
|
|
779
|
-
}
|
|
780
|
-
// Convert EnhancedGenerateResult to TextGenerationResult format
|
|
781
|
-
return {
|
|
782
|
-
content: result.content || "",
|
|
783
|
-
provider: result.provider || this.providerName,
|
|
784
|
-
model: result.model || this.modelName,
|
|
785
|
-
usage: result.usage || {
|
|
786
|
-
input: 0,
|
|
787
|
-
output: 0,
|
|
788
|
-
total: 0,
|
|
789
|
-
},
|
|
790
|
-
responseTime: 0, // BaseProvider doesn't track response time directly
|
|
791
|
-
toolsUsed: result.toolsUsed || [],
|
|
792
|
-
enhancedWithTools: !!(result.toolsUsed && result.toolsUsed.length > 0),
|
|
793
|
-
analytics: result.analytics,
|
|
794
|
-
evaluation: result.evaluation,
|
|
795
|
-
audio: result.audio,
|
|
796
|
-
};
|
|
797
|
-
}
|
|
798
|
-
/**
|
|
799
|
-
* Generate embeddings for text
|
|
800
|
-
*
|
|
801
|
-
* This is a default implementation that throws an error.
|
|
802
|
-
* Providers that support embeddings (OpenAI, Google Vertex, Amazon Bedrock)
|
|
803
|
-
* should override this method with their specific implementation.
|
|
804
|
-
*
|
|
805
|
-
* @param text - The text to embed
|
|
806
|
-
* @param _modelName - Optional embedding model name (provider-specific)
|
|
807
|
-
* @returns Promise resolving to the embedding vector (array of numbers)
|
|
808
|
-
* @throws Error if the provider does not support embeddings
|
|
809
|
-
*
|
|
810
|
-
* @example
|
|
811
|
-
* ```typescript
|
|
812
|
-
* const provider = await ProviderFactory.createProvider('openai', 'text-embedding-3-small');
|
|
813
|
-
* const embedding = await provider.embed('Hello world');
|
|
814
|
-
* console.log(embedding); // [0.123, -0.456, ...]
|
|
815
|
-
* ```
|
|
816
|
-
*/
|
|
817
|
-
async embed(text, _modelName) {
|
|
818
|
-
logger.warn(`embed() called on ${this.providerName} which does not have a native implementation`, {
|
|
819
|
-
textLength: text.length,
|
|
820
|
-
});
|
|
821
|
-
throw new Error(`Embedding generation is not supported by the ${this.providerName} provider. ` +
|
|
822
|
-
`Supported providers: openai, vertex/google, bedrock. ` +
|
|
823
|
-
`Use an embedding model like text-embedding-3-small (OpenAI), text-embedding-004 (Vertex), ` +
|
|
824
|
-
`or amazon.titan-embed-text-v2:0 (Bedrock).`);
|
|
825
|
-
}
|
|
826
|
-
/**
|
|
827
|
-
* Generate embeddings for multiple texts in a single batch
|
|
828
|
-
*
|
|
829
|
-
* This is a default implementation that throws an error.
|
|
830
|
-
* Providers that support embeddings should override this method.
|
|
831
|
-
* The AI SDK's embedMany automatically handles chunking for models with batch limits.
|
|
832
|
-
*
|
|
833
|
-
* @param texts - The texts to embed
|
|
834
|
-
* @param _modelName - Optional embedding model name (provider-specific)
|
|
835
|
-
* @returns Promise resolving to an array of embedding vectors
|
|
836
|
-
* @throws Error if the provider does not support embeddings
|
|
837
|
-
*/
|
|
838
|
-
async embedMany(texts, _modelName) {
|
|
839
|
-
logger.warn(`embedMany() called on ${this.providerName} which does not have a native implementation`, {
|
|
840
|
-
count: texts.length,
|
|
841
|
-
});
|
|
842
|
-
throw new Error(`Batch embedding generation is not supported by the ${this.providerName} provider. ` +
|
|
843
|
-
`Supported providers: openai, googleAiStudio, vertex/google, bedrock. ` +
|
|
844
|
-
`Use an embedding model like text-embedding-3-small (OpenAI), gemini-embedding-001 (Google AI), ` +
|
|
845
|
-
`text-embedding-004 (Vertex), or amazon.titan-embed-text-v2:0 (Bedrock).`);
|
|
846
|
-
}
|
|
847
|
-
/**
|
|
848
|
-
* Get the default embedding model for this provider
|
|
849
|
-
*
|
|
850
|
-
* Override in subclasses to provide provider-specific defaults.
|
|
851
|
-
* Returns undefined for providers that don't support embeddings.
|
|
852
|
-
*
|
|
853
|
-
* @returns The default embedding model name, or undefined if not supported
|
|
854
|
-
*/
|
|
855
|
-
getDefaultEmbeddingModel() {
|
|
856
|
-
// Default implementation returns undefined - providers override this
|
|
857
|
-
return undefined;
|
|
858
|
-
}
|
|
859
|
-
/**
|
|
860
|
-
* Get AI SDK model with middleware applied
|
|
861
|
-
* This method wraps the base model with any configured middleware
|
|
862
|
-
* TODO: Implement global level middlewares that can be used
|
|
863
|
-
*/
|
|
864
|
-
async getAISDKModelWithMiddleware(options = {}) {
|
|
865
|
-
// Get the base model
|
|
866
|
-
const baseModel = await this.getAISDKModel();
|
|
867
|
-
logger.debug(`Retrieved base model for ${this.providerName}`, {
|
|
868
|
-
provider: this.providerName,
|
|
869
|
-
model: this.modelName,
|
|
870
|
-
hasMiddlewareConfig: !!this.middlewareOptions,
|
|
871
|
-
timestamp: Date.now(),
|
|
872
|
-
});
|
|
873
|
-
// Check if middleware should be applied
|
|
874
|
-
const middlewareOptions = this.extractMiddlewareOptions(options);
|
|
875
|
-
logger.debug(`Middleware extraction result`, {
|
|
876
|
-
provider: this.providerName,
|
|
877
|
-
model: this.modelName,
|
|
878
|
-
middlewareOptions,
|
|
879
|
-
});
|
|
880
|
-
if (!middlewareOptions) {
|
|
881
|
-
return baseModel;
|
|
882
|
-
}
|
|
883
|
-
try {
|
|
884
|
-
logger.debug(`Applying middleware to ${this.providerName} model`, {
|
|
885
|
-
provider: this.providerName,
|
|
886
|
-
model: this.modelName,
|
|
887
|
-
middlewareOptions,
|
|
888
|
-
});
|
|
889
|
-
// Create a new factory instance with the specified options
|
|
890
|
-
const factory = new MiddlewareFactory(middlewareOptions);
|
|
891
|
-
// Create middleware context
|
|
892
|
-
const context = factory.createContext(this.providerName, this.modelName, options, {
|
|
893
|
-
sessionId: this.sessionId,
|
|
894
|
-
userId: this.userId,
|
|
895
|
-
});
|
|
896
|
-
// Apply middleware to the model
|
|
897
|
-
const wrappedModel = factory.applyMiddleware(baseModel, context, middlewareOptions);
|
|
898
|
-
logger.debug(`Applied middleware to ${this.providerName} model`, {
|
|
899
|
-
provider: this.providerName,
|
|
900
|
-
model: this.modelName,
|
|
901
|
-
hasMiddleware: true,
|
|
902
|
-
});
|
|
903
|
-
return wrappedModel;
|
|
904
|
-
}
|
|
905
|
-
catch (error) {
|
|
906
|
-
logger.warn(`Failed to apply middleware to ${this.providerName}, using base model`, {
|
|
907
|
-
error: error instanceof Error ? error.message : String(error),
|
|
908
|
-
});
|
|
909
|
-
// Return base model on middleware failure to maintain functionality
|
|
910
|
-
return baseModel;
|
|
911
|
-
}
|
|
912
|
-
}
|
|
913
|
-
/**
|
|
914
|
-
* Extract middleware options - delegated to Utilities
|
|
915
|
-
*/
|
|
916
|
-
extractMiddlewareOptions(options) {
|
|
917
|
-
return this.utilities.extractMiddlewareOptions(options);
|
|
918
|
-
}
|
|
919
|
-
// ===================
|
|
920
|
-
// TOOL MANAGEMENT
|
|
921
|
-
// ===================
|
|
922
|
-
/**
|
|
923
|
-
* Check if a schema is a Zod schema - delegated to Utilities
|
|
924
|
-
*/
|
|
925
|
-
isZodSchema(schema) {
|
|
926
|
-
return this.utilities.isZodSchema(schema);
|
|
927
|
-
}
|
|
928
|
-
/**
|
|
929
|
-
* Convert tool execution result - delegated to Utilities
|
|
930
|
-
*/
|
|
931
|
-
async convertToolResult(result) {
|
|
932
|
-
return this.utilities.convertToolResult(result);
|
|
933
|
-
}
|
|
934
|
-
/**
|
|
935
|
-
* Fix JSON Schema for OpenAI strict mode - delegated to Utilities
|
|
936
|
-
*/
|
|
937
|
-
fixSchemaForOpenAIStrictMode(schema) {
|
|
938
|
-
return this.utilities.fixSchemaForOpenAIStrictMode(schema);
|
|
939
|
-
}
|
|
940
|
-
/**
|
|
941
|
-
* Get all available tools - delegated to ToolsManager
|
|
942
|
-
*/
|
|
943
|
-
async getAllTools() {
|
|
944
|
-
return this.toolsManager.getAllTools();
|
|
945
|
-
}
|
|
946
|
-
/**
|
|
947
|
-
* Calculate actual cost - delegated to TelemetryHandler
|
|
948
|
-
*/
|
|
949
|
-
async calculateActualCost(usage) {
|
|
950
|
-
return this.telemetryHandler.calculateActualCost(usage);
|
|
951
|
-
}
|
|
952
|
-
/**
|
|
953
|
-
* Create a permissive Zod schema - delegated to Utilities
|
|
954
|
-
*/
|
|
955
|
-
createPermissiveZodSchema() {
|
|
956
|
-
return this.utilities.createPermissiveZodSchema();
|
|
957
|
-
}
|
|
958
|
-
/**
|
|
959
|
-
* Set session context for MCP tools - delegated to ToolsManager
|
|
960
|
-
*/
|
|
961
|
-
setSessionContext(sessionId, userId) {
|
|
962
|
-
this.sessionId = sessionId;
|
|
963
|
-
this.userId = userId;
|
|
964
|
-
this.toolsManager.setSessionContext(sessionId, userId);
|
|
965
|
-
}
|
|
966
|
-
/**
|
|
967
|
-
* Handle provider errors with abort passthrough.
|
|
968
|
-
* AbortErrors are never wrapped — they must propagate with their
|
|
969
|
-
* original identity so that isAbortError() can detect them in
|
|
970
|
-
* retry/fallback loops (directProviderGeneration, performMCPGenerationRetries).
|
|
971
|
-
*/
|
|
972
|
-
handleProviderError(error) {
|
|
973
|
-
if (isAbortError(error)) {
|
|
974
|
-
// Preserve AbortError identity — never wrap in provider-specific formatting
|
|
975
|
-
return error instanceof Error
|
|
976
|
-
? error
|
|
977
|
-
: new DOMException("The operation was aborted", "AbortError");
|
|
978
|
-
}
|
|
979
|
-
return this.formatProviderError(error);
|
|
980
|
-
}
|
|
981
|
-
/**
|
|
982
|
-
* Image generation method. Providers that support it should override this.
|
|
983
|
-
* By default, it throws an error indicating that the functionality is not supported.
|
|
984
|
-
* @param _options The generation options.
|
|
985
|
-
* @returns A promise that resolves to the generation result.
|
|
986
|
-
*/
|
|
987
|
-
async executeImageGeneration(_options) {
|
|
988
|
-
throw new Error(`Image generation is not supported by the ${this.providerName} provider or the selected model.`);
|
|
989
|
-
}
|
|
990
|
-
// ===================
|
|
991
|
-
// CONSOLIDATED PROVIDER METHODS - MOVED FROM INDIVIDUAL PROVIDERS
|
|
992
|
-
// ===================
|
|
993
|
-
/**
|
|
994
|
-
* Execute operation with timeout and proper cleanup
|
|
995
|
-
* Consolidates identical timeout handling from 8/10 providers
|
|
996
|
-
*/
|
|
997
|
-
async executeWithTimeout(operation, options) {
|
|
998
|
-
const timeout = this.getTimeout(options);
|
|
999
|
-
const timeoutController = createTimeoutController(timeout, this.providerName, options.operationType || "generate");
|
|
1000
|
-
try {
|
|
1001
|
-
if (timeoutController) {
|
|
1002
|
-
return await Promise.race([
|
|
1003
|
-
operation(),
|
|
1004
|
-
new Promise((_, reject) => {
|
|
1005
|
-
timeoutController.controller.signal.addEventListener("abort", () => {
|
|
1006
|
-
reject(new TimeoutError(`${this.providerName} operation timed out`, timeoutController.timeoutMs, this.providerName, options.operationType ||
|
|
1007
|
-
"generate"));
|
|
1008
|
-
});
|
|
1009
|
-
}),
|
|
1010
|
-
]);
|
|
1011
|
-
}
|
|
1012
|
-
else {
|
|
1013
|
-
return await operation();
|
|
1014
|
-
}
|
|
1015
|
-
}
|
|
1016
|
-
finally {
|
|
1017
|
-
timeoutController?.cleanup();
|
|
1018
|
-
}
|
|
1019
|
-
}
|
|
1020
|
-
/**
|
|
1021
|
-
* Validate stream options - delegated to StreamHandler
|
|
1022
|
-
*/
|
|
1023
|
-
validateStreamOptions(options) {
|
|
1024
|
-
this.streamHandler.validateStreamOptions(options);
|
|
1025
|
-
}
|
|
1026
|
-
/**
|
|
1027
|
-
* Create text stream transformation - delegated to StreamHandler
|
|
1028
|
-
*/
|
|
1029
|
-
createTextStream(result) {
|
|
1030
|
-
return this.streamHandler.createTextStream(result);
|
|
1031
|
-
}
|
|
1032
|
-
/**
|
|
1033
|
-
* Create standardized stream result - delegated to StreamHandler
|
|
1034
|
-
*/
|
|
1035
|
-
createStreamResult(stream, additionalProps = {}) {
|
|
1036
|
-
return this.streamHandler.createStreamResult(stream, additionalProps);
|
|
1037
|
-
}
|
|
1038
|
-
/**
|
|
1039
|
-
* Create stream analytics - delegated to StreamHandler
|
|
1040
|
-
*/
|
|
1041
|
-
async createStreamAnalytics(result, startTime, options) {
|
|
1042
|
-
return this.streamHandler.createStreamAnalytics(result, startTime, options);
|
|
1043
|
-
}
|
|
1044
|
-
/**
|
|
1045
|
-
* Handle common error patterns - delegated to Utilities
|
|
1046
|
-
*/
|
|
1047
|
-
handleCommonErrors(error) {
|
|
1048
|
-
return this.utilities.handleCommonErrors(error);
|
|
1049
|
-
}
|
|
1050
|
-
/**
|
|
1051
|
-
* Set up tool executor - delegated to ToolsManager
|
|
1052
|
-
* @param sdk - The NeuroLinkSDK instance for tool execution
|
|
1053
|
-
* @param functionTag - Function name for logging
|
|
1054
|
-
*/
|
|
1055
|
-
setupToolExecutor(sdk, functionTag) {
|
|
1056
|
-
this.toolsManager.setupToolExecutor(sdk, functionTag);
|
|
1057
|
-
}
|
|
1058
|
-
// ===================
|
|
1059
|
-
// TEMPLATE METHODS - COMMON FUNCTIONALITY
|
|
1060
|
-
// ===================
|
|
1061
|
-
/**
|
|
1062
|
-
* Normalize text generation options - delegated to Utilities
|
|
1063
|
-
*/
|
|
1064
|
-
normalizeTextOptions(optionsOrPrompt) {
|
|
1065
|
-
return this.utilities.normalizeTextOptions(optionsOrPrompt);
|
|
1066
|
-
}
|
|
1067
|
-
/**
|
|
1068
|
-
* Normalize stream options - delegated to Utilities
|
|
1069
|
-
*/
|
|
1070
|
-
normalizeStreamOptions(optionsOrPrompt) {
|
|
1071
|
-
return this.utilities.normalizeStreamOptions(optionsOrPrompt);
|
|
1072
|
-
}
|
|
1073
|
-
async enhanceResult(result, options, startTime) {
|
|
1074
|
-
const responseTime = Date.now() - startTime;
|
|
1075
|
-
// CRITICAL FIX: Store imageOutput separately to ensure it's preserved
|
|
1076
|
-
const imageOutput = result.imageOutput;
|
|
1077
|
-
let enhancedResult = { ...result };
|
|
1078
|
-
if (options.enableAnalytics) {
|
|
1079
|
-
try {
|
|
1080
|
-
const analytics = await this.createAnalytics(result, responseTime, options);
|
|
1081
|
-
// Preserve ALL fields including imageOutput when adding analytics
|
|
1082
|
-
enhancedResult = { ...enhancedResult, analytics, imageOutput };
|
|
1083
|
-
}
|
|
1084
|
-
catch (error) {
|
|
1085
|
-
logger.warn(`Analytics creation failed for ${this.providerName}:`, error);
|
|
1086
|
-
}
|
|
1087
|
-
}
|
|
1088
|
-
if (options.enableEvaluation) {
|
|
1089
|
-
try {
|
|
1090
|
-
const evaluation = await this.createEvaluation(result, options);
|
|
1091
|
-
// Preserve ALL fields including imageOutput when adding evaluation
|
|
1092
|
-
enhancedResult = { ...enhancedResult, evaluation, imageOutput };
|
|
1093
|
-
}
|
|
1094
|
-
catch (error) {
|
|
1095
|
-
logger.warn(`Evaluation creation failed for ${this.providerName}:`, error);
|
|
1096
|
-
}
|
|
1097
|
-
}
|
|
1098
|
-
// CRITICAL FIX: Always restore imageOutput if it existed in the original result
|
|
1099
|
-
if (imageOutput) {
|
|
1100
|
-
enhancedResult.imageOutput = imageOutput;
|
|
1101
|
-
}
|
|
1102
|
-
return enhancedResult;
|
|
1103
|
-
}
|
|
1104
|
-
/**
|
|
1105
|
-
* Handle video generation mode
|
|
1106
|
-
*
|
|
1107
|
-
* Generates video from input image + text prompt using Vertex AI Veo 3.1.
|
|
1108
|
-
*
|
|
1109
|
-
* @param options - Text generation options with video configuration
|
|
1110
|
-
* @param startTime - Generation start timestamp for metrics
|
|
1111
|
-
* @returns Enhanced result with video data
|
|
1112
|
-
*
|
|
1113
|
-
* @example
|
|
1114
|
-
* ```typescript
|
|
1115
|
-
* const result = await provider.generate({
|
|
1116
|
-
* input: { text: "Product showcase", images: [imageBuffer] },
|
|
1117
|
-
* output: { mode: "video", video: { resolution: "1080p" } }
|
|
1118
|
-
* });
|
|
1119
|
-
* // result.video contains the generated video
|
|
1120
|
-
* ```
|
|
1121
|
-
*/
|
|
1122
|
-
async handleVideoGeneration(options, startTime) {
|
|
1123
|
-
// Dynamic imports to avoid loading video dependencies unless needed
|
|
1124
|
-
const { generateVideoWithVertex, VideoError, VIDEO_ERROR_CODES } = await import("../adapters/video/vertexVideoHandler.js");
|
|
1125
|
-
const { validateVideoGenerationInput, validateImageForVideo, validateDirectorModeInput, } = await import("../utils/parameterValidation.js");
|
|
1126
|
-
const { ErrorFactory } = await import("../utils/errorHandling.js");
|
|
1127
|
-
// Build GenerateOptions for validation
|
|
1128
|
-
const generateOptions = {
|
|
1129
|
-
input: options.input || { text: options.prompt || "" },
|
|
1130
|
-
output: options.output,
|
|
1131
|
-
provider: options.provider,
|
|
1132
|
-
model: options.model,
|
|
1133
|
-
};
|
|
1134
|
-
// ===== DIRECTOR MODE =====
|
|
1135
|
-
// Route to Director pipeline when segments are provided
|
|
1136
|
-
if (generateOptions.input?.segments &&
|
|
1137
|
-
Array.isArray(generateOptions.input.segments) &&
|
|
1138
|
-
generateOptions.input.segments.length > 0) {
|
|
1139
|
-
// Type narrowing: segments is guaranteed to exist here
|
|
1140
|
-
const segments = generateOptions.input.segments;
|
|
1141
|
-
const directorValidation = validateDirectorModeInput(generateOptions);
|
|
1142
|
-
if (!directorValidation.isValid) {
|
|
1143
|
-
throw ErrorFactory.invalidParameters("director-mode", new Error(directorValidation.errors
|
|
1144
|
-
.map((e) => e.message)
|
|
1145
|
-
.join("; ")), { errors: directorValidation.errors });
|
|
1146
|
-
}
|
|
1147
|
-
if (directorValidation.warnings.length > 0) {
|
|
1148
|
-
for (const warning of directorValidation.warnings) {
|
|
1149
|
-
logger.warn(`Director Mode warning: ${warning}`);
|
|
1150
|
-
}
|
|
1151
|
-
}
|
|
1152
|
-
const { executeDirectorPipeline, DIRECTOR_PIPELINE_TIMEOUT_MS } = await import("../adapters/video/directorPipeline.js");
|
|
1153
|
-
// Use caller's timeout if provided, otherwise use default Director timeout
|
|
1154
|
-
const directorTimeout = options.timeout ?? DIRECTOR_PIPELINE_TIMEOUT_MS;
|
|
1155
|
-
const videoResult = await this.executeWithTimeout(() => executeDirectorPipeline(segments, generateOptions.output?.video ?? {}, generateOptions.output?.director ?? {}, options.region), { timeout: directorTimeout, operationType: "generate" });
|
|
1156
|
-
// Build content summary with metadata
|
|
1157
|
-
const joinedPrompts = generateOptions.input.segments
|
|
1158
|
-
.map((s) => s.prompt)
|
|
1159
|
-
.join(" → ");
|
|
1160
|
-
const segmentCount = videoResult.metadata?.segmentCount ??
|
|
1161
|
-
generateOptions.input.segments.length;
|
|
1162
|
-
const transitionCount = videoResult.metadata?.transitionCount ?? Math.max(0, segmentCount - 1);
|
|
1163
|
-
const totalDuration = videoResult.metadata?.duration ?? 0;
|
|
1164
|
-
const contentSummary = `${joinedPrompts} — duration: ${totalDuration}s, segments: ${segmentCount}, transitions: ${transitionCount}`;
|
|
1165
|
-
const baseResult = {
|
|
1166
|
-
content: contentSummary,
|
|
1167
|
-
provider: "vertex",
|
|
1168
|
-
model: options.model || "veo-3.1-generate-001",
|
|
1169
|
-
usage: { input: 0, output: 0, total: 0 },
|
|
1170
|
-
video: videoResult,
|
|
1171
|
-
};
|
|
1172
|
-
return await this.enhanceResult(baseResult, options, startTime);
|
|
1173
|
-
}
|
|
1174
|
-
// ===== STANDARD SINGLE-CLIP VIDEO GENERATION =====
|
|
1175
|
-
// Validate video generation input
|
|
1176
|
-
const validation = validateVideoGenerationInput(generateOptions);
|
|
1177
|
-
if (!validation.isValid) {
|
|
1178
|
-
throw ErrorFactory.invalidParameters("video-generation", new Error(validation.errors.map((e) => e.message).join("; ")), { errors: validation.errors });
|
|
1179
|
-
}
|
|
1180
|
-
// Log warnings if any
|
|
1181
|
-
if (validation.warnings.length > 0) {
|
|
1182
|
-
for (const warning of validation.warnings) {
|
|
1183
|
-
logger.warn(`Video generation warning: ${warning}`);
|
|
1184
|
-
}
|
|
1185
|
-
}
|
|
1186
|
-
// Extract image from input
|
|
1187
|
-
const imageInput = options.input?.images?.[0];
|
|
1188
|
-
if (!imageInput) {
|
|
1189
|
-
throw new VideoError({
|
|
1190
|
-
code: VIDEO_ERROR_CODES.INVALID_INPUT,
|
|
1191
|
-
message: "Video generation requires an input image. Provide via input.images array.",
|
|
1192
|
-
retriable: false,
|
|
1193
|
-
context: { field: "input.images" },
|
|
1194
|
-
});
|
|
1195
|
-
}
|
|
1196
|
-
// Timeout for image IO operations (15 seconds)
|
|
1197
|
-
const IMAGE_IO_TIMEOUT_MS = 15000;
|
|
1198
|
-
// Load image buffer if path/URL
|
|
1199
|
-
let imageBuffer;
|
|
1200
|
-
if (typeof imageInput === "string") {
|
|
1201
|
-
if (imageInput.startsWith("http://") ||
|
|
1202
|
-
imageInput.startsWith("https://")) {
|
|
1203
|
-
// URL - fetch the image with timeout
|
|
1204
|
-
logger.debug("Fetching image from URL for video generation", {
|
|
1205
|
-
url: imageInput.substring(0, 100),
|
|
1206
|
-
});
|
|
1207
|
-
let response;
|
|
1208
|
-
try {
|
|
1209
|
-
response = await this.executeWithTimeout(() => fetch(imageInput), {
|
|
1210
|
-
timeout: IMAGE_IO_TIMEOUT_MS,
|
|
1211
|
-
operationType: "generate", // Part of video generation flow
|
|
1212
|
-
});
|
|
1213
|
-
}
|
|
1214
|
-
catch (error) {
|
|
1215
|
-
throw new VideoError({
|
|
1216
|
-
code: VIDEO_ERROR_CODES.INVALID_INPUT,
|
|
1217
|
-
message: `Failed to fetch image from URL: ${error instanceof Error ? error.message : "Request timed out"}`,
|
|
1218
|
-
retriable: true,
|
|
1219
|
-
context: { url: imageInput, timeout: IMAGE_IO_TIMEOUT_MS },
|
|
1220
|
-
originalError: error instanceof Error ? error : undefined,
|
|
1221
|
-
});
|
|
1222
|
-
}
|
|
1223
|
-
if (!response.ok) {
|
|
1224
|
-
throw new VideoError({
|
|
1225
|
-
code: VIDEO_ERROR_CODES.INVALID_INPUT,
|
|
1226
|
-
message: `Failed to fetch image from URL: ${response.status} ${response.statusText}`,
|
|
1227
|
-
retriable: response.status >= 500,
|
|
1228
|
-
context: { url: imageInput, status: response.status },
|
|
1229
|
-
});
|
|
1230
|
-
}
|
|
1231
|
-
imageBuffer = Buffer.from(await response.arrayBuffer());
|
|
1232
|
-
}
|
|
1233
|
-
else {
|
|
1234
|
-
// File path - read from disk with timeout
|
|
1235
|
-
logger.debug("Reading image from path for video generation", {
|
|
1236
|
-
path: imageInput,
|
|
1237
|
-
});
|
|
1238
|
-
const fs = await import("node:fs/promises");
|
|
1239
|
-
try {
|
|
1240
|
-
imageBuffer = await this.executeWithTimeout(() => fs.readFile(imageInput), { timeout: IMAGE_IO_TIMEOUT_MS, operationType: "generate" });
|
|
1241
|
-
}
|
|
1242
|
-
catch (error) {
|
|
1243
|
-
throw new VideoError({
|
|
1244
|
-
code: VIDEO_ERROR_CODES.INVALID_INPUT,
|
|
1245
|
-
message: `Failed to read image file: ${error instanceof Error ? error.message : String(error)}`,
|
|
1246
|
-
retriable: false,
|
|
1247
|
-
context: { path: imageInput, timeout: IMAGE_IO_TIMEOUT_MS },
|
|
1248
|
-
originalError: error instanceof Error ? error : undefined,
|
|
1249
|
-
});
|
|
1250
|
-
}
|
|
1251
|
-
}
|
|
1252
|
-
}
|
|
1253
|
-
else if (Buffer.isBuffer(imageInput)) {
|
|
1254
|
-
imageBuffer = imageInput;
|
|
1255
|
-
}
|
|
1256
|
-
else if (typeof imageInput === "object" && "data" in imageInput) {
|
|
1257
|
-
// ImageWithAltText type
|
|
1258
|
-
const imgData = imageInput.data;
|
|
1259
|
-
if (typeof imgData === "string") {
|
|
1260
|
-
imageBuffer = Buffer.from(imgData, "base64");
|
|
1261
|
-
}
|
|
1262
|
-
else if (Buffer.isBuffer(imgData)) {
|
|
1263
|
-
imageBuffer = imgData;
|
|
1264
|
-
}
|
|
1265
|
-
else {
|
|
1266
|
-
throw new VideoError({
|
|
1267
|
-
code: VIDEO_ERROR_CODES.INVALID_INPUT,
|
|
1268
|
-
message: "ImageWithAltText.data must be a base64 string or Buffer.",
|
|
1269
|
-
retriable: false,
|
|
1270
|
-
context: { field: "input.images[0].data", type: typeof imgData },
|
|
1271
|
-
});
|
|
1272
|
-
}
|
|
1273
|
-
}
|
|
1274
|
-
else {
|
|
1275
|
-
throw new VideoError({
|
|
1276
|
-
code: VIDEO_ERROR_CODES.INVALID_INPUT,
|
|
1277
|
-
message: "Invalid image input type. Provide Buffer, path string, URL, or ImageWithAltText.",
|
|
1278
|
-
retriable: false,
|
|
1279
|
-
context: { field: "input.images[0]", type: typeof imageInput },
|
|
1280
|
-
});
|
|
1281
|
-
}
|
|
1282
|
-
// Validate image format and size (for Buffer inputs)
|
|
1283
|
-
const imageValidation = validateImageForVideo(imageBuffer);
|
|
1284
|
-
if (imageValidation) {
|
|
1285
|
-
throw ErrorFactory.invalidParameters("video-generation", new Error(imageValidation.message), {
|
|
1286
|
-
field: "input.images[0]",
|
|
1287
|
-
validation: imageValidation,
|
|
1288
|
-
});
|
|
1289
|
-
}
|
|
1290
|
-
// Get prompt text
|
|
1291
|
-
const prompt = options.prompt || options.input?.text || "";
|
|
1292
|
-
logger.info("Starting video generation", {
|
|
1293
|
-
provider: "vertex",
|
|
1294
|
-
model: options.model || "veo-3.1-generate-001",
|
|
1295
|
-
promptLength: prompt.length,
|
|
1296
|
-
imageSize: imageBuffer.length,
|
|
1297
|
-
resolution: options.output?.video?.resolution || "720p",
|
|
1298
|
-
duration: options.output?.video?.length || 6,
|
|
1299
|
-
});
|
|
1300
|
-
// Generate video using Vertex handler (no processor abstraction)
|
|
1301
|
-
const videoResult = await generateVideoWithVertex(imageBuffer, prompt, options.output?.video, options.region);
|
|
1302
|
-
logger.info("Video generation complete", {
|
|
1303
|
-
videoSize: videoResult.data.length,
|
|
1304
|
-
duration: videoResult.metadata?.duration,
|
|
1305
|
-
processingTime: videoResult.metadata?.processingTime,
|
|
1306
|
-
});
|
|
1307
|
-
// Build result
|
|
1308
|
-
const baseResult = {
|
|
1309
|
-
content: prompt, // Echo the prompt as content
|
|
1310
|
-
provider: "vertex",
|
|
1311
|
-
model: options.model || "veo-3.1-generate-001",
|
|
1312
|
-
usage: { input: 0, output: 0, total: 0 },
|
|
1313
|
-
video: videoResult,
|
|
1314
|
-
};
|
|
1315
|
-
return await this.enhanceResult(baseResult, options, startTime);
|
|
1316
|
-
}
|
|
1317
|
-
/**
|
|
1318
|
-
* Create analytics - delegated to TelemetryHandler
|
|
1319
|
-
*/
|
|
1320
|
-
async createAnalytics(result, responseTime, options) {
|
|
1321
|
-
return this.telemetryHandler.createAnalytics(result, responseTime, options.context);
|
|
1322
|
-
}
|
|
1323
|
-
/**
|
|
1324
|
-
* Create evaluation - delegated to TelemetryHandler
|
|
1325
|
-
*/
|
|
1326
|
-
async createEvaluation(result, options) {
|
|
1327
|
-
return this.telemetryHandler.createEvaluation(result, options);
|
|
1328
|
-
}
|
|
1329
|
-
/**
|
|
1330
|
-
* Validate text generation options - delegated to Utilities
|
|
1331
|
-
*/
|
|
1332
|
-
validateOptions(options) {
|
|
1333
|
-
this.utilities.validateOptions(options);
|
|
1334
|
-
}
|
|
1335
|
-
/**
|
|
1336
|
-
* Get provider information - delegated to Utilities
|
|
1337
|
-
*/
|
|
1338
|
-
getProviderInfo() {
|
|
1339
|
-
return this.utilities.getProviderInfo();
|
|
1340
|
-
}
|
|
1341
|
-
/**
|
|
1342
|
-
* Get timeout value in milliseconds - delegated to Utilities
|
|
1343
|
-
*/
|
|
1344
|
-
getTimeout(options) {
|
|
1345
|
-
return this.utilities.getTimeout(options);
|
|
1346
|
-
}
|
|
1347
|
-
/**
|
|
1348
|
-
* Check if tool executions should be stored and handle storage
|
|
1349
|
-
*/
|
|
1350
|
-
async handleToolExecutionStorage(toolCalls, toolResults, options, currentTime) {
|
|
1351
|
-
return this.telemetryHandler.handleToolExecutionStorage(toolCalls, toolResults, options, currentTime);
|
|
1352
|
-
}
|
|
1353
|
-
/**
|
|
1354
|
-
* Utility method to chunk large prompts into smaller pieces
|
|
1355
|
-
* @param prompt The prompt to chunk
|
|
1356
|
-
* @param maxChunkSize Maximum size per chunk (default: 900,000 characters)
|
|
1357
|
-
* @param overlap Overlap between chunks to maintain context (default: 100 characters)
|
|
1358
|
-
* @returns Array of prompt chunks
|
|
1359
|
-
*/
|
|
1360
|
-
static chunkPrompt(prompt, maxChunkSize = 900000, overlap = 100) {
|
|
1361
|
-
if (prompt.length <= maxChunkSize) {
|
|
1362
|
-
return [prompt];
|
|
1363
|
-
}
|
|
1364
|
-
const chunks = [];
|
|
1365
|
-
let start = 0;
|
|
1366
|
-
while (start < prompt.length) {
|
|
1367
|
-
const end = Math.min(start + maxChunkSize, prompt.length);
|
|
1368
|
-
chunks.push(prompt.slice(start, end));
|
|
1369
|
-
// Break if we've reached the end
|
|
1370
|
-
if (end >= prompt.length) {
|
|
1371
|
-
break;
|
|
1372
|
-
}
|
|
1373
|
-
// Move start forward, accounting for overlap
|
|
1374
|
-
const nextStart = end - overlap;
|
|
1375
|
-
// Ensure we make progress (avoid infinite loops)
|
|
1376
|
-
if (nextStart <= start) {
|
|
1377
|
-
start = end;
|
|
1378
|
-
}
|
|
1379
|
-
else {
|
|
1380
|
-
start = Math.max(nextStart, 0);
|
|
1381
|
-
}
|
|
1382
|
-
}
|
|
1383
|
-
return chunks;
|
|
1384
|
-
}
|
|
1385
|
-
}
|