@juspay/neurolink 9.32.0 → 9.32.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/dist/auth/anthropicOAuth.js +1 -1
- package/dist/cli/commands/proxy.js +18 -5
- package/dist/client/aiSdkAdapter.js +1 -1
- package/dist/client/index.js +137 -501
- package/dist/core/factory.js +0 -1
- package/dist/core/redisConversationMemoryManager.js +1 -1
- package/dist/features/ppt/slideGenerator.js +0 -1
- package/dist/features/ppt/utils.js +0 -1
- package/dist/lib/server/routes/claudeProxyRoutes.js +45 -9
- package/dist/mcp/elicitationProtocol.js +1 -1
- package/dist/mcp/servers/agent/directToolsServer.js +0 -1
- package/dist/providers/azureOpenai.js +1 -1
- package/dist/providers/huggingFace.js +0 -1
- package/dist/providers/openaiCompatible.js +0 -1
- package/dist/sdk/toolRegistration.js +0 -1
- package/dist/server/openapi/generator.js +1 -1
- package/dist/server/routes/claudeProxyRoutes.js +45 -9
- package/dist/types/configTypes.js +0 -5
- package/dist/types/modelTypes.js +0 -1
- package/dist/types/tools.js +0 -1
- package/dist/types/typeAliases.js +0 -1
- package/dist/types/utilities.js +1 -1
- package/dist/types/workflowTypes.js +0 -1
- package/dist/utils/providerRetry.js +0 -1
- package/dist/utils/providerUtils.js +0 -1
- package/package.json +2 -2
- package/dist/client/adapters/providerImageAdapter.js +0 -588
- package/dist/client/adapters/tts/googleTTSHandler.js +0 -344
- package/dist/client/adapters/video/directorPipeline.js +0 -516
- package/dist/client/adapters/video/ffmpegAdapter.js +0 -206
- package/dist/client/adapters/video/frameExtractor.js +0 -143
- package/dist/client/adapters/video/vertexVideoHandler.js +0 -763
- package/dist/client/adapters/video/videoAnalyzer.js +0 -238
- package/dist/client/adapters/video/videoMerger.js +0 -171
- package/dist/client/agent/directTools.js +0 -840
- package/dist/client/auth/AuthProviderFactory.js +0 -111
- package/dist/client/auth/AuthProviderRegistry.js +0 -190
- package/dist/client/auth/RequestContext.js +0 -78
- package/dist/client/auth/accountPool.js +0 -178
- package/dist/client/auth/anthropicOAuth.js +0 -974
- package/dist/client/auth/authContext.js +0 -314
- package/dist/client/auth/errors.js +0 -39
- package/dist/client/auth/index.js +0 -61
- package/dist/client/auth/middleware/AuthMiddleware.js +0 -519
- package/dist/client/auth/middleware/rateLimitByUser.js +0 -554
- package/dist/client/auth/providers/BaseAuthProvider.js +0 -723
- package/dist/client/auth/providers/CognitoProvider.js +0 -304
- package/dist/client/auth/providers/KeycloakProvider.js +0 -393
- package/dist/client/auth/providers/auth0.js +0 -274
- package/dist/client/auth/providers/betterAuth.js +0 -182
- package/dist/client/auth/providers/clerk.js +0 -317
- package/dist/client/auth/providers/custom.js +0 -112
- package/dist/client/auth/providers/firebase.js +0 -226
- package/dist/client/auth/providers/jwt.js +0 -212
- package/dist/client/auth/providers/oauth2.js +0 -303
- package/dist/client/auth/providers/supabase.js +0 -259
- package/dist/client/auth/providers/workos.js +0 -284
- package/dist/client/auth/serverBridge.js +0 -25
- package/dist/client/auth/sessionManager.js +0 -437
- package/dist/client/auth/tokenStore.js +0 -799
- package/dist/client/client/aiSdkAdapter.js +0 -487
- package/dist/client/client/auth.js +0 -473
- package/dist/client/client/errors.js +0 -552
- package/dist/client/client/httpClient.js +0 -837
- package/dist/client/client/index.js +0 -172
- package/dist/client/client/interceptors.js +0 -601
- package/dist/client/client/sseClient.js +0 -545
- package/dist/client/client/streamingClient.js +0 -917
- package/dist/client/client/wsClient.js +0 -369
- package/dist/client/config/configManager.js +0 -303
- package/dist/client/config/conversationMemory.js +0 -86
- package/dist/client/config/taskClassificationConfig.js +0 -148
- package/dist/client/constants/contextWindows.js +0 -295
- package/dist/client/constants/enums.js +0 -853
- package/dist/client/constants/index.js +0 -207
- package/dist/client/constants/performance.js +0 -389
- package/dist/client/constants/retry.js +0 -266
- package/dist/client/constants/timeouts.js +0 -182
- package/dist/client/constants/tokens.js +0 -380
- package/dist/client/constants/videoErrors.js +0 -46
- package/dist/client/context/budgetChecker.js +0 -98
- package/dist/client/context/contextCompactor.js +0 -205
- package/dist/client/context/emergencyTruncation.js +0 -88
- package/dist/client/context/errorDetection.js +0 -171
- package/dist/client/context/errors.js +0 -21
- package/dist/client/context/fileTokenBudget.js +0 -127
- package/dist/client/context/prompts/summarizationPrompt.js +0 -117
- package/dist/client/context/stages/fileReadDeduplicator.js +0 -66
- package/dist/client/context/stages/slidingWindowTruncator.js +0 -190
- package/dist/client/context/stages/structuredSummarizer.js +0 -99
- package/dist/client/context/stages/toolOutputPruner.js +0 -52
- package/dist/client/context/summarizationEngine.js +0 -136
- package/dist/client/context/toolOutputLimits.js +0 -78
- package/dist/client/context/toolPairRepair.js +0 -66
- package/dist/client/core/analytics.js +0 -88
- package/dist/client/core/baseProvider.js +0 -1385
- package/dist/client/core/constants.js +0 -140
- package/dist/client/core/conversationMemoryFactory.js +0 -141
- package/dist/client/core/conversationMemoryInitializer.js +0 -128
- package/dist/client/core/conversationMemoryManager.js +0 -344
- package/dist/client/core/dynamicModels.js +0 -358
- package/dist/client/core/evaluation.js +0 -309
- package/dist/client/core/evaluationProviders.js +0 -248
- package/dist/client/core/factory.js +0 -412
- package/dist/client/core/infrastructure/baseError.js +0 -22
- package/dist/client/core/infrastructure/baseFactory.js +0 -54
- package/dist/client/core/infrastructure/baseRegistry.js +0 -53
- package/dist/client/core/infrastructure/index.js +0 -5
- package/dist/client/core/infrastructure/retry.js +0 -20
- package/dist/client/core/infrastructure/typedEventEmitter.js +0 -23
- package/dist/client/core/modelConfiguration.js +0 -851
- package/dist/client/core/modules/GenerationHandler.js +0 -588
- package/dist/client/core/modules/MessageBuilder.js +0 -273
- package/dist/client/core/modules/StreamHandler.js +0 -185
- package/dist/client/core/modules/TelemetryHandler.js +0 -203
- package/dist/client/core/modules/ToolsManager.js +0 -499
- package/dist/client/core/modules/Utilities.js +0 -331
- package/dist/client/core/redisConversationMemoryManager.js +0 -1435
- package/dist/client/core/streamAnalytics.js +0 -131
- package/dist/client/evaluation/contextBuilder.js +0 -134
- package/dist/client/evaluation/index.js +0 -61
- package/dist/client/evaluation/prompts.js +0 -73
- package/dist/client/evaluation/ragasEvaluator.js +0 -110
- package/dist/client/evaluation/retryManager.js +0 -78
- package/dist/client/evaluation/scoring.js +0 -61
- package/dist/client/factories/providerFactory.js +0 -166
- package/dist/client/factories/providerRegistry.js +0 -166
- package/dist/client/features/ppt/constants.js +0 -896
- package/dist/client/features/ppt/contentPlanner.js +0 -529
- package/dist/client/features/ppt/presentationOrchestrator.js +0 -236
- package/dist/client/features/ppt/slideGenerator.js +0 -532
- package/dist/client/features/ppt/slideRenderers.js +0 -2383
- package/dist/client/features/ppt/slideTypeInference.js +0 -405
- package/dist/client/features/ppt/types.js +0 -13
- package/dist/client/features/ppt/utils.js +0 -443
- package/dist/client/files/fileReferenceRegistry.js +0 -1543
- package/dist/client/files/fileTools.js +0 -450
- package/dist/client/files/streamingReader.js +0 -321
- package/dist/client/files/types.js +0 -23
- package/dist/client/hitl/hitlErrors.js +0 -54
- package/dist/client/hitl/hitlManager.js +0 -460
- package/dist/client/mcp/agentExposure.js +0 -356
- package/dist/client/mcp/auth/index.js +0 -11
- package/dist/client/mcp/auth/oauthClientProvider.js +0 -325
- package/dist/client/mcp/auth/tokenStorage.js +0 -134
- package/dist/client/mcp/batching/index.js +0 -10
- package/dist/client/mcp/batching/requestBatcher.js +0 -441
- package/dist/client/mcp/caching/index.js +0 -10
- package/dist/client/mcp/caching/toolCache.js +0 -433
- package/dist/client/mcp/elicitation/elicitationManager.js +0 -376
- package/dist/client/mcp/elicitation/index.js +0 -11
- package/dist/client/mcp/elicitation/types.js +0 -10
- package/dist/client/mcp/elicitationProtocol.js +0 -375
- package/dist/client/mcp/enhancedToolDiscovery.js +0 -481
- package/dist/client/mcp/externalServerManager.js +0 -1478
- package/dist/client/mcp/factory.js +0 -161
- package/dist/client/mcp/flexibleToolValidator.js +0 -161
- package/dist/client/mcp/httpRateLimiter.js +0 -391
- package/dist/client/mcp/httpRetryHandler.js +0 -178
- package/dist/client/mcp/index.js +0 -74
- package/dist/client/mcp/mcpCircuitBreaker.js +0 -427
- package/dist/client/mcp/mcpClientFactory.js +0 -708
- package/dist/client/mcp/mcpRegistryClient.js +0 -488
- package/dist/client/mcp/mcpServerBase.js +0 -373
- package/dist/client/mcp/multiServerManager.js +0 -579
- package/dist/client/mcp/registry.js +0 -158
- package/dist/client/mcp/routing/index.js +0 -10
- package/dist/client/mcp/routing/toolRouter.js +0 -416
- package/dist/client/mcp/serverCapabilities.js +0 -502
- package/dist/client/mcp/servers/agent/directToolsServer.js +0 -150
- package/dist/client/mcp/toolAnnotations.js +0 -239
- package/dist/client/mcp/toolConverter.js +0 -258
- package/dist/client/mcp/toolDiscoveryService.js +0 -798
- package/dist/client/mcp/toolIntegration.js +0 -334
- package/dist/client/mcp/toolRegistry.js +0 -729
- package/dist/client/memory/hippocampusInitializer.js +0 -19
- package/dist/client/memory/memoryRetrievalTools.js +0 -166
- package/dist/client/middleware/builtin/analytics.js +0 -132
- package/dist/client/middleware/builtin/autoEvaluation.js +0 -203
- package/dist/client/middleware/builtin/guardrails.js +0 -109
- package/dist/client/middleware/builtin/lifecycle.js +0 -168
- package/dist/client/middleware/factory.js +0 -327
- package/dist/client/middleware/registry.js +0 -295
- package/dist/client/middleware/utils/guardrailsUtils.js +0 -396
- package/dist/client/models/anthropicModels.js +0 -527
- package/dist/client/neurolink.js +0 -8233
- package/dist/client/observability/exporterRegistry.js +0 -413
- package/dist/client/observability/exporters/arizeExporter.js +0 -138
- package/dist/client/observability/exporters/baseExporter.js +0 -190
- package/dist/client/observability/exporters/braintrustExporter.js +0 -154
- package/dist/client/observability/exporters/datadogExporter.js +0 -196
- package/dist/client/observability/exporters/laminarExporter.js +0 -302
- package/dist/client/observability/exporters/langfuseExporter.js +0 -209
- package/dist/client/observability/exporters/langsmithExporter.js +0 -143
- package/dist/client/observability/exporters/otelExporter.js +0 -164
- package/dist/client/observability/exporters/posthogExporter.js +0 -287
- package/dist/client/observability/exporters/sentryExporter.js +0 -165
- package/dist/client/observability/index.js +0 -31
- package/dist/client/observability/metricsAggregator.js +0 -556
- package/dist/client/observability/otelBridge.js +0 -131
- package/dist/client/observability/retryPolicy.js +0 -383
- package/dist/client/observability/sampling/samplers.js +0 -216
- package/dist/client/observability/spanProcessor.js +0 -303
- package/dist/client/observability/tokenTracker.js +0 -413
- package/dist/client/observability/types/exporterTypes.js +0 -5
- package/dist/client/observability/types/index.js +0 -4
- package/dist/client/observability/types/spanTypes.js +0 -92
- package/dist/client/observability/utils/safeMetadata.js +0 -25
- package/dist/client/observability/utils/spanSerializer.js +0 -292
- package/dist/client/processors/archive/ArchiveProcessor.js +0 -1308
- package/dist/client/processors/base/BaseFileProcessor.js +0 -614
- package/dist/client/processors/base/types.js +0 -82
- package/dist/client/processors/config/fileTypes.js +0 -520
- package/dist/client/processors/config/index.js +0 -92
- package/dist/client/processors/config/languageMap.js +0 -410
- package/dist/client/processors/config/mimeTypes.js +0 -363
- package/dist/client/processors/config/sizeLimits.js +0 -258
- package/dist/client/processors/document/ExcelProcessor.js +0 -590
- package/dist/client/processors/document/OpenDocumentProcessor.js +0 -212
- package/dist/client/processors/document/PptxProcessor.js +0 -157
- package/dist/client/processors/document/RtfProcessor.js +0 -361
- package/dist/client/processors/document/WordProcessor.js +0 -353
- package/dist/client/processors/errors/FileErrorCode.js +0 -255
- package/dist/client/processors/errors/errorHelpers.js +0 -386
- package/dist/client/processors/errors/errorSerializer.js +0 -507
- package/dist/client/processors/errors/index.js +0 -49
- package/dist/client/processors/markup/SvgProcessor.js +0 -240
- package/dist/client/processors/media/AudioProcessor.js +0 -707
- package/dist/client/processors/media/VideoProcessor.js +0 -1045
- package/dist/client/providers/amazonBedrock.js +0 -1512
- package/dist/client/providers/amazonSagemaker.js +0 -162
- package/dist/client/providers/anthropic.js +0 -831
- package/dist/client/providers/azureOpenai.js +0 -143
- package/dist/client/providers/googleAiStudio.js +0 -1200
- package/dist/client/providers/googleNativeGemini3.js +0 -543
- package/dist/client/providers/googleVertex.js +0 -2936
- package/dist/client/providers/huggingFace.js +0 -315
- package/dist/client/providers/litellm.js +0 -488
- package/dist/client/providers/mistral.js +0 -157
- package/dist/client/providers/ollama.js +0 -1579
- package/dist/client/providers/openAI.js +0 -627
- package/dist/client/providers/openRouter.js +0 -543
- package/dist/client/providers/openaiCompatible.js +0 -290
- package/dist/client/providers/providerTypeUtils.js +0 -46
- package/dist/client/providers/sagemaker/adaptive-semaphore.js +0 -215
- package/dist/client/providers/sagemaker/client.js +0 -472
- package/dist/client/providers/sagemaker/config.js +0 -317
- package/dist/client/providers/sagemaker/detection.js +0 -606
- package/dist/client/providers/sagemaker/error-constants.js +0 -227
- package/dist/client/providers/sagemaker/errors.js +0 -299
- package/dist/client/providers/sagemaker/language-model.js +0 -775
- package/dist/client/providers/sagemaker/parsers.js +0 -634
- package/dist/client/providers/sagemaker/streaming.js +0 -331
- package/dist/client/providers/sagemaker/structured-parser.js +0 -625
- package/dist/client/proxy/accountQuota.js +0 -162
- package/dist/client/proxy/claudeFormat.js +0 -595
- package/dist/client/proxy/modelRouter.js +0 -29
- package/dist/client/proxy/oauthFetch.js +0 -367
- package/dist/client/proxy/proxyFetch.js +0 -586
- package/dist/client/proxy/requestLogger.js +0 -207
- package/dist/client/proxy/tokenRefresh.js +0 -124
- package/dist/client/proxy/usageStats.js +0 -74
- package/dist/client/proxy/utils/noProxyUtils.js +0 -149
- package/dist/client/rag/ChunkerFactory.js +0 -320
- package/dist/client/rag/ChunkerRegistry.js +0 -421
- package/dist/client/rag/chunkers/BaseChunker.js +0 -143
- package/dist/client/rag/chunkers/CharacterChunker.js +0 -28
- package/dist/client/rag/chunkers/HTMLChunker.js +0 -38
- package/dist/client/rag/chunkers/JSONChunker.js +0 -68
- package/dist/client/rag/chunkers/LaTeXChunker.js +0 -63
- package/dist/client/rag/chunkers/MarkdownChunker.js +0 -306
- package/dist/client/rag/chunkers/RecursiveChunker.js +0 -139
- package/dist/client/rag/chunkers/SemanticMarkdownChunker.js +0 -138
- package/dist/client/rag/chunkers/SentenceChunker.js +0 -66
- package/dist/client/rag/chunkers/TokenChunker.js +0 -61
- package/dist/client/rag/chunkers/index.js +0 -15
- package/dist/client/rag/chunking/characterChunker.js +0 -142
- package/dist/client/rag/chunking/chunkerRegistry.js +0 -194
- package/dist/client/rag/chunking/htmlChunker.js +0 -247
- package/dist/client/rag/chunking/index.js +0 -17
- package/dist/client/rag/chunking/jsonChunker.js +0 -281
- package/dist/client/rag/chunking/latexChunker.js +0 -251
- package/dist/client/rag/chunking/markdownChunker.js +0 -373
- package/dist/client/rag/chunking/recursiveChunker.js +0 -148
- package/dist/client/rag/chunking/semanticChunker.js +0 -306
- package/dist/client/rag/chunking/sentenceChunker.js +0 -230
- package/dist/client/rag/chunking/tokenChunker.js +0 -183
- package/dist/client/rag/document/MDocument.js +0 -392
- package/dist/client/rag/document/index.js +0 -5
- package/dist/client/rag/document/loaders.js +0 -500
- package/dist/client/rag/errors/RAGError.js +0 -274
- package/dist/client/rag/errors/index.js +0 -6
- package/dist/client/rag/graphRag/graphRAG.js +0 -401
- package/dist/client/rag/graphRag/index.js +0 -4
- package/dist/client/rag/index.js +0 -141
- package/dist/client/rag/metadata/MetadataExtractorFactory.js +0 -418
- package/dist/client/rag/metadata/MetadataExtractorRegistry.js +0 -362
- package/dist/client/rag/metadata/index.js +0 -9
- package/dist/client/rag/metadata/metadataExtractor.js +0 -280
- package/dist/client/rag/pipeline/RAGPipeline.js +0 -436
- package/dist/client/rag/pipeline/contextAssembly.js +0 -341
- package/dist/client/rag/pipeline/index.js +0 -5
- package/dist/client/rag/ragIntegration.js +0 -321
- package/dist/client/rag/reranker/RerankerFactory.js +0 -430
- package/dist/client/rag/reranker/RerankerRegistry.js +0 -402
- package/dist/client/rag/reranker/index.js +0 -9
- package/dist/client/rag/reranker/reranker.js +0 -277
- package/dist/client/rag/resilience/CircuitBreaker.js +0 -431
- package/dist/client/rag/resilience/RetryHandler.js +0 -304
- package/dist/client/rag/resilience/index.js +0 -7
- package/dist/client/rag/retrieval/hybridSearch.js +0 -335
- package/dist/client/rag/retrieval/index.js +0 -5
- package/dist/client/rag/retrieval/vectorQueryTool.js +0 -307
- package/dist/client/rag/types.js +0 -8
- package/dist/client/sdk/toolRegistration.js +0 -377
- package/dist/client/server/abstract/baseServerAdapter.js +0 -575
- package/dist/client/server/adapters/expressAdapter.js +0 -486
- package/dist/client/server/adapters/fastifyAdapter.js +0 -472
- package/dist/client/server/adapters/honoAdapter.js +0 -632
- package/dist/client/server/adapters/koaAdapter.js +0 -510
- package/dist/client/server/errors.js +0 -486
- package/dist/client/server/factory/serverAdapterFactory.js +0 -160
- package/dist/client/server/index.js +0 -108
- package/dist/client/server/middleware/abortSignal.js +0 -111
- package/dist/client/server/middleware/auth.js +0 -388
- package/dist/client/server/middleware/cache.js +0 -359
- package/dist/client/server/middleware/common.js +0 -281
- package/dist/client/server/middleware/deprecation.js +0 -190
- package/dist/client/server/middleware/mcpBodyAttachment.js +0 -63
- package/dist/client/server/middleware/rateLimit.js +0 -227
- package/dist/client/server/middleware/validation.js +0 -388
- package/dist/client/server/openapi/generator.js +0 -398
- package/dist/client/server/openapi/index.js +0 -36
- package/dist/client/server/openapi/schemas.js +0 -695
- package/dist/client/server/openapi/templates.js +0 -374
- package/dist/client/server/routes/agentRoutes.js +0 -189
- package/dist/client/server/routes/claudeProxyRoutes.js +0 -1600
- package/dist/client/server/routes/healthRoutes.js +0 -187
- package/dist/client/server/routes/index.js +0 -57
- package/dist/client/server/routes/mcpRoutes.js +0 -342
- package/dist/client/server/routes/memoryRoutes.js +0 -350
- package/dist/client/server/routes/openApiRoutes.js +0 -126
- package/dist/client/server/routes/toolRoutes.js +0 -199
- package/dist/client/server/streaming/dataStream.js +0 -486
- package/dist/client/server/streaming/index.js +0 -11
- package/dist/client/server/types.js +0 -67
- package/dist/client/server/utils/redaction.js +0 -334
- package/dist/client/server/utils/validation.js +0 -243
- package/dist/client/server/websocket/WebSocketHandler.js +0 -383
- package/dist/client/server/websocket/index.js +0 -4
- package/dist/client/services/server/ai/observability/instrumentation.js +0 -808
- package/dist/client/telemetry/attributes.js +0 -100
- package/dist/client/telemetry/index.js +0 -26
- package/dist/client/telemetry/telemetryService.js +0 -308
- package/dist/client/telemetry/tracers.js +0 -17
- package/dist/client/telemetry/withSpan.js +0 -34
- package/dist/client/types/actionTypes.js +0 -6
- package/dist/client/types/analytics.js +0 -5
- package/dist/client/types/authTypes.js +0 -9
- package/dist/client/types/circuitBreakerErrors.js +0 -34
- package/dist/client/types/cli.js +0 -21
- package/dist/client/types/clientTypes.js +0 -10
- package/dist/client/types/common.js +0 -51
- package/dist/client/types/configTypes.js +0 -49
- package/dist/client/types/content.js +0 -19
- package/dist/client/types/contextTypes.js +0 -400
- package/dist/client/types/conversation.js +0 -47
- package/dist/client/types/conversationMemoryInterface.js +0 -6
- package/dist/client/types/domainTypes.js +0 -5
- package/dist/client/types/errors.js +0 -167
- package/dist/client/types/evaluation.js +0 -5
- package/dist/client/types/evaluationProviders.js +0 -5
- package/dist/client/types/evaluationTypes.js +0 -1
- package/dist/client/types/externalMcp.js +0 -6
- package/dist/client/types/fileReferenceTypes.js +0 -8
- package/dist/client/types/fileTypes.js +0 -4
- package/dist/client/types/generateTypes.js +0 -1
- package/dist/client/types/guardrails.js +0 -1
- package/dist/client/types/hitlTypes.js +0 -8
- package/dist/client/types/index.js +0 -57
- package/dist/client/types/mcpTypes.js +0 -5
- package/dist/client/types/middlewareTypes.js +0 -1
- package/dist/client/types/modelTypes.js +0 -30
- package/dist/client/types/multimodal.js +0 -135
- package/dist/client/types/observability.js +0 -6
- package/dist/client/types/pptTypes.js +0 -82
- package/dist/client/types/providers.js +0 -111
- package/dist/client/types/proxyTypes.js +0 -16
- package/dist/client/types/ragTypes.js +0 -7
- package/dist/client/types/sdkTypes.js +0 -8
- package/dist/client/types/serviceTypes.js +0 -5
- package/dist/client/types/streamTypes.js +0 -1
- package/dist/client/types/subscriptionTypes.js +0 -9
- package/dist/client/types/taskClassificationTypes.js +0 -5
- package/dist/client/types/tools.js +0 -24
- package/dist/client/types/ttsTypes.js +0 -57
- package/dist/client/types/typeAliases.js +0 -48
- package/dist/client/types/utilities.js +0 -4
- package/dist/client/types/workflowTypes.js +0 -30
- package/dist/client/utils/async/withTimeout.js +0 -98
- package/dist/client/utils/asyncMutex.js +0 -60
- package/dist/client/utils/conversationMemory.js +0 -431
- package/dist/client/utils/csvProcessor.js +0 -846
- package/dist/client/utils/errorHandling.js +0 -936
- package/dist/client/utils/evaluationUtils.js +0 -131
- package/dist/client/utils/factoryProcessing.js +0 -589
- package/dist/client/utils/fileDetector.js +0 -2161
- package/dist/client/utils/imageCache.js +0 -376
- package/dist/client/utils/imageProcessor.js +0 -704
- package/dist/client/utils/logger.js +0 -491
- package/dist/client/utils/mcpDefaults.js +0 -134
- package/dist/client/utils/messageBuilder.js +0 -1653
- package/dist/client/utils/modelAliasResolver.js +0 -54
- package/dist/client/utils/modelDetection.js +0 -80
- package/dist/client/utils/modelRouter.js +0 -292
- package/dist/client/utils/multimodalOptionsBuilder.js +0 -65
- package/dist/client/utils/observabilityHelpers.js +0 -47
- package/dist/client/utils/parameterValidation.js +0 -966
- package/dist/client/utils/pdfProcessor.js +0 -410
- package/dist/client/utils/performance.js +0 -222
- package/dist/client/utils/pricing.js +0 -340
- package/dist/client/utils/promptRedaction.js +0 -62
- package/dist/client/utils/providerConfig.js +0 -1009
- package/dist/client/utils/providerHealth.js +0 -1237
- package/dist/client/utils/providerRetry.js +0 -112
- package/dist/client/utils/providerUtils.js +0 -434
- package/dist/client/utils/rateLimiter.js +0 -200
- package/dist/client/utils/redis.js +0 -368
- package/dist/client/utils/retryHandler.js +0 -269
- package/dist/client/utils/retryability.js +0 -22
- package/dist/client/utils/sanitizers/svg.js +0 -481
- package/dist/client/utils/schemaConversion.js +0 -255
- package/dist/client/utils/taskClassificationUtils.js +0 -149
- package/dist/client/utils/taskClassifier.js +0 -94
- package/dist/client/utils/thinkingConfig.js +0 -104
- package/dist/client/utils/timeout.js +0 -359
- package/dist/client/utils/tokenEstimation.js +0 -142
- package/dist/client/utils/tokenLimits.js +0 -125
- package/dist/client/utils/tokenUtils.js +0 -239
- package/dist/client/utils/toolUtils.js +0 -75
- package/dist/client/utils/transformationUtils.js +0 -554
- package/dist/client/utils/ttsProcessor.js +0 -286
- package/dist/client/utils/typeUtils.js +0 -97
- package/dist/client/utils/videoAnalysisProcessor.js +0 -67
- package/dist/client/workflow/config.js +0 -398
- package/dist/client/workflow/core/ensembleExecutor.js +0 -407
- package/dist/client/workflow/core/judgeScorer.js +0 -544
- package/dist/client/workflow/core/responseConditioner.js +0 -225
- package/dist/client/workflow/core/types/conditionerTypes.js +0 -7
- package/dist/client/workflow/core/types/ensembleTypes.js +0 -7
- package/dist/client/workflow/core/types/index.js +0 -7
- package/dist/client/workflow/core/types/judgeTypes.js +0 -7
- package/dist/client/workflow/core/types/layerTypes.js +0 -7
- package/dist/client/workflow/core/types/registryTypes.js +0 -7
- package/dist/client/workflow/core/workflowRegistry.js +0 -304
- package/dist/client/workflow/core/workflowRunner.js +0 -586
- package/dist/client/workflow/index.js +0 -50
- package/dist/client/workflow/types.js +0 -9
- package/dist/client/workflow/utils/types/index.js +0 -7
- package/dist/client/workflow/utils/workflowMetrics.js +0 -311
- package/dist/client/workflow/utils/workflowValidation.js +0 -420
- package/dist/client/workflow/workflows/adaptiveWorkflow.js +0 -366
- package/dist/client/workflow/workflows/consensusWorkflow.js +0 -192
- package/dist/client/workflow/workflows/fallbackWorkflow.js +0 -225
- package/dist/client/workflow/workflows/multiJudgeWorkflow.js +0 -351
- /package/dist/client/{client/reactHooks.js → reactHooks.js} +0 -0
|
@@ -1,131 +0,0 @@
|
|
|
1
|
-
import { createAnalytics } from "./analytics.js";
|
|
2
|
-
import { logger } from "../utils/logger.js";
|
|
3
|
-
import { extractTokenUsage, createEmptyTokenUsage, } from "../utils/tokenUtils.js";
|
|
4
|
-
import { NoOutputGeneratedError } from "ai";
|
|
5
|
-
/**
|
|
6
|
-
* Base implementation for collecting analytics from Vercel AI SDK stream results
|
|
7
|
-
*/
|
|
8
|
-
export class BaseStreamAnalyticsCollector {
|
|
9
|
-
/**
|
|
10
|
-
* Collect token usage from stream result
|
|
11
|
-
* Uses centralized tokenUtils for consistent extraction across providers
|
|
12
|
-
*/
|
|
13
|
-
async collectUsage(result) {
|
|
14
|
-
try {
|
|
15
|
-
const usage = await result.usage;
|
|
16
|
-
if (!usage) {
|
|
17
|
-
logger.debug("No usage data available from stream result");
|
|
18
|
-
return createEmptyTokenUsage();
|
|
19
|
-
}
|
|
20
|
-
// Use centralized token extraction utility
|
|
21
|
-
// Handles multiple provider formats, cache tokens, reasoning tokens,
|
|
22
|
-
// and cache savings calculation
|
|
23
|
-
return extractTokenUsage(usage);
|
|
24
|
-
}
|
|
25
|
-
catch (error) {
|
|
26
|
-
if (NoOutputGeneratedError.isInstance(error)) {
|
|
27
|
-
logger.debug("No output generated from stream — returning empty usage");
|
|
28
|
-
}
|
|
29
|
-
else {
|
|
30
|
-
logger.warn("Failed to collect usage from stream result", { error });
|
|
31
|
-
}
|
|
32
|
-
return createEmptyTokenUsage();
|
|
33
|
-
}
|
|
34
|
-
}
|
|
35
|
-
/**
|
|
36
|
-
* Collect response metadata from stream result
|
|
37
|
-
*/
|
|
38
|
-
async collectMetadata(result) {
|
|
39
|
-
try {
|
|
40
|
-
const [response, finishReason] = await Promise.all([
|
|
41
|
-
result.response,
|
|
42
|
-
result.finishReason,
|
|
43
|
-
]);
|
|
44
|
-
return {
|
|
45
|
-
id: response?.id,
|
|
46
|
-
model: response?.model,
|
|
47
|
-
timestamp: response?.timestamp instanceof Date
|
|
48
|
-
? response.timestamp.getTime()
|
|
49
|
-
: response?.timestamp || Date.now(),
|
|
50
|
-
finishReason: finishReason,
|
|
51
|
-
};
|
|
52
|
-
}
|
|
53
|
-
catch (error) {
|
|
54
|
-
if (NoOutputGeneratedError.isInstance(error)) {
|
|
55
|
-
logger.debug("No output generated from stream — returning default metadata");
|
|
56
|
-
}
|
|
57
|
-
else {
|
|
58
|
-
logger.warn("Failed to collect metadata from stream result", {
|
|
59
|
-
error,
|
|
60
|
-
});
|
|
61
|
-
}
|
|
62
|
-
return {
|
|
63
|
-
timestamp: Date.now(),
|
|
64
|
-
finishReason: "error",
|
|
65
|
-
};
|
|
66
|
-
}
|
|
67
|
-
}
|
|
68
|
-
/**
|
|
69
|
-
* Create comprehensive analytics from stream result
|
|
70
|
-
*/
|
|
71
|
-
async createAnalytics(provider, model, result, responseTime, metadata) {
|
|
72
|
-
try {
|
|
73
|
-
// Collect analytics data in parallel
|
|
74
|
-
const [usage, responseMetadata] = await Promise.all([
|
|
75
|
-
this.collectUsage(result),
|
|
76
|
-
this.collectMetadata(result),
|
|
77
|
-
]);
|
|
78
|
-
// Get final text content and finish reason.
|
|
79
|
-
// Guard each promise individually: AI SDK v6 rejects all of these with
|
|
80
|
-
// NoOutputGeneratedError when the stream produced no output.
|
|
81
|
-
const [content, finishReason, toolResults, toolCalls] = await Promise.all([
|
|
82
|
-
Promise.resolve(result.text).catch(() => ""),
|
|
83
|
-
Promise.resolve(result.finishReason).catch(() => "error"),
|
|
84
|
-
Promise.resolve(result.toolResults || []).catch(() => []),
|
|
85
|
-
Promise.resolve(result.toolCalls || []).catch(() => []),
|
|
86
|
-
]);
|
|
87
|
-
// Create comprehensive analytics
|
|
88
|
-
return createAnalytics(provider, model, {
|
|
89
|
-
usage,
|
|
90
|
-
content,
|
|
91
|
-
response: responseMetadata,
|
|
92
|
-
finishReason: finishReason,
|
|
93
|
-
toolResults: toolResults,
|
|
94
|
-
toolCalls: toolCalls,
|
|
95
|
-
}, responseTime, {
|
|
96
|
-
...metadata,
|
|
97
|
-
streamingMode: true,
|
|
98
|
-
responseId: responseMetadata.id,
|
|
99
|
-
finishReason: finishReason,
|
|
100
|
-
});
|
|
101
|
-
}
|
|
102
|
-
catch (error) {
|
|
103
|
-
logger.error("Failed to create analytics from stream result", {
|
|
104
|
-
provider,
|
|
105
|
-
model,
|
|
106
|
-
error: error instanceof Error ? error.message : String(error),
|
|
107
|
-
});
|
|
108
|
-
// Return minimal analytics on error
|
|
109
|
-
return createAnalytics(provider, model, { usage: { input: 0, output: 0, total: 0 } }, responseTime, {
|
|
110
|
-
...metadata,
|
|
111
|
-
streamingMode: true,
|
|
112
|
-
analyticsError: true,
|
|
113
|
-
});
|
|
114
|
-
}
|
|
115
|
-
}
|
|
116
|
-
/**
|
|
117
|
-
* Clean up resources and force garbage collection if needed
|
|
118
|
-
*/
|
|
119
|
-
cleanup() {
|
|
120
|
-
// Only force garbage collection if memory usage exceeds 500 MB
|
|
121
|
-
const heapUsed = process.memoryUsage().heapUsed;
|
|
122
|
-
const GC_THRESHOLD = 500 * 1024 * 1024; // 500 MB
|
|
123
|
-
if (typeof global !== "undefined" && global.gc && heapUsed > GC_THRESHOLD) {
|
|
124
|
-
global.gc();
|
|
125
|
-
}
|
|
126
|
-
}
|
|
127
|
-
}
|
|
128
|
-
/**
|
|
129
|
-
* Global instance of stream analytics collector
|
|
130
|
-
*/
|
|
131
|
-
export const streamAnalyticsCollector = new BaseStreamAnalyticsCollector();
|
|
@@ -1,134 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @file Implements the ContextBuilder class for creating rich evaluation context.
|
|
3
|
-
*/
|
|
4
|
-
import { logger } from "../utils/logger.js";
|
|
5
|
-
/**
|
|
6
|
-
* Builds the enhanced context required for a RAGAS-style evaluation.
|
|
7
|
-
* This class gathers data from the generation options and results to create a
|
|
8
|
-
* rich snapshot of the interaction, which is then used by the evaluator.
|
|
9
|
-
*/
|
|
10
|
-
export class ContextBuilder {
|
|
11
|
-
attemptNumber = 1;
|
|
12
|
-
previousEvaluations = [];
|
|
13
|
-
extractTextFromContent(content) {
|
|
14
|
-
if (typeof content === "string") {
|
|
15
|
-
return content;
|
|
16
|
-
}
|
|
17
|
-
if (Array.isArray(content)) {
|
|
18
|
-
return content
|
|
19
|
-
.filter((part) => part.type === "text" && "text" in part)
|
|
20
|
-
.map((part) => part.text)
|
|
21
|
-
.join("");
|
|
22
|
-
}
|
|
23
|
-
return "";
|
|
24
|
-
}
|
|
25
|
-
/**
|
|
26
|
-
* Builds the full evaluation context for a single evaluation attempt.
|
|
27
|
-
*
|
|
28
|
-
* @param options The original `TextGenerationOptions` used for the request.
|
|
29
|
-
* @param result The `GenerateResult` from the provider.
|
|
30
|
-
* @returns An `EnhancedEvaluationContext` object ready for evaluation.
|
|
31
|
-
*/
|
|
32
|
-
buildContext(options, result) {
|
|
33
|
-
const userMessages = options.prompt.filter((p) => p.role === "user");
|
|
34
|
-
const lastUserMessage = userMessages[userMessages.length - 1];
|
|
35
|
-
const userQuery = this.extractTextFromContent(lastUserMessage?.content ?? "");
|
|
36
|
-
const systemPromptMessage = options.prompt.find((p) => p.role === "system");
|
|
37
|
-
const systemPrompt = this.extractTextFromContent(systemPromptMessage?.content ?? "");
|
|
38
|
-
const queryAnalysis = this.analyzeQuery(userQuery);
|
|
39
|
-
const toolExecutions = this.mapToolExecutions(result);
|
|
40
|
-
const context = {
|
|
41
|
-
userQuery,
|
|
42
|
-
queryAnalysis,
|
|
43
|
-
aiResponse: result.content,
|
|
44
|
-
provider: result.provider || "unknown",
|
|
45
|
-
model: result.model || "unknown",
|
|
46
|
-
generationParams: {
|
|
47
|
-
temperature: options.temperature,
|
|
48
|
-
maxTokens: options.maxOutputTokens,
|
|
49
|
-
systemPrompt: systemPrompt || undefined,
|
|
50
|
-
},
|
|
51
|
-
toolExecutions,
|
|
52
|
-
conversationHistory: (options.prompt || [])
|
|
53
|
-
.filter((p) => p.role !== "system")
|
|
54
|
-
.map((turn) => ({
|
|
55
|
-
role: turn.role,
|
|
56
|
-
content: this.extractTextFromContent(turn.content),
|
|
57
|
-
timestamp: new Date().toISOString(),
|
|
58
|
-
})),
|
|
59
|
-
responseTime: result.responseTime || 0,
|
|
60
|
-
tokenUsage: result.usage || { input: 0, output: 0, total: 0 },
|
|
61
|
-
previousEvaluations: this.previousEvaluations,
|
|
62
|
-
attemptNumber: this.attemptNumber,
|
|
63
|
-
};
|
|
64
|
-
logger.debug("Built Evaluation Context:", context);
|
|
65
|
-
return context;
|
|
66
|
-
}
|
|
67
|
-
/**
|
|
68
|
-
* Records the result of an evaluation and increments the internal attempt counter.
|
|
69
|
-
* This is used to build up the `previousEvaluations` array for subsequent retries.
|
|
70
|
-
*
|
|
71
|
-
* @param evaluation The `EvaluationResult` from the last attempt.
|
|
72
|
-
*/
|
|
73
|
-
recordEvaluation(evaluation) {
|
|
74
|
-
this.previousEvaluations.push(evaluation);
|
|
75
|
-
this.attemptNumber++;
|
|
76
|
-
}
|
|
77
|
-
/**
|
|
78
|
-
* Resets the internal state of the context builder. This should be called
|
|
79
|
-
* before starting a new, independent evaluation sequence.
|
|
80
|
-
*/
|
|
81
|
-
reset() {
|
|
82
|
-
this.attemptNumber = 1;
|
|
83
|
-
this.previousEvaluations = [];
|
|
84
|
-
}
|
|
85
|
-
/**
|
|
86
|
-
* Analyzes the user's query to determine intent and complexity.
|
|
87
|
-
* @param query The user's input query.
|
|
88
|
-
* @returns A QueryIntentAnalysis object.
|
|
89
|
-
*/
|
|
90
|
-
analyzeQuery(query) {
|
|
91
|
-
const lowerCaseQuery = query.toLowerCase();
|
|
92
|
-
let type;
|
|
93
|
-
if (lowerCaseQuery.startsWith("what") ||
|
|
94
|
-
lowerCaseQuery.startsWith("how") ||
|
|
95
|
-
lowerCaseQuery.startsWith("why")) {
|
|
96
|
-
type = "question";
|
|
97
|
-
}
|
|
98
|
-
else if (lowerCaseQuery.length < 20) {
|
|
99
|
-
type = "greeting";
|
|
100
|
-
}
|
|
101
|
-
else {
|
|
102
|
-
type = "command";
|
|
103
|
-
}
|
|
104
|
-
const complexity = query.length > 100 ? "high" : query.length > 40 ? "medium" : "low";
|
|
105
|
-
return {
|
|
106
|
-
type,
|
|
107
|
-
complexity,
|
|
108
|
-
shouldHaveUsedTools: false, // This would require deeper analysis
|
|
109
|
-
};
|
|
110
|
-
}
|
|
111
|
-
/**
|
|
112
|
-
* Maps the tool execution format from GenerateResult to the canonical ToolExecution type.
|
|
113
|
-
* @param result The result from the generate call.
|
|
114
|
-
* @returns An array of ToolExecution objects.
|
|
115
|
-
*/
|
|
116
|
-
mapToolExecutions(result) {
|
|
117
|
-
if (!result.toolExecutions) {
|
|
118
|
-
return [];
|
|
119
|
-
}
|
|
120
|
-
return result.toolExecutions.map((exec) => {
|
|
121
|
-
const toolResult = {
|
|
122
|
-
success: true,
|
|
123
|
-
data: exec.output,
|
|
124
|
-
};
|
|
125
|
-
return {
|
|
126
|
-
toolName: exec.name,
|
|
127
|
-
params: exec.input,
|
|
128
|
-
result: toolResult,
|
|
129
|
-
executionTime: 0,
|
|
130
|
-
timestamp: Date.now(),
|
|
131
|
-
};
|
|
132
|
-
});
|
|
133
|
-
}
|
|
134
|
-
}
|
|
@@ -1,61 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @file This file exports the main Evaluator class, which serves as the central entry point for the evaluation system.
|
|
3
|
-
*/
|
|
4
|
-
import { ContextBuilder } from "./contextBuilder.js";
|
|
5
|
-
import { RAGASEvaluator } from "./ragasEvaluator.js";
|
|
6
|
-
import { mapToEvaluationData } from "./scoring.js";
|
|
7
|
-
/**
|
|
8
|
-
* A centralized class for performing response evaluations. It supports different
|
|
9
|
-
* evaluation strategies, with RAGAS-style model-based evaluation as the default.
|
|
10
|
-
* This class orchestrates the context building and evaluation process.
|
|
11
|
-
*/
|
|
12
|
-
export class Evaluator {
|
|
13
|
-
contextBuilder;
|
|
14
|
-
config;
|
|
15
|
-
ragasEvaluator;
|
|
16
|
-
constructor(config = {}) {
|
|
17
|
-
this.config = config;
|
|
18
|
-
this.contextBuilder = new ContextBuilder();
|
|
19
|
-
this.ragasEvaluator = new RAGASEvaluator(this.config.evaluationModel, this.config.provider, this.config.threshold, this.config.promptGenerator);
|
|
20
|
-
}
|
|
21
|
-
/**
|
|
22
|
-
* The main entry point for performing an evaluation. It selects the evaluation
|
|
23
|
-
* strategy based on the configuration and executes it.
|
|
24
|
-
*
|
|
25
|
-
* @param options The original `TextGenerationOptions` from the user request.
|
|
26
|
-
* @param result The `GenerateResult` from the provider.
|
|
27
|
-
* @returns A promise that resolves to the `EvaluationResult`.
|
|
28
|
-
*/
|
|
29
|
-
async evaluate(options, result, threshold, config) {
|
|
30
|
-
const evaluationStrategy = this.config.evaluationStrategy || "ragas";
|
|
31
|
-
const customEvaluator = this.config.customEvaluator;
|
|
32
|
-
switch (evaluationStrategy) {
|
|
33
|
-
case "ragas": {
|
|
34
|
-
const { evaluationResult, evalContext } = await this.evaluateWithRAGAS(options, result);
|
|
35
|
-
const evaluationData = mapToEvaluationData(evalContext, evaluationResult, threshold, config.offTopicThreshold, config.highSeverityThreshold);
|
|
36
|
-
return evaluationData;
|
|
37
|
-
}
|
|
38
|
-
case "custom": {
|
|
39
|
-
if (customEvaluator) {
|
|
40
|
-
const { evaluationResult, evalContext } = await customEvaluator(options, result);
|
|
41
|
-
return mapToEvaluationData(evalContext, evaluationResult, threshold, config.offTopicThreshold, config.highSeverityThreshold);
|
|
42
|
-
}
|
|
43
|
-
throw new Error("Custom evaluator function not provided in config.");
|
|
44
|
-
}
|
|
45
|
-
default:
|
|
46
|
-
throw new Error(`Unsupported evaluation strategy: ${evaluationStrategy} `);
|
|
47
|
-
}
|
|
48
|
-
}
|
|
49
|
-
/**
|
|
50
|
-
* Performs evaluation using the RAGAS-style model-based evaluator.
|
|
51
|
-
*
|
|
52
|
-
* @param options The original `TextGenerationOptions`.
|
|
53
|
-
* @param result The `GenerateResult` to be evaluated.
|
|
54
|
-
* @returns A promise that resolves to the `EvaluationResult`.
|
|
55
|
-
*/
|
|
56
|
-
async evaluateWithRAGAS(options, result) {
|
|
57
|
-
const evalContext = this.contextBuilder.buildContext(options, result);
|
|
58
|
-
const evaluationResult = await this.ragasEvaluator.evaluate(evalContext);
|
|
59
|
-
return { evaluationResult, evalContext };
|
|
60
|
-
}
|
|
61
|
-
}
|
|
@@ -1,73 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* A flexible class for building evaluation prompts. It allows for custom prompt
|
|
3
|
-
* generation logic to be injected while ensuring a consistent output format.
|
|
4
|
-
*/
|
|
5
|
-
export class PromptBuilder {
|
|
6
|
-
/**
|
|
7
|
-
* Builds the final evaluation prompt.
|
|
8
|
-
*
|
|
9
|
-
* @param context The rich context for the evaluation.
|
|
10
|
-
* @param getPrompt An optional function to generate the main body of the prompt.
|
|
11
|
-
* If not provided, a default prompt is used.
|
|
12
|
-
* @returns The complete prompt string to be sent to the judge LLM.
|
|
13
|
-
*/
|
|
14
|
-
buildEvaluationPrompt(context, getPrompt) {
|
|
15
|
-
const { userQuery, aiResponse, conversationHistory, toolExecutions, previousEvaluations, } = context;
|
|
16
|
-
const historyStr = conversationHistory
|
|
17
|
-
.map((turn) => `${turn.role}: ${turn.content}`)
|
|
18
|
-
.join("\n");
|
|
19
|
-
const toolsStr = toolExecutions.length > 0
|
|
20
|
-
? `Tools were used: ${toolExecutions.map((t) => t.toolName).join(", ")}`
|
|
21
|
-
: "No tools were used.";
|
|
22
|
-
const retryStr = previousEvaluations && previousEvaluations.length > 0
|
|
23
|
-
? `This is attempt #${context.attemptNumber}. Previous reasoning: ${previousEvaluations
|
|
24
|
-
.map((e) => e.reasoning)
|
|
25
|
-
.join("; ")} Previous suggested Improvements: ${previousEvaluations.map((e) => e.suggestedImprovements).join("; ")}`
|
|
26
|
-
: "This is the first attempt.";
|
|
27
|
-
const promptContext = {
|
|
28
|
-
userQuery,
|
|
29
|
-
history: historyStr,
|
|
30
|
-
tools: toolsStr,
|
|
31
|
-
retryInfo: retryStr,
|
|
32
|
-
aiResponse,
|
|
33
|
-
};
|
|
34
|
-
const mainPrompt = getPrompt
|
|
35
|
-
? getPrompt(promptContext)
|
|
36
|
-
: this.getDefaultPrompt(promptContext);
|
|
37
|
-
return `
|
|
38
|
-
${mainPrompt}
|
|
39
|
-
|
|
40
|
-
**Output Format (JSON):**
|
|
41
|
-
{
|
|
42
|
-
"relevanceScore": <1-10>,
|
|
43
|
-
"accuracyScore": <1-10>,
|
|
44
|
-
"completenessScore": <1-10>,
|
|
45
|
-
"finalScore": <1-10>,
|
|
46
|
-
"reasoning": "<Your constructive reasoning here>",
|
|
47
|
-
"suggestedImprovements": "<How the response can be improved>"
|
|
48
|
-
}
|
|
49
|
-
`;
|
|
50
|
-
}
|
|
51
|
-
/**
|
|
52
|
-
* The default prompt generation logic.
|
|
53
|
-
* @param context The prepared context strings.
|
|
54
|
-
* @returns The default prompt body.
|
|
55
|
-
*/
|
|
56
|
-
getDefaultPrompt(context) {
|
|
57
|
-
return `
|
|
58
|
-
You are an expert AI quality evaluator. Your task is to evaluate the AI assistant's response based on the provided context.
|
|
59
|
-
Provide a score from 1 to 10 for each of the following criteria: Relevance, Accuracy, and Completeness.
|
|
60
|
-
Finally, provide an overall finalScore and constructive feedback for improvement.
|
|
61
|
-
|
|
62
|
-
**Evaluation Context:**
|
|
63
|
-
- User Query: ${context.userQuery}
|
|
64
|
-
- Conversation History:
|
|
65
|
-
${context.history}
|
|
66
|
-
- Tools Executed: ${context.tools}
|
|
67
|
-
- Retry Information: ${context.retryInfo}
|
|
68
|
-
|
|
69
|
-
**AI Assistant's Response to Evaluate:**
|
|
70
|
-
${context.aiResponse}
|
|
71
|
-
`;
|
|
72
|
-
}
|
|
73
|
-
}
|
|
@@ -1,110 +0,0 @@
|
|
|
1
|
-
import { AIProviderFactory } from "../core/factory.js";
|
|
2
|
-
import { PromptBuilder } from "./prompts.js";
|
|
3
|
-
import { logger } from "../utils/logger.js";
|
|
4
|
-
import { SpanSerializer, SpanType, SpanStatus, } from "../observability/index.js";
|
|
5
|
-
import { getMetricsAggregator } from "../observability/index.js";
|
|
6
|
-
/**
|
|
7
|
-
* Implements a RAGAS-style evaluator that uses a "judge" LLM to score the
|
|
8
|
-
* quality of an AI response based on rich, contextual information.
|
|
9
|
-
*/
|
|
10
|
-
export class RAGASEvaluator {
|
|
11
|
-
evaluationModel;
|
|
12
|
-
providerName;
|
|
13
|
-
threshold;
|
|
14
|
-
promptBuilder;
|
|
15
|
-
promptGenerator;
|
|
16
|
-
constructor(evaluationModel, providerName, threshold, promptGenerator) {
|
|
17
|
-
this.evaluationModel =
|
|
18
|
-
evaluationModel ||
|
|
19
|
-
process.env.NEUROLINK_RAGAS_EVALUATION_MODEL ||
|
|
20
|
-
"gemini-1.5-flash";
|
|
21
|
-
this.providerName =
|
|
22
|
-
providerName ||
|
|
23
|
-
process.env.NEUROLINK_RAGAS_EVALUATION_PROVIDER ||
|
|
24
|
-
"vertex";
|
|
25
|
-
this.threshold =
|
|
26
|
-
threshold || Number(process.env.NEUROLINK_EVALUATION_THRESHOLD) || 7;
|
|
27
|
-
this.promptBuilder = new PromptBuilder();
|
|
28
|
-
this.promptGenerator = promptGenerator;
|
|
29
|
-
}
|
|
30
|
-
/**
|
|
31
|
-
* Evaluates an AI-generated response using a model-based approach.
|
|
32
|
-
*
|
|
33
|
-
* @param context The rich, contextual information for the evaluation.
|
|
34
|
-
* @returns A promise that resolves to a detailed `EvaluationResult`.
|
|
35
|
-
*/
|
|
36
|
-
async evaluate(context) {
|
|
37
|
-
const span = SpanSerializer.createSpan(SpanType.EVALUATION, "evaluation.ragas", {
|
|
38
|
-
"evaluation.dimension": "relevance|accuracy|completeness",
|
|
39
|
-
"ai.provider": this.providerName,
|
|
40
|
-
"ai.model": this.evaluationModel,
|
|
41
|
-
});
|
|
42
|
-
const startTime = Date.now();
|
|
43
|
-
try {
|
|
44
|
-
const prompt = this.promptBuilder.buildEvaluationPrompt(context, this.promptGenerator);
|
|
45
|
-
const provider = await AIProviderFactory.createProvider(this.providerName, this.evaluationModel);
|
|
46
|
-
const result = await provider.generate({
|
|
47
|
-
input: { text: prompt },
|
|
48
|
-
});
|
|
49
|
-
if (!result) {
|
|
50
|
-
throw new Error("Evaluation generation failed to return a result.");
|
|
51
|
-
}
|
|
52
|
-
const rawEvaluationResponse = result.content;
|
|
53
|
-
const parsedResult = this.parseEvaluationResponse(rawEvaluationResponse);
|
|
54
|
-
const evaluationTime = Date.now() - startTime;
|
|
55
|
-
const finalResult = {
|
|
56
|
-
...parsedResult,
|
|
57
|
-
isPassing: parsedResult.finalScore >= this.threshold, // This will be recalculated, but is needed for the type
|
|
58
|
-
evaluationModel: this.evaluationModel,
|
|
59
|
-
evaluationTime,
|
|
60
|
-
attemptNumber: context.attemptNumber,
|
|
61
|
-
rawEvaluationResponse,
|
|
62
|
-
};
|
|
63
|
-
span.durationMs = Date.now() - startTime;
|
|
64
|
-
const endedSpan = SpanSerializer.endSpan(span, SpanStatus.OK);
|
|
65
|
-
getMetricsAggregator().recordSpan(endedSpan);
|
|
66
|
-
return finalResult;
|
|
67
|
-
}
|
|
68
|
-
catch (error) {
|
|
69
|
-
span.durationMs = Date.now() - startTime;
|
|
70
|
-
const endedSpan = SpanSerializer.endSpan(span, SpanStatus.ERROR);
|
|
71
|
-
endedSpan.statusMessage =
|
|
72
|
-
error instanceof Error ? error.message : String(error);
|
|
73
|
-
getMetricsAggregator().recordSpan(endedSpan);
|
|
74
|
-
throw error;
|
|
75
|
-
}
|
|
76
|
-
}
|
|
77
|
-
/**
|
|
78
|
-
* Parses the raw JSON string from the judge LLM into a structured `EvaluationResult` object.
|
|
79
|
-
* It includes error handling to gracefully manage malformed JSON.
|
|
80
|
-
*
|
|
81
|
-
* @param rawResponse The raw string response from the evaluation model.
|
|
82
|
-
* @returns A structured object containing the evaluation scores and feedback.
|
|
83
|
-
*/
|
|
84
|
-
parseEvaluationResponse(rawResponse) {
|
|
85
|
-
try {
|
|
86
|
-
const cleanedResponse = rawResponse.replace(/```json\n|```/g, "").trim();
|
|
87
|
-
const parsed = JSON.parse(cleanedResponse);
|
|
88
|
-
logger.debug("Parsed evaluation response for RAGAS Evaluator:", parsed);
|
|
89
|
-
return {
|
|
90
|
-
relevanceScore: Number(parsed.relevanceScore) || 0,
|
|
91
|
-
accuracyScore: Number(parsed.accuracyScore) || 0,
|
|
92
|
-
completenessScore: Number(parsed.completenessScore) || 0,
|
|
93
|
-
finalScore: Number(parsed.finalScore) || 0,
|
|
94
|
-
suggestedImprovements: parsed.suggestedImprovements || "No suggestions provided.",
|
|
95
|
-
reasoning: parsed.reasoning || "No reasoning provided.",
|
|
96
|
-
};
|
|
97
|
-
}
|
|
98
|
-
catch (error) {
|
|
99
|
-
logger.error("Failed to parse evaluation response:", error);
|
|
100
|
-
return {
|
|
101
|
-
relevanceScore: 0,
|
|
102
|
-
accuracyScore: 0,
|
|
103
|
-
completenessScore: 0,
|
|
104
|
-
finalScore: 0,
|
|
105
|
-
reasoning: "Error parsing evaluation response.",
|
|
106
|
-
suggestedImprovements: "Error parsing evaluation response.",
|
|
107
|
-
};
|
|
108
|
-
}
|
|
109
|
-
}
|
|
110
|
-
}
|
|
@@ -1,78 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @file Implements the RetryManager class for handling evaluation retries.
|
|
3
|
-
*/
|
|
4
|
-
/**
|
|
5
|
-
* Manages the retry logic for the auto-evaluation middleware. It decides if a
|
|
6
|
-
* retry is warranted based on the evaluation score and prepares the options
|
|
7
|
-
* for the next generation attempt by incorporating feedback into the prompt.
|
|
8
|
-
*/
|
|
9
|
-
export class RetryManager {
|
|
10
|
-
maxRetries;
|
|
11
|
-
constructor(maxRetries = 2) {
|
|
12
|
-
// Total 3 attempts: 1 initial + 2 retries
|
|
13
|
-
this.maxRetries = maxRetries;
|
|
14
|
-
}
|
|
15
|
-
/**
|
|
16
|
-
* Determines if a retry should be attempted based on the evaluation result.
|
|
17
|
-
*
|
|
18
|
-
* @param evaluation The `EvaluationResult` of the last attempt.
|
|
19
|
-
* @returns `true` if the response did not pass and the maximum number of retries has not been reached.
|
|
20
|
-
*/
|
|
21
|
-
shouldRetry(evaluation) {
|
|
22
|
-
// Attempt number is 1-based. If attempt 1 fails, we can retry.
|
|
23
|
-
// If attempt 3 (maxRetries + 1) fails, we stop.
|
|
24
|
-
return !evaluation.isPassing && evaluation.attemptNumber <= this.maxRetries;
|
|
25
|
-
}
|
|
26
|
-
/**
|
|
27
|
-
* Prepares the options for the next generation attempt by creating a new,
|
|
28
|
-
* improved prompt that includes feedback from the failed evaluation.
|
|
29
|
-
*
|
|
30
|
-
* @param originalOptions The original `TextGenerationOptions` from the user request.
|
|
31
|
-
* @param evaluation The `EvaluationResult` of the failed attempt.
|
|
32
|
-
* @returns A new `TextGenerationOptions` object with an improved prompt.
|
|
33
|
-
*/
|
|
34
|
-
prepareRetryOptions(originalOptions, evaluation) {
|
|
35
|
-
const originalPrompt = originalOptions.prompt || originalOptions.input?.text || "";
|
|
36
|
-
const newPrompt = this.buildRetryPrompt(originalPrompt, evaluation.suggestedImprovements, evaluation.attemptNumber + 1);
|
|
37
|
-
// Return a new options object with the updated prompt
|
|
38
|
-
return {
|
|
39
|
-
...originalOptions,
|
|
40
|
-
prompt: newPrompt,
|
|
41
|
-
// Ensure input is not carried over if prompt is now the source of truth
|
|
42
|
-
input: undefined,
|
|
43
|
-
// Carry over the original prompt for context in subsequent retries if needed
|
|
44
|
-
originalPrompt: originalOptions.originalPrompt || originalPrompt,
|
|
45
|
-
};
|
|
46
|
-
}
|
|
47
|
-
/**
|
|
48
|
-
* Builds a new prompt for a retry attempt by incorporating feedback from the
|
|
49
|
-
* evaluation. The instructions become progressively more direct with each attempt.
|
|
50
|
-
*
|
|
51
|
-
* @param originalPrompt The user's original prompt.
|
|
52
|
-
* @param feedback The constructive feedback from the evaluation.
|
|
53
|
-
* @param attemptNumber The upcoming attempt number (e.g., 2 for the first retry).
|
|
54
|
-
* @returns A new, enhanced prompt string.
|
|
55
|
-
*/
|
|
56
|
-
buildRetryPrompt(originalPrompt, feedback, attemptNumber) {
|
|
57
|
-
let instruction;
|
|
58
|
-
switch (attemptNumber) {
|
|
59
|
-
case 2: // First retry
|
|
60
|
-
instruction = `The previous response was not satisfactory. Please improve it based on the following feedback: "${feedback}".`;
|
|
61
|
-
break;
|
|
62
|
-
case 3: // Second retry
|
|
63
|
-
instruction = `The last response still requires improvement. Pay close attention to this feedback: "${feedback}". You MUST address these points.`;
|
|
64
|
-
break;
|
|
65
|
-
default: // Final retry or unexpected attempt number
|
|
66
|
-
instruction = `This is the final attempt. You MUST address the following feedback to generate a satisfactory response: "${feedback}".`;
|
|
67
|
-
break;
|
|
68
|
-
}
|
|
69
|
-
return `
|
|
70
|
-
Original Request: ${originalPrompt}
|
|
71
|
-
|
|
72
|
-
**Correction Instructions:**
|
|
73
|
-
${instruction}
|
|
74
|
-
|
|
75
|
-
Generate a new, complete response that incorporates this feedback.
|
|
76
|
-
`;
|
|
77
|
-
}
|
|
78
|
-
}
|
|
@@ -1,61 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @file Contains the logic for mapping raw evaluation results to the structured EvaluationData type.
|
|
3
|
-
*/
|
|
4
|
-
import { SpanSerializer, SpanType, SpanStatus, } from "../observability/index.js";
|
|
5
|
-
import { getMetricsAggregator } from "../observability/index.js";
|
|
6
|
-
/**
|
|
7
|
-
* Maps a raw `EvaluationResult` to the structured `EvaluationData` format.
|
|
8
|
-
* This includes calculating derived fields like `isOffTopic` and `alertSeverity`.
|
|
9
|
-
*
|
|
10
|
-
* @param result The raw `EvaluationResult` from the evaluator.
|
|
11
|
-
* @param threshold The score threshold to determine if the evaluation is passing.
|
|
12
|
-
* @param offTopicThreshold The score below which a response is considered off-topic.
|
|
13
|
-
* @param highSeverityThreshold The score below which a failing response is high severity.
|
|
14
|
-
* @returns A structured `EvaluationData` object.
|
|
15
|
-
*/
|
|
16
|
-
export function mapToEvaluationData(evalContext, result, threshold, offTopicThreshold = 5, highSeverityThreshold = 4) {
|
|
17
|
-
const span = SpanSerializer.createSpan(SpanType.EVALUATION, "evaluation.score", {
|
|
18
|
-
"evaluation.dimension": "relevance|accuracy|completeness|overall",
|
|
19
|
-
scores: {
|
|
20
|
-
relevance: result.relevanceScore,
|
|
21
|
-
accuracy: result.accuracyScore,
|
|
22
|
-
completeness: result.completenessScore,
|
|
23
|
-
overall: result.finalScore,
|
|
24
|
-
},
|
|
25
|
-
});
|
|
26
|
-
const startTime = Date.now();
|
|
27
|
-
try {
|
|
28
|
-
const isPassing = result.finalScore >= threshold;
|
|
29
|
-
const evaluationData = {
|
|
30
|
-
relevance: result.relevanceScore,
|
|
31
|
-
accuracy: result.accuracyScore,
|
|
32
|
-
completeness: result.completenessScore,
|
|
33
|
-
overall: result.finalScore,
|
|
34
|
-
isOffTopic: result.finalScore < offTopicThreshold,
|
|
35
|
-
alertSeverity: isPassing
|
|
36
|
-
? "none"
|
|
37
|
-
: result.finalScore < highSeverityThreshold
|
|
38
|
-
? "high"
|
|
39
|
-
: "medium",
|
|
40
|
-
reasoning: result.reasoning,
|
|
41
|
-
suggestedImprovements: result.suggestedImprovements,
|
|
42
|
-
evaluationModel: result.evaluationModel,
|
|
43
|
-
evaluationTime: result.evaluationTime,
|
|
44
|
-
evaluationAttempt: result.attemptNumber,
|
|
45
|
-
responseContent: evalContext.aiResponse,
|
|
46
|
-
queryContent: evalContext.userQuery,
|
|
47
|
-
};
|
|
48
|
-
span.durationMs = Date.now() - startTime;
|
|
49
|
-
const endedSpan = SpanSerializer.endSpan(span, SpanStatus.OK);
|
|
50
|
-
getMetricsAggregator().recordSpan(endedSpan);
|
|
51
|
-
return evaluationData;
|
|
52
|
-
}
|
|
53
|
-
catch (error) {
|
|
54
|
-
span.durationMs = Date.now() - startTime;
|
|
55
|
-
const endedSpan = SpanSerializer.endSpan(span, SpanStatus.ERROR);
|
|
56
|
-
endedSpan.statusMessage =
|
|
57
|
-
error instanceof Error ? error.message : String(error);
|
|
58
|
-
getMetricsAggregator().recordSpan(endedSpan);
|
|
59
|
-
throw error;
|
|
60
|
-
}
|
|
61
|
-
}
|