@juspay/neurolink 9.30.0 → 9.31.1

package/dist/client/core/evaluation.js

@@ -0,0 +1,309 @@
+/**
+ * NeuroLink Evaluation System
+ */
+import { logger } from "../utils/logger.js";
+import { AIProviderFactory } from "./factory.js";
+import { z } from "zod";
+import { ProviderRegistry } from "../factories/providerRegistry.js";
+import { modelConfig } from "./modelConfiguration.js";
+import { normalizeEvaluationData } from "../utils/evaluationUtils.js";
+// Zod schema for validation
+const EvaluationSchema = z.object({
+    relevance: z.number().min(1).max(10),
+    accuracy: z.number().min(1).max(10),
+    completeness: z.number().min(1).max(10),
+    overall: z.number().min(1).max(10),
+    domainAlignment: z.number().min(1).max(10).optional(),
+    terminologyAccuracy: z.number().min(1).max(10).optional(),
+    toolEffectiveness: z.number().min(1).max(10).optional(),
+});
+/**
+ * Get default evaluation when evaluation fails
+ */
+function getDefaultEvaluation(reason, evaluationTime, context) {
+    const functionTag = "getDefaultEvaluation";
+    logger.debug(`[${functionTag}] Creating default evaluation`, {
+        reason,
+        evaluationTime,
+        hasContext: !!context,
+    });
+    return {
+        relevance: 1,
+        accuracy: 1,
+        completeness: 1,
+        overall: 1,
+        domainAlignment: 1,
+        terminologyAccuracy: 1,
+        toolEffectiveness: 1,
+        isOffTopic: false,
+        alertSeverity: "low",
+        reasoning: `Default evaluation used due to: ${reason}`,
+        contextUtilization: {
+            conversationUsed: false,
+            toolsUsed: false,
+            domainKnowledgeUsed: false,
+        },
+        evaluationContext: {
+            domain: context.primaryDomain || "general",
+            toolsEvaluated: [],
+            conversationTurns: 0,
+        },
+        evaluationModel: "default",
+        evaluationTime,
+        evaluationProvider: "default",
+        evaluationAttempt: 1,
+        evaluationConfig: {
+            mode: "fallback",
+            fallbackUsed: true,
+            costEstimate: 0,
+        },
+    };
+}
+/**
+ * Parse unified evaluation result from text response
+ */
+function parseEvaluationResult(response, context) {
+    const functionTag = "parseEvaluationResult";
+    try {
+        logger.debug(`[${functionTag}] Parsing evaluation response`, {
+            responseLength: response.length,
+            domain: context.primaryDomain,
+            hasToolUsage: !!context.toolUsage?.length,
+            hasConversationHistory: !!context.conversationHistory?.length,
+        });
+        // Try JSON parsing first
+        const jsonMatch = response.match(/\{[^}]*\}/s);
+        if (jsonMatch) {
+            try {
+                const parsed = JSON.parse(jsonMatch[0]);
+                return parsed;
+            }
+            catch (jsonError) {
+                logger.debug(`[${functionTag}] JSON parsing failed, trying regex`, {
+                    error: jsonError instanceof Error ? jsonError.message : String(jsonError),
+                    jsonContent: jsonMatch[0].substring(0, 100), // First 100 chars for debugging
+                });
+            }
+        }
+        // Fallback to regex parsing
+        const result = {};
+        const patterns = {
+            relevance: /relevance[:\s]*([0-9]+(?:\.[0-9]+)?)/i,
+            accuracy: /accuracy[:\s]*([0-9]+(?:\.[0-9]+)?)/i,
+            completeness: /completeness[:\s]*([0-9]+(?:\.[0-9]+)?)/i,
+            overall: /overall[:\s]*([0-9]+(?:\.[0-9]+)?)/i,
+            reasoning: /reasoning[:\s]*(.+?)(?=\n\s*\w+:|\n\s*$|$)/is,
+        };
+        for (const [key, pattern] of Object.entries(patterns)) {
+            const match = response.match(pattern);
+            if (match) {
+                if (key === "reasoning") {
+                    // Extract reasoning text
+                    result.reasoning = match[1].trim();
+                }
+                else {
+                    // Extract numerical scores
+                    const value = parseFloat(match[1]);
+                    if (value >= 1 && value <= 10) {
+                        const roundedValue = Math.round(value);
+                        if (key === "relevance") {
+                            result.relevance = roundedValue;
+                        }
+                        else if (key === "accuracy") {
+                            result.accuracy = roundedValue;
+                        }
+                        else if (key === "completeness") {
+                            result.completeness = roundedValue;
+                        }
+                        else if (key === "overall") {
+                            result.overall = roundedValue;
+                        }
+                    }
+                }
+            }
+        }
+        // Ensure minimum valid scores and validate with schema
+        // Use context to enhance evaluation data
+        const evaluationData = {
+            relevance: result.relevance || 1,
+            accuracy: result.accuracy || 1,
+            completeness: result.completeness || 1,
+            overall: result.overall || 1,
+            domainAlignment: result.domainAlignment || (context.primaryDomain ? 5 : undefined), // Default to 5 if domain-specific
+            terminologyAccuracy: result.terminologyAccuracy || (context.primaryDomain ? 5 : undefined),
+            toolEffectiveness: result.toolEffectiveness || (context.toolUsage?.length ? 5 : undefined), // Default to 5 if tools were used
+        };
+        // Validate against schema
+        try {
+            const validated = EvaluationSchema.parse(evaluationData);
+            // Enhance reasoning with context information
+            let enhancedReasoning = result.reasoning || "No detailed reasoning provided";
+            if (context.primaryDomain) {
+                enhancedReasoning += ` (Domain: ${context.primaryDomain})`;
+            }
+            if (context.toolUsage?.length) {
+                enhancedReasoning += ` (Tools used: ${context.toolUsage.map((t) => t.toolName).join(", ")})`;
+            }
+            if (context.conversationHistory?.length) {
+                enhancedReasoning += ` (Conversation turns: ${context.conversationHistory.length})`;
+            }
+            return {
+                ...validated,
+                reasoning: enhancedReasoning,
+            };
+        }
+        catch (validationError) {
+            logger.warn(`[${functionTag}] Schema validation failed, using fallback`, {
+                validationError,
+                originalData: evaluationData,
+            });
+            return {
+                relevance: Math.max(1, Math.min(10, result.relevance || 1)),
+                accuracy: Math.max(1, Math.min(10, result.accuracy || 1)),
+                completeness: Math.max(1, Math.min(10, result.completeness || 1)),
+                overall: Math.max(1, Math.min(10, result.overall || 1)),
+                reasoning: result.reasoning || "No detailed reasoning provided",
+            };
+        }
+    }
+    catch (error) {
+        logger.error(`[${functionTag}] Failed to parse evaluation result`, {
+            error,
+        });
+        return {
+            relevance: 1,
+            accuracy: 1,
+            completeness: 1,
+            overall: 1,
+            reasoning: "Error occurred during evaluation parsing",
+        };
+    }
+}
+/**
+ * Main unified evaluation function
+ */
+export async function generateEvaluation(context) {
+    const functionTag = "generateEvaluation";
+    const startTime = Date.now();
+    logger.debug(`[${functionTag}] Starting evaluation`, {
+        hasUserQuery: !!context.userQuery,
+        hasAiResponse: !!context.aiResponse,
+        domain: context.primaryDomain,
+    });
+    try {
+        // Ensure providers are registered
+        await ProviderRegistry.registerAllProviders();
+        // Get evaluation provider
+        const evaluationProvider = process.env.NEUROLINK_EVALUATION_PROVIDER || "google-ai";
+        // Use configurable model selection instead of hardcoded default
+        const evaluationModel = process.env.NEUROLINK_EVALUATION_MODEL ||
+            modelConfig.getModelForTier(evaluationProvider, "fast") ||
+            "gemini-2.5-flash"; // Ultimate fallback
+        logger.debug(`[${functionTag}] Using provider: ${evaluationProvider}, model: ${evaluationModel}`);
+        const provider = await AIProviderFactory.createProvider(evaluationProvider, evaluationModel);
+        if (!provider) {
+            logger.debug(`[${functionTag}] No evaluation provider available, returning defaults`);
+            return getDefaultEvaluation("no-provider", Date.now() - startTime, context);
+        }
+        // Create evaluation prompt
+        const prompt = `
+Evaluate this AI response on a scale of 1-10 for each criterion:
+
+User Query: ${context.userQuery}
+AI Response: ${context.aiResponse}
+
+Rate on these criteria (1-10 scale):
+- Relevance: How well does the response address the user's question?
+- Accuracy: How factually correct and precise is the information?
+- Completeness: How thoroughly does it cover the topic?
+- Overall: General quality assessment
+
+Respond in this exact format:
+Relevance: [score]
+Accuracy: [score]
+Completeness: [score]
+Overall: [score]
+Reasoning: [Provide a detailed explanation of your evaluation, explaining why you gave these scores. Include specific observations about the response's strengths and all possible areas for improvement.]
+`;
+        // Generate evaluation (simple text prompt only - no file processing)
+        const result = await provider.generate({
+            input: { text: prompt },
+            disableTools: true, // Evaluation doesn't need tools
+        });
+        if (!result) {
+            logger.debug(`[${functionTag}] No response from provider`);
+            return getDefaultEvaluation("no-response", Date.now() - startTime, context);
+        }
+        // Extract text from result
+        const response = typeof result === "string" ? result : result?.content || String(result);
+        // Parse evaluation result
+        const parsed = parseEvaluationResult(response, context);
+        // Validate and enhance result using schema
+        const baseResult = {
+            relevance: parsed.relevance || 1,
+            accuracy: parsed.accuracy || 1,
+            completeness: parsed.completeness || 1,
+            overall: parsed.overall || 1,
+            domainAlignment: parsed.domainAlignment,
+            terminologyAccuracy: parsed.terminologyAccuracy,
+            toolEffectiveness: parsed.toolEffectiveness,
+        };
+        // Validate against schema before finalizing
+        try {
+            const validatedScores = EvaluationSchema.parse(baseResult);
+            const validatedResult = normalizeEvaluationData({
+                ...parsed,
+                ...validatedScores,
+                evaluationModel: `${evaluationProvider}/${evaluationModel}`,
+                evaluationTime: Date.now() - startTime,
+                evaluationProvider,
+                evaluationAttempt: 1,
+                evaluationConfig: {
+                    mode: "standard",
+                    fallbackUsed: false,
+                    costEstimate: 0.001, // Rough estimate
+                },
+            });
+            logger.debug(`[${functionTag}] Schema validation passed`, {
+                validatedScores,
+            });
+            return validatedResult;
+        }
+        catch (validationError) {
+            logger.warn(`[${functionTag}] Schema validation failed in main evaluation`, {
+                validationError,
+                baseResult,
+            });
+            // Fallback with clamped values using normalizer
+            const validatedResult = normalizeEvaluationData({
+                ...parsed,
+                relevance: parsed.relevance || 1,
+                accuracy: parsed.accuracy || 1,
+                completeness: parsed.completeness || 1,
+                overall: parsed.overall || 1,
+                evaluationModel: `${evaluationProvider}/${evaluationModel}`,
+                evaluationTime: Date.now() - startTime,
+                evaluationProvider,
+                evaluationAttempt: 1,
+                evaluationConfig: {
+                    mode: "standard",
+                    fallbackUsed: false,
+                    costEstimate: 0.001, // Rough estimate
+                },
+            });
+            return validatedResult;
+        }
+    }
+    catch (error) {
+        logger.error(`[${functionTag}] Evaluation failed`, {
+            error: error instanceof Error ? error.message : String(error),
+        });
+        return getDefaultEvaluation(error instanceof Error ? error.message : "unknown-error", Date.now() - startTime, context);
+    }
+}
+// Simplified evaluation function
+export async function evaluateResponse(context) {
+    return generateEvaluation(context);
+}
+// Export additional utilities
+export { getDefaultEvaluation, parseEvaluationResult };

package/dist/client/core/evaluationProviders.js

@@ -0,0 +1,248 @@
+import { modelConfig } from "./modelConfiguration.js";
+const PERFORMANCE_THRESHOLDS = {
+    EXCELLENT_SUCCESS_RATE: 0.95,
+    EXCELLENT_RESPONSE_TIME_MS: 2000,
+    GOOD_SUCCESS_RATE: 0.9,
+    FAIR_SUCCESS_RATE: 0.8,
+};
+const providerMetrics = new Map();
+/**
+ * Convert new configuration format to legacy format for backwards compatibility
+ */
+function convertToLegacyFormat(config) {
+    return {
+        provider: config.provider,
+        models: Object.keys(config.models || {}),
+        costPerToken: config.defaultCost,
+        requiresApiKey: config.requiredEnvVars,
+        performance: config.performance,
+    };
+}
+/**
+ * Get all provider configurations using the new configuration system
+ * Replaces hardcoded EVALUATION_PROVIDER_CONFIGS with configurable system
+ */
+function getEvaluationProviderConfigs() {
+    const configs = {};
+    const allConfigs = modelConfig.getAllConfigurations();
+    for (const [provider, config] of allConfigs) {
+        configs[provider] = convertToLegacyFormat(config);
+    }
+    return configs;
+}
+/**
+ * Dynamic provider configurations for evaluation
+ * Now uses configurable system instead of hardcoded values
+ */
+export const EVALUATION_PROVIDER_CONFIGS = getEvaluationProviderConfigs();
+/**
+ * Get provider configuration by name
+ * Now uses the configurable system
+ */
+export function getProviderConfig(providerName) {
+    const config = modelConfig.getProviderConfiguration(providerName);
+    return config ? convertToLegacyFormat(config) : null;
+}
+/**
+ * Get all available providers with required API keys present
+ * Now uses the configurable system
+ */
+export function getAvailableProviders() {
+    return modelConfig
+        .getAvailableProviders()
+        .map((config) => convertToLegacyFormat(config));
+}
+/**
+ * Sort providers by preference (cost, speed, quality)
+ */
+export function sortProvidersByPreference(providers, preferCheap = true) {
+    return [...providers].sort((a, b) => {
+        const aPerf = a.performance || { cost: 0, speed: 0, quality: 0 };
+        const bPerf = b.performance || { cost: 0, speed: 0, quality: 0 };
+        if (preferCheap) {
+            // Cost > Speed > Quality for cheap preference
+            if ((aPerf.cost || 0) !== (bPerf.cost || 0)) {
+                return (bPerf.cost || 0) - (aPerf.cost || 0);
+            }
+            if ((aPerf.speed || 0) !== (bPerf.speed || 0)) {
+                return (bPerf.speed || 0) - (aPerf.speed || 0);
+            }
+            return (bPerf.quality || 0) - (aPerf.quality || 0);
+        }
+        else {
+            // Quality > Speed > Cost for quality preference
+            if ((aPerf.quality || 0) !== (bPerf.quality || 0)) {
+                return (bPerf.quality || 0) - (aPerf.quality || 0);
+            }
+            if ((aPerf.speed || 0) !== (bPerf.speed || 0)) {
+                return (bPerf.speed || 0) - (aPerf.speed || 0);
+            }
+            return (bPerf.cost || 0) - (aPerf.cost || 0);
+        }
+    });
+}
+/**
+ * Estimate cost for a specific provider and token usage
+ * Now uses the configurable system
+ */
+export function estimateProviderCost(providerName, input, output) {
+    const costInfo = modelConfig.getCostInfo(providerName);
+    if (!costInfo) {
+        return 0;
+    }
+    return input * costInfo.input + output * costInfo.output;
+}
+/**
+ * Check if a provider is available (has required API keys)
+ * Now uses the configurable system
+ */
+export function isProviderAvailable(providerName) {
+    return modelConfig.isProviderAvailable(providerName);
+}
+/**
+ * Get the best available provider based on preference
+ */
+export function getBestAvailableProvider(preferCheap = true) {
+    const availableProviders = getAvailableProviders();
+    if (availableProviders.length === 0) {
+        return null;
+    }
+    const sortedProviders = sortProvidersByPreference(availableProviders, preferCheap);
+    return sortedProviders[0];
+}
+/**
+ Record actual provider performance for optimization
+ */
+export function recordProviderPerformanceFromMetrics(providerName, metrics) {
+    const existing = providerMetrics.get(providerName) || {
+        responseTime: [],
+        successRate: 0,
+        tokenThroughput: 0,
+        costEfficiency: 0,
+        lastUpdated: new Date(),
+        sampleCount: 0,
+    };
+    // Keep rolling window of last 50 response times
+    existing.responseTime.push(metrics.responseTime);
+    if (existing.responseTime.length > 50) {
+        existing.responseTime.shift();
+    }
+    // Update success rate
+    const totalSamples = existing.sampleCount + 1;
+    existing.successRate =
+        (existing.successRate * existing.sampleCount + (metrics.success ? 1 : 0)) /
+            totalSamples;
+    // Update throughput (tokens per second)
+    if (metrics.responseTime > 0) {
+        const tokensPerSecond = metrics.tokensGenerated / (metrics.responseTime / 1000);
+        existing.tokenThroughput =
+            (existing.tokenThroughput * existing.sampleCount + tokensPerSecond) /
+                totalSamples;
+    }
+    // If responseTime is 0, skip updating tokenThroughput for this sample
+    // but still update other metrics
+    // Update cost efficiency (tokens per dollar)
+    if (metrics.cost > 0) {
+        const tokensPerDollar = metrics.tokensGenerated / metrics.cost;
+        existing.costEfficiency =
+            (existing.costEfficiency * existing.sampleCount + tokensPerDollar) /
+                totalSamples;
+    }
+    existing.sampleCount = totalSamples;
+    existing.lastUpdated = new Date();
+    providerMetrics.set(providerName, existing);
+}
+/**
+ * Get performance-optimized provider based on real metrics
+ */
+export function getPerformanceOptimizedProvider(priority = "speed") {
+    const availableProviders = getAvailableProviders();
+    if (availableProviders.length === 0) {
+        return null;
+    }
+    // Score providers based on real performance data.
+    // Only consider providers that have runtime metrics (sampleCount >= 3).
+    // Providers without metrics are excluded to avoid misleading recommendations
+    // (e.g. ollama appearing as "recommended" when it has no requiredEnvVars and
+    // thus always passes the "available" check, even when not actually in use).
+    const scoredProviders = [];
+    for (const provider of availableProviders) {
+        const metrics = providerMetrics.get(provider.provider);
+        if (!metrics || metrics.sampleCount < 3) {
+            // Skip providers without sufficient runtime data
+            continue;
+        }
+        let score = 0;
+        switch (priority) {
+            case "speed": {
+                const avgResponseTime = metrics.responseTime.reduce((a, b) => a + b, 0) /
+                    metrics.responseTime.length;
+                score =
+                    metrics.tokenThroughput * 0.6 +
+                        (5000 / Math.max(avgResponseTime, 100)) * 0.4;
+                break;
+            }
+            case "cost": {
+                score = metrics.costEfficiency * 0.7 + metrics.successRate * 0.3;
+                break;
+            }
+            case "reliability": {
+                score = metrics.successRate * 0.8 + (metrics.sampleCount / 100) * 0.2;
+                break;
+            }
+        }
+        scoredProviders.push({ provider, score });
+    }
+    if (scoredProviders.length === 0) {
+        // No providers have sufficient runtime data to make a recommendation
+        return null;
+    }
+    // Sort by score and return best
+    scoredProviders.sort((a, b) => b.score - a.score);
+    return scoredProviders[0].provider;
+}
+export function getProviderPerformanceAnalytics() {
+    const analytics = {};
+    for (const [providerName, metrics] of providerMetrics.entries()) {
+        if (metrics.sampleCount === 0) {
+            continue;
+        }
+        const avgResponseTime = metrics.responseTime.reduce((a, b) => a + b, 0) /
+            metrics.responseTime.length;
+        let recommendation;
+        if (metrics.successRate > PERFORMANCE_THRESHOLDS.EXCELLENT_SUCCESS_RATE &&
+            avgResponseTime < PERFORMANCE_THRESHOLDS.EXCELLENT_RESPONSE_TIME_MS) {
+            recommendation = "Excellent - Recommended for production";
+        }
+        else if (metrics.successRate > PERFORMANCE_THRESHOLDS.GOOD_SUCCESS_RATE) {
+            recommendation = "Good - Suitable for most tasks";
+        }
+        else if (metrics.successRate > PERFORMANCE_THRESHOLDS.FAIR_SUCCESS_RATE) {
+            recommendation = "Fair - Monitor closely";
+        }
+        else {
+            recommendation = "Poor - Consider alternative providers";
+        }
+        analytics[providerName] = {
+            avgResponseTime,
+            successRate: metrics.successRate,
+            tokenThroughput: metrics.tokenThroughput,
+            costEfficiency: metrics.costEfficiency,
+            recommendation,
+            sampleCount: metrics.sampleCount,
+        };
+    }
+    return analytics;
+}
+/**
+ * Reset performance metrics for a provider or all providers.
+ * @param providerName - (Optional) The name of the provider to reset. If omitted, resets all providers.
+ */
+export function resetProviderMetrics(providerName) {
+    if (providerName) {
+        providerMetrics.delete(providerName);
+    }
+    else {
+        providerMetrics.clear();
+    }
+}