@juspay/neurolink 9.32.0 → 9.33.0

package/dist/client/core/evaluation.js

@@ -1,309 +0,0 @@
-/**
- * NeuroLink Evaluation System
- */
-import { logger } from "../utils/logger.js";
-import { AIProviderFactory } from "./factory.js";
-import { z } from "zod";
-import { ProviderRegistry } from "../factories/providerRegistry.js";
-import { modelConfig } from "./modelConfiguration.js";
-import { normalizeEvaluationData } from "../utils/evaluationUtils.js";
-// Zod schema for validation
-const EvaluationSchema = z.object({
-    relevance: z.number().min(1).max(10),
-    accuracy: z.number().min(1).max(10),
-    completeness: z.number().min(1).max(10),
-    overall: z.number().min(1).max(10),
-    domainAlignment: z.number().min(1).max(10).optional(),
-    terminologyAccuracy: z.number().min(1).max(10).optional(),
-    toolEffectiveness: z.number().min(1).max(10).optional(),
-});
-/**
- * Get default evaluation when evaluation fails
- */
-function getDefaultEvaluation(reason, evaluationTime, context) {
-    const functionTag = "getDefaultEvaluation";
-    logger.debug(`[${functionTag}] Creating default evaluation`, {
-        reason,
-        evaluationTime,
-        hasContext: !!context,
-    });
-    return {
-        relevance: 1,
-        accuracy: 1,
-        completeness: 1,
-        overall: 1,
-        domainAlignment: 1,
-        terminologyAccuracy: 1,
-        toolEffectiveness: 1,
-        isOffTopic: false,
-        alertSeverity: "low",
-        reasoning: `Default evaluation used due to: ${reason}`,
-        contextUtilization: {
-            conversationUsed: false,
-            toolsUsed: false,
-            domainKnowledgeUsed: false,
-        },
-        evaluationContext: {
-            domain: context.primaryDomain || "general",
-            toolsEvaluated: [],
-            conversationTurns: 0,
-        },
-        evaluationModel: "default",
-        evaluationTime,
-        evaluationProvider: "default",
-        evaluationAttempt: 1,
-        evaluationConfig: {
-            mode: "fallback",
-            fallbackUsed: true,
-            costEstimate: 0,
-        },
-    };
-}
-/**
- * Parse unified evaluation result from text response
- */
-function parseEvaluationResult(response, context) {
-    const functionTag = "parseEvaluationResult";
-    try {
-        logger.debug(`[${functionTag}] Parsing evaluation response`, {
-            responseLength: response.length,
-            domain: context.primaryDomain,
-            hasToolUsage: !!context.toolUsage?.length,
-            hasConversationHistory: !!context.conversationHistory?.length,
-        });
-        // Try JSON parsing first
-        const jsonMatch = response.match(/\{[^}]*\}/s);
-        if (jsonMatch) {
-            try {
-                const parsed = JSON.parse(jsonMatch[0]);
-                return parsed;
-            }
-            catch (jsonError) {
-                logger.debug(`[${functionTag}] JSON parsing failed, trying regex`, {
-                    error: jsonError instanceof Error ? jsonError.message : String(jsonError),
-                    jsonContent: jsonMatch[0].substring(0, 100), // First 100 chars for debugging
-                });
-            }
-        }
-        // Fallback to regex parsing
-        const result = {};
-        const patterns = {
-            relevance: /relevance[:\s]*([0-9]+(?:\.[0-9]+)?)/i,
-            accuracy: /accuracy[:\s]*([0-9]+(?:\.[0-9]+)?)/i,
-            completeness: /completeness[:\s]*([0-9]+(?:\.[0-9]+)?)/i,
-            overall: /overall[:\s]*([0-9]+(?:\.[0-9]+)?)/i,
-            reasoning: /reasoning[:\s]*(.+?)(?=\n\s*\w+:|\n\s*$|$)/is,
-        };
-        for (const [key, pattern] of Object.entries(patterns)) {
-            const match = response.match(pattern);
-            if (match) {
-                if (key === "reasoning") {
-                    // Extract reasoning text
-                    result.reasoning = match[1].trim();
-                }
-                else {
-                    // Extract numerical scores
-                    const value = parseFloat(match[1]);
-                    if (value >= 1 && value <= 10) {
-                        const roundedValue = Math.round(value);
-                        if (key === "relevance") {
-                            result.relevance = roundedValue;
-                        }
-                        else if (key === "accuracy") {
-                            result.accuracy = roundedValue;
-                        }
-                        else if (key === "completeness") {
-                            result.completeness = roundedValue;
-                        }
-                        else if (key === "overall") {
-                            result.overall = roundedValue;
-                        }
-                    }
-                }
-            }
-        }
-        // Ensure minimum valid scores and validate with schema
-        // Use context to enhance evaluation data
-        const evaluationData = {
-            relevance: result.relevance || 1,
-            accuracy: result.accuracy || 1,
-            completeness: result.completeness || 1,
-            overall: result.overall || 1,
-            domainAlignment: result.domainAlignment || (context.primaryDomain ? 5 : undefined), // Default to 5 if domain-specific
-            terminologyAccuracy: result.terminologyAccuracy || (context.primaryDomain ? 5 : undefined),
-            toolEffectiveness: result.toolEffectiveness || (context.toolUsage?.length ? 5 : undefined), // Default to 5 if tools were used
-        };
-        // Validate against schema
-        try {
-            const validated = EvaluationSchema.parse(evaluationData);
-            // Enhance reasoning with context information
-            let enhancedReasoning = result.reasoning || "No detailed reasoning provided";
-            if (context.primaryDomain) {
-                enhancedReasoning += ` (Domain: ${context.primaryDomain})`;
-            }
-            if (context.toolUsage?.length) {
-                enhancedReasoning += ` (Tools used: ${context.toolUsage.map((t) => t.toolName).join(", ")})`;
-            }
-            if (context.conversationHistory?.length) {
-                enhancedReasoning += ` (Conversation turns: ${context.conversationHistory.length})`;
-            }
-            return {
-                ...validated,
-                reasoning: enhancedReasoning,
-            };
-        }
-        catch (validationError) {
-            logger.warn(`[${functionTag}] Schema validation failed, using fallback`, {
-                validationError,
-                originalData: evaluationData,
-            });
-            return {
-                relevance: Math.max(1, Math.min(10, result.relevance || 1)),
-                accuracy: Math.max(1, Math.min(10, result.accuracy || 1)),
-                completeness: Math.max(1, Math.min(10, result.completeness || 1)),
-                overall: Math.max(1, Math.min(10, result.overall || 1)),
-                reasoning: result.reasoning || "No detailed reasoning provided",
-            };
-        }
-    }
-    catch (error) {
-        logger.error(`[${functionTag}] Failed to parse evaluation result`, {
-            error,
-        });
-        return {
-            relevance: 1,
-            accuracy: 1,
-            completeness: 1,
-            overall: 1,
-            reasoning: "Error occurred during evaluation parsing",
-        };
-    }
-}
-/**
- * Main unified evaluation function
- */
-export async function generateEvaluation(context) {
-    const functionTag = "generateEvaluation";
-    const startTime = Date.now();
-    logger.debug(`[${functionTag}] Starting evaluation`, {
-        hasUserQuery: !!context.userQuery,
-        hasAiResponse: !!context.aiResponse,
-        domain: context.primaryDomain,
-    });
-    try {
-        // Ensure providers are registered
-        await ProviderRegistry.registerAllProviders();
-        // Get evaluation provider
-        const evaluationProvider = process.env.NEUROLINK_EVALUATION_PROVIDER || "google-ai";
-        // Use configurable model selection instead of hardcoded default
-        const evaluationModel = process.env.NEUROLINK_EVALUATION_MODEL ||
-            modelConfig.getModelForTier(evaluationProvider, "fast") ||
-            "gemini-2.5-flash"; // Ultimate fallback
-        logger.debug(`[${functionTag}] Using provider: ${evaluationProvider}, model: ${evaluationModel}`);
-        const provider = await AIProviderFactory.createProvider(evaluationProvider, evaluationModel);
-        if (!provider) {
-            logger.debug(`[${functionTag}] No evaluation provider available, returning defaults`);
-            return getDefaultEvaluation("no-provider", Date.now() - startTime, context);
-        }
-        // Create evaluation prompt
-        const prompt = `
-Evaluate this AI response on a scale of 1-10 for each criterion:
-
-User Query: ${context.userQuery}
-AI Response: ${context.aiResponse}
-
-Rate on these criteria (1-10 scale):
-- Relevance: How well does the response address the user's question?
-- Accuracy: How factually correct and precise is the information?
-- Completeness: How thoroughly does it cover the topic?
-- Overall: General quality assessment
-
-Respond in this exact format:
-Relevance: [score]
-Accuracy: [score]
-Completeness: [score]
-Overall: [score]
-Reasoning: [Provide a detailed explanation of your evaluation, explaining why you gave these scores. Include specific observations about the response's strengths and all possible areas for improvement.]
-`;
-        // Generate evaluation (simple text prompt only - no file processing)
-        const result = await provider.generate({
-            input: { text: prompt },
-            disableTools: true, // Evaluation doesn't need tools
-        });
-        if (!result) {
-            logger.debug(`[${functionTag}] No response from provider`);
-            return getDefaultEvaluation("no-response", Date.now() - startTime, context);
-        }
-        // Extract text from result
-        const response = typeof result === "string" ? result : result?.content || String(result);
-        // Parse evaluation result
-        const parsed = parseEvaluationResult(response, context);
-        // Validate and enhance result using schema
-        const baseResult = {
-            relevance: parsed.relevance || 1,
-            accuracy: parsed.accuracy || 1,
-            completeness: parsed.completeness || 1,
-            overall: parsed.overall || 1,
-            domainAlignment: parsed.domainAlignment,
-            terminologyAccuracy: parsed.terminologyAccuracy,
-            toolEffectiveness: parsed.toolEffectiveness,
-        };
-        // Validate against schema before finalizing
-        try {
-            const validatedScores = EvaluationSchema.parse(baseResult);
-            const validatedResult = normalizeEvaluationData({
-                ...parsed,
-                ...validatedScores,
-                evaluationModel: `${evaluationProvider}/${evaluationModel}`,
-                evaluationTime: Date.now() - startTime,
-                evaluationProvider,
-                evaluationAttempt: 1,
-                evaluationConfig: {
-                    mode: "standard",
-                    fallbackUsed: false,
-                    costEstimate: 0.001, // Rough estimate
-                },
-            });
-            logger.debug(`[${functionTag}] Schema validation passed`, {
-                validatedScores,
-            });
-            return validatedResult;
-        }
-        catch (validationError) {
-            logger.warn(`[${functionTag}] Schema validation failed in main evaluation`, {
-                validationError,
-                baseResult,
-            });
-            // Fallback with clamped values using normalizer
-            const validatedResult = normalizeEvaluationData({
-                ...parsed,
-                relevance: parsed.relevance || 1,
-                accuracy: parsed.accuracy || 1,
-                completeness: parsed.completeness || 1,
-                overall: parsed.overall || 1,
-                evaluationModel: `${evaluationProvider}/${evaluationModel}`,
-                evaluationTime: Date.now() - startTime,
-                evaluationProvider,
-                evaluationAttempt: 1,
-                evaluationConfig: {
-                    mode: "standard",
-                    fallbackUsed: false,
-                    costEstimate: 0.001, // Rough estimate
-                },
-            });
-            return validatedResult;
-        }
-    }
-    catch (error) {
-        logger.error(`[${functionTag}] Evaluation failed`, {
-            error: error instanceof Error ? error.message : String(error),
-        });
-        return getDefaultEvaluation(error instanceof Error ? error.message : "unknown-error", Date.now() - startTime, context);
-    }
-}
-// Simplified evaluation function
-export async function evaluateResponse(context) {
-    return generateEvaluation(context);
-}
-// Export additional utilities
-export { getDefaultEvaluation, parseEvaluationResult };

package/dist/client/core/evaluationProviders.js

@@ -1,248 +0,0 @@
-import { modelConfig } from "./modelConfiguration.js";
-const PERFORMANCE_THRESHOLDS = {
-    EXCELLENT_SUCCESS_RATE: 0.95,
-    EXCELLENT_RESPONSE_TIME_MS: 2000,
-    GOOD_SUCCESS_RATE: 0.9,
-    FAIR_SUCCESS_RATE: 0.8,
-};
-const providerMetrics = new Map();
-/**
- * Convert new configuration format to legacy format for backwards compatibility
- */
-function convertToLegacyFormat(config) {
-    return {
-        provider: config.provider,
-        models: Object.keys(config.models || {}),
-        costPerToken: config.defaultCost,
-        requiresApiKey: config.requiredEnvVars,
-        performance: config.performance,
-    };
-}
-/**
- * Get all provider configurations using the new configuration system
- * Replaces hardcoded EVALUATION_PROVIDER_CONFIGS with configurable system
- */
-function getEvaluationProviderConfigs() {
-    const configs = {};
-    const allConfigs = modelConfig.getAllConfigurations();
-    for (const [provider, config] of allConfigs) {
-        configs[provider] = convertToLegacyFormat(config);
-    }
-    return configs;
-}
-/**
- * Dynamic provider configurations for evaluation
- * Now uses configurable system instead of hardcoded values
- */
-export const EVALUATION_PROVIDER_CONFIGS = getEvaluationProviderConfigs();
-/**
- * Get provider configuration by name
- * Now uses the configurable system
- */
-export function getProviderConfig(providerName) {
-    const config = modelConfig.getProviderConfiguration(providerName);
-    return config ? convertToLegacyFormat(config) : null;
-}
-/**
- * Get all available providers with required API keys present
- * Now uses the configurable system
- */
-export function getAvailableProviders() {
-    return modelConfig
-        .getAvailableProviders()
-        .map((config) => convertToLegacyFormat(config));
-}
-/**
- * Sort providers by preference (cost, speed, quality)
- */
-export function sortProvidersByPreference(providers, preferCheap = true) {
-    return [...providers].sort((a, b) => {
-        const aPerf = a.performance || { cost: 0, speed: 0, quality: 0 };
-        const bPerf = b.performance || { cost: 0, speed: 0, quality: 0 };
-        if (preferCheap) {
-            // Cost > Speed > Quality for cheap preference
-            if ((aPerf.cost || 0) !== (bPerf.cost || 0)) {
-                return (bPerf.cost || 0) - (aPerf.cost || 0);
-            }
-            if ((aPerf.speed || 0) !== (bPerf.speed || 0)) {
-                return (bPerf.speed || 0) - (aPerf.speed || 0);
-            }
-            return (bPerf.quality || 0) - (aPerf.quality || 0);
-        }
-        else {
-            // Quality > Speed > Cost for quality preference
-            if ((aPerf.quality || 0) !== (bPerf.quality || 0)) {
-                return (bPerf.quality || 0) - (aPerf.quality || 0);
-            }
-            if ((aPerf.speed || 0) !== (bPerf.speed || 0)) {
-                return (bPerf.speed || 0) - (aPerf.speed || 0);
-            }
-            return (bPerf.cost || 0) - (aPerf.cost || 0);
-        }
-    });
-}
-/**
- * Estimate cost for a specific provider and token usage
- * Now uses the configurable system
- */
-export function estimateProviderCost(providerName, input, output) {
-    const costInfo = modelConfig.getCostInfo(providerName);
-    if (!costInfo) {
-        return 0;
-    }
-    return input * costInfo.input + output * costInfo.output;
-}
-/**
- * Check if a provider is available (has required API keys)
- * Now uses the configurable system
- */
-export function isProviderAvailable(providerName) {
-    return modelConfig.isProviderAvailable(providerName);
-}
-/**
- * Get the best available provider based on preference
- */
-export function getBestAvailableProvider(preferCheap = true) {
-    const availableProviders = getAvailableProviders();
-    if (availableProviders.length === 0) {
-        return null;
-    }
-    const sortedProviders = sortProvidersByPreference(availableProviders, preferCheap);
-    return sortedProviders[0];
-}
-/**
- Record actual provider performance for optimization
- */
-export function recordProviderPerformanceFromMetrics(providerName, metrics) {
-    const existing = providerMetrics.get(providerName) || {
-        responseTime: [],
-        successRate: 0,
-        tokenThroughput: 0,
-        costEfficiency: 0,
-        lastUpdated: new Date(),
-        sampleCount: 0,
-    };
-    // Keep rolling window of last 50 response times
-    existing.responseTime.push(metrics.responseTime);
-    if (existing.responseTime.length > 50) {
-        existing.responseTime.shift();
-    }
-    // Update success rate
-    const totalSamples = existing.sampleCount + 1;
-    existing.successRate =
-        (existing.successRate * existing.sampleCount + (metrics.success ? 1 : 0)) /
-            totalSamples;
-    // Update throughput (tokens per second)
-    if (metrics.responseTime > 0) {
-        const tokensPerSecond = metrics.tokensGenerated / (metrics.responseTime / 1000);
-        existing.tokenThroughput =
-            (existing.tokenThroughput * existing.sampleCount + tokensPerSecond) /
-                totalSamples;
-    }
-    // If responseTime is 0, skip updating tokenThroughput for this sample
-    // but still update other metrics
-    // Update cost efficiency (tokens per dollar)
-    if (metrics.cost > 0) {
-        const tokensPerDollar = metrics.tokensGenerated / metrics.cost;
-        existing.costEfficiency =
-            (existing.costEfficiency * existing.sampleCount + tokensPerDollar) /
-                totalSamples;
-    }
-    existing.sampleCount = totalSamples;
-    existing.lastUpdated = new Date();
-    providerMetrics.set(providerName, existing);
-}
-/**
- * Get performance-optimized provider based on real metrics
- */
-export function getPerformanceOptimizedProvider(priority = "speed") {
-    const availableProviders = getAvailableProviders();
-    if (availableProviders.length === 0) {
-        return null;
-    }
-    // Score providers based on real performance data.
-    // Only consider providers that have runtime metrics (sampleCount >= 3).
-    // Providers without metrics are excluded to avoid misleading recommendations
-    // (e.g. ollama appearing as "recommended" when it has no requiredEnvVars and
-    // thus always passes the "available" check, even when not actually in use).
-    const scoredProviders = [];
-    for (const provider of availableProviders) {
-        const metrics = providerMetrics.get(provider.provider);
-        if (!metrics || metrics.sampleCount < 3) {
-            // Skip providers without sufficient runtime data
-            continue;
-        }
-        let score = 0;
-        switch (priority) {
-            case "speed": {
-                const avgResponseTime = metrics.responseTime.reduce((a, b) => a + b, 0) /
-                    metrics.responseTime.length;
-                score =
-                    metrics.tokenThroughput * 0.6 +
-                        (5000 / Math.max(avgResponseTime, 100)) * 0.4;
-                break;
-            }
-            case "cost": {
-                score = metrics.costEfficiency * 0.7 + metrics.successRate * 0.3;
-                break;
-            }
-            case "reliability": {
-                score = metrics.successRate * 0.8 + (metrics.sampleCount / 100) * 0.2;
-                break;
-            }
-        }
-        scoredProviders.push({ provider, score });
-    }
-    if (scoredProviders.length === 0) {
-        // No providers have sufficient runtime data to make a recommendation
-        return null;
-    }
-    // Sort by score and return best
-    scoredProviders.sort((a, b) => b.score - a.score);
-    return scoredProviders[0].provider;
-}
-export function getProviderPerformanceAnalytics() {
-    const analytics = {};
-    for (const [providerName, metrics] of providerMetrics.entries()) {
-        if (metrics.sampleCount === 0) {
-            continue;
-        }
-        const avgResponseTime = metrics.responseTime.reduce((a, b) => a + b, 0) /
-            metrics.responseTime.length;
-        let recommendation;
-        if (metrics.successRate > PERFORMANCE_THRESHOLDS.EXCELLENT_SUCCESS_RATE &&
-            avgResponseTime < PERFORMANCE_THRESHOLDS.EXCELLENT_RESPONSE_TIME_MS) {
-            recommendation = "Excellent - Recommended for production";
-        }
-        else if (metrics.successRate > PERFORMANCE_THRESHOLDS.GOOD_SUCCESS_RATE) {
-            recommendation = "Good - Suitable for most tasks";
-        }
-        else if (metrics.successRate > PERFORMANCE_THRESHOLDS.FAIR_SUCCESS_RATE) {
-            recommendation = "Fair - Monitor closely";
-        }
-        else {
-            recommendation = "Poor - Consider alternative providers";
-        }
-        analytics[providerName] = {
-            avgResponseTime,
-            successRate: metrics.successRate,
-            tokenThroughput: metrics.tokenThroughput,
-            costEfficiency: metrics.costEfficiency,
-            recommendation,
-            sampleCount: metrics.sampleCount,
-        };
-    }
-    return analytics;
-}
-/**
- * Reset performance metrics for a provider or all providers.
- * @param providerName - (Optional) The name of the provider to reset. If omitted, resets all providers.
- */
-export function resetProviderMetrics(providerName) {
-    if (providerName) {
-        providerMetrics.delete(providerName);
-    }
-    else {
-        providerMetrics.clear();
-    }
-}