@juspay/neurolink 9.29.1 → 9.31.0

package/dist/client/evaluation/index.js

@@ -0,0 +1,61 @@
+/**
+ * @file This file exports the main Evaluator class, which serves as the central entry point for the evaluation system.
+ */
+import { ContextBuilder } from "./contextBuilder.js";
+import { RAGASEvaluator } from "./ragasEvaluator.js";
+import { mapToEvaluationData } from "./scoring.js";
+/**
+ * A centralized class for performing response evaluations. It supports different
+ * evaluation strategies, with RAGAS-style model-based evaluation as the default.
+ * This class orchestrates the context building and evaluation process.
+ */
+export class Evaluator {
+    contextBuilder;
+    config;
+    ragasEvaluator;
+    constructor(config = {}) {
+        this.config = config;
+        this.contextBuilder = new ContextBuilder();
+        this.ragasEvaluator = new RAGASEvaluator(this.config.evaluationModel, this.config.provider, this.config.threshold, this.config.promptGenerator);
+    }
+    /**
+     * The main entry point for performing an evaluation. It selects the evaluation
+     * strategy based on the configuration and executes it.
+     *
+     * @param options The original `TextGenerationOptions` from the user request.
+     * @param result The `GenerateResult` from the provider.
+     * @returns A promise that resolves to the `EvaluationResult`.
+     */
+    async evaluate(options, result, threshold, config) {
+        const evaluationStrategy = this.config.evaluationStrategy || "ragas";
+        const customEvaluator = this.config.customEvaluator;
+        switch (evaluationStrategy) {
+            case "ragas": {
+                const { evaluationResult, evalContext } = await this.evaluateWithRAGAS(options, result);
+                const evaluationData = mapToEvaluationData(evalContext, evaluationResult, threshold, config.offTopicThreshold, config.highSeverityThreshold);
+                return evaluationData;
+            }
+            case "custom": {
+                if (customEvaluator) {
+                    const { evaluationResult, evalContext } = await customEvaluator(options, result);
+                    return mapToEvaluationData(evalContext, evaluationResult, threshold, config.offTopicThreshold, config.highSeverityThreshold);
+                }
+                throw new Error("Custom evaluator function not provided in config.");
+            }
+            default:
+                throw new Error(`Unsupported evaluation strategy: ${evaluationStrategy} `);
+        }
+    }
+    /**
+     * Performs evaluation using the RAGAS-style model-based evaluator.
+     *
+     * @param options The original `TextGenerationOptions`.
+     * @param result The `GenerateResult` to be evaluated.
+     * @returns A promise that resolves to the `EvaluationResult`.
+     */
+    async evaluateWithRAGAS(options, result) {
+        const evalContext = this.contextBuilder.buildContext(options, result);
+        const evaluationResult = await this.ragasEvaluator.evaluate(evalContext);
+        return { evaluationResult, evalContext };
+    }
+}

package/dist/client/evaluation/prompts.js

@@ -0,0 +1,73 @@
+/**
+ * A flexible class for building evaluation prompts. It allows for custom prompt
+ * generation logic to be injected while ensuring a consistent output format.
+ */
+export class PromptBuilder {
+    /**
+     * Builds the final evaluation prompt.
+     *
+     * @param context The rich context for the evaluation.
+     * @param getPrompt An optional function to generate the main body of the prompt.
+     *                  If not provided, a default prompt is used.
+     * @returns The complete prompt string to be sent to the judge LLM.
+     */
+    buildEvaluationPrompt(context, getPrompt) {
+        const { userQuery, aiResponse, conversationHistory, toolExecutions, previousEvaluations, } = context;
+        const historyStr = conversationHistory
+            .map((turn) => `${turn.role}: ${turn.content}`)
+            .join("\n");
+        const toolsStr = toolExecutions.length > 0
+            ? `Tools were used: ${toolExecutions.map((t) => t.toolName).join(", ")}`
+            : "No tools were used.";
+        const retryStr = previousEvaluations && previousEvaluations.length > 0
+            ? `This is attempt #${context.attemptNumber}. Previous reasoning: ${previousEvaluations
+                .map((e) => e.reasoning)
+                .join("; ")} Previous suggested Improvements: ${previousEvaluations.map((e) => e.suggestedImprovements).join("; ")}`
+            : "This is the first attempt.";
+        const promptContext = {
+            userQuery,
+            history: historyStr,
+            tools: toolsStr,
+            retryInfo: retryStr,
+            aiResponse,
+        };
+        const mainPrompt = getPrompt
+            ? getPrompt(promptContext)
+            : this.getDefaultPrompt(promptContext);
+        return `
+      ${mainPrompt}
+
+      **Output Format (JSON):**
+      {
+        "relevanceScore": <1-10>,
+        "accuracyScore": <1-10>,
+        "completenessScore": <1-10>,
+        "finalScore": <1-10>,
+        "reasoning": "<Your constructive reasoning here>",
+        "suggestedImprovements": "<How the response can be improved>"
+      }
+    `;
+    }
+    /**
+     * The default prompt generation logic.
+     * @param context The prepared context strings.
+     * @returns The default prompt body.
+     */
+    getDefaultPrompt(context) {
+        return `
+      You are an expert AI quality evaluator. Your task is to evaluate the AI assistant's response based on the provided context.
+      Provide a score from 1 to 10 for each of the following criteria: Relevance, Accuracy, and Completeness.
+      Finally, provide an overall finalScore and constructive feedback for improvement.
+
+      **Evaluation Context:**
+      - User Query: ${context.userQuery}
+      - Conversation History:
+      ${context.history}
+      - Tools Executed: ${context.tools}
+      - Retry Information: ${context.retryInfo}
+
+      **AI Assistant's Response to Evaluate:**
+      ${context.aiResponse}
+    `;
+    }
+}

package/dist/client/evaluation/ragasEvaluator.js

@@ -0,0 +1,110 @@
+import { AIProviderFactory } from "../core/factory.js";
+import { PromptBuilder } from "./prompts.js";
+import { logger } from "../utils/logger.js";
+import { SpanSerializer, SpanType, SpanStatus, } from "../observability/index.js";
+import { getMetricsAggregator } from "../observability/index.js";
+/**
+ * Implements a RAGAS-style evaluator that uses a "judge" LLM to score the
+ * quality of an AI response based on rich, contextual information.
+ */
+export class RAGASEvaluator {
+    evaluationModel;
+    providerName;
+    threshold;
+    promptBuilder;
+    promptGenerator;
+    constructor(evaluationModel, providerName, threshold, promptGenerator) {
+        this.evaluationModel =
+            evaluationModel ||
+                process.env.NEUROLINK_RAGAS_EVALUATION_MODEL ||
+                "gemini-1.5-flash";
+        this.providerName =
+            providerName ||
+                process.env.NEUROLINK_RAGAS_EVALUATION_PROVIDER ||
+                "vertex";
+        this.threshold =
+            threshold || Number(process.env.NEUROLINK_EVALUATION_THRESHOLD) || 7;
+        this.promptBuilder = new PromptBuilder();
+        this.promptGenerator = promptGenerator;
+    }
+    /**
+     * Evaluates an AI-generated response using a model-based approach.
+     *
+     * @param context The rich, contextual information for the evaluation.
+     * @returns A promise that resolves to a detailed `EvaluationResult`.
+     */
+    async evaluate(context) {
+        const span = SpanSerializer.createSpan(SpanType.EVALUATION, "evaluation.ragas", {
+            "evaluation.dimension": "relevance|accuracy|completeness",
+            "ai.provider": this.providerName,
+            "ai.model": this.evaluationModel,
+        });
+        const startTime = Date.now();
+        try {
+            const prompt = this.promptBuilder.buildEvaluationPrompt(context, this.promptGenerator);
+            const provider = await AIProviderFactory.createProvider(this.providerName, this.evaluationModel);
+            const result = await provider.generate({
+                input: { text: prompt },
+            });
+            if (!result) {
+                throw new Error("Evaluation generation failed to return a result.");
+            }
+            const rawEvaluationResponse = result.content;
+            const parsedResult = this.parseEvaluationResponse(rawEvaluationResponse);
+            const evaluationTime = Date.now() - startTime;
+            const finalResult = {
+                ...parsedResult,
+                isPassing: parsedResult.finalScore >= this.threshold, // This will be recalculated, but is needed for the type
+                evaluationModel: this.evaluationModel,
+                evaluationTime,
+                attemptNumber: context.attemptNumber,
+                rawEvaluationResponse,
+            };
+            span.durationMs = Date.now() - startTime;
+            const endedSpan = SpanSerializer.endSpan(span, SpanStatus.OK);
+            getMetricsAggregator().recordSpan(endedSpan);
+            return finalResult;
+        }
+        catch (error) {
+            span.durationMs = Date.now() - startTime;
+            const endedSpan = SpanSerializer.endSpan(span, SpanStatus.ERROR);
+            endedSpan.statusMessage =
+                error instanceof Error ? error.message : String(error);
+            getMetricsAggregator().recordSpan(endedSpan);
+            throw error;
+        }
+    }
+    /**
+     * Parses the raw JSON string from the judge LLM into a structured `EvaluationResult` object.
+     * It includes error handling to gracefully manage malformed JSON.
+     *
+     * @param rawResponse The raw string response from the evaluation model.
+     * @returns A structured object containing the evaluation scores and feedback.
+     */
+    parseEvaluationResponse(rawResponse) {
+        try {
+            const cleanedResponse = rawResponse.replace(/```json\n|```/g, "").trim();
+            const parsed = JSON.parse(cleanedResponse);
+            logger.debug("Parsed evaluation response for RAGAS Evaluator:", parsed);
+            return {
+                relevanceScore: Number(parsed.relevanceScore) || 0,
+                accuracyScore: Number(parsed.accuracyScore) || 0,
+                completenessScore: Number(parsed.completenessScore) || 0,
+                finalScore: Number(parsed.finalScore) || 0,
+                suggestedImprovements: parsed.suggestedImprovements || "No suggestions provided.",
+                reasoning: parsed.reasoning || "No reasoning provided.",
+            };
+        }
+        catch (error) {
+            logger.error("Failed to parse evaluation response:", error);
+            return {
+                relevanceScore: 0,
+                accuracyScore: 0,
+                completenessScore: 0,
+                finalScore: 0,
+                reasoning: "Error parsing evaluation response.",
+                suggestedImprovements: "Error parsing evaluation response.",
+            };
+        }
+    }
+}

package/dist/client/evaluation/retryManager.js

@@ -0,0 +1,78 @@
+/**
+ * @file Implements the RetryManager class for handling evaluation retries.
+ */
+/**
+ * Manages the retry logic for the auto-evaluation middleware. It decides if a
+ * retry is warranted based on the evaluation score and prepares the options
+ * for the next generation attempt by incorporating feedback into the prompt.
+ */
+export class RetryManager {
+    maxRetries;
+    constructor(maxRetries = 2) {
+        // Total 3 attempts: 1 initial + 2 retries
+        this.maxRetries = maxRetries;
+    }
+    /**
+     * Determines if a retry should be attempted based on the evaluation result.
+     *
+     * @param evaluation The `EvaluationResult` of the last attempt.
+     * @returns `true` if the response did not pass and the maximum number of retries has not been reached.
+     */
+    shouldRetry(evaluation) {
+        // Attempt number is 1-based. If attempt 1 fails, we can retry.
+        // If attempt 3 (maxRetries + 1) fails, we stop.
+        return !evaluation.isPassing && evaluation.attemptNumber <= this.maxRetries;
+    }
+    /**
+     * Prepares the options for the next generation attempt by creating a new,
+     * improved prompt that includes feedback from the failed evaluation.
+     *
+     * @param originalOptions The original `TextGenerationOptions` from the user request.
+     * @param evaluation The `EvaluationResult` of the failed attempt.
+     * @returns A new `TextGenerationOptions` object with an improved prompt.
+     */
+    prepareRetryOptions(originalOptions, evaluation) {
+        const originalPrompt = originalOptions.prompt || originalOptions.input?.text || "";
+        const newPrompt = this.buildRetryPrompt(originalPrompt, evaluation.suggestedImprovements, evaluation.attemptNumber + 1);
+        // Return a new options object with the updated prompt
+        return {
+            ...originalOptions,
+            prompt: newPrompt,
+            // Ensure input is not carried over if prompt is now the source of truth
+            input: undefined,
+            // Carry over the original prompt for context in subsequent retries if needed
+            originalPrompt: originalOptions.originalPrompt || originalPrompt,
+        };
+    }
+    /**
+     * Builds a new prompt for a retry attempt by incorporating feedback from the
+     * evaluation. The instructions become progressively more direct with each attempt.
+     *
+     * @param originalPrompt The user's original prompt.
+     * @param feedback The constructive feedback from the evaluation.
+     * @param attemptNumber The upcoming attempt number (e.g., 2 for the first retry).
+     * @returns A new, enhanced prompt string.
+     */
+    buildRetryPrompt(originalPrompt, feedback, attemptNumber) {
+        let instruction;
+        switch (attemptNumber) {
+            case 2: // First retry
+                instruction = `The previous response was not satisfactory. Please improve it based on the following feedback: "${feedback}".`;
+                break;
+            case 3: // Second retry
+                instruction = `The last response still requires improvement. Pay close attention to this feedback: "${feedback}". You MUST address these points.`;
+                break;
+            default: // Final retry or unexpected attempt number
+                instruction = `This is the final attempt. You MUST address the following feedback to generate a satisfactory response: "${feedback}".`;
+                break;
+        }
+        return `
+    Original Request: ${originalPrompt}
+
+    **Correction Instructions:**
+    ${instruction}
+
+    Generate a new, complete response that incorporates this feedback.
+  `;
+    }
+}

package/dist/client/evaluation/scoring.js

@@ -0,0 +1,61 @@
+/**
+ * @file Contains the logic for mapping raw evaluation results to the structured EvaluationData type.
+ */
+import { SpanSerializer, SpanType, SpanStatus, } from "../observability/index.js";
+import { getMetricsAggregator } from "../observability/index.js";
+/**
+ * Maps a raw `EvaluationResult` to the structured `EvaluationData` format.
+ * This includes calculating derived fields like `isOffTopic` and `alertSeverity`.
+ *
+ * @param result The raw `EvaluationResult` from the evaluator.
+ * @param threshold The score threshold to determine if the evaluation is passing.
+ * @param offTopicThreshold The score below which a response is considered off-topic.
+ * @param highSeverityThreshold The score below which a failing response is high severity.
+ * @returns A structured `EvaluationData` object.
+ */
+export function mapToEvaluationData(evalContext, result, threshold, offTopicThreshold = 5, highSeverityThreshold = 4) {
+    const span = SpanSerializer.createSpan(SpanType.EVALUATION, "evaluation.score", {
+        "evaluation.dimension": "relevance|accuracy|completeness|overall",
+        scores: {
+            relevance: result.relevanceScore,
+            accuracy: result.accuracyScore,
+            completeness: result.completenessScore,
+            overall: result.finalScore,
+        },
+    });
+    const startTime = Date.now();
+    try {
+        const isPassing = result.finalScore >= threshold;
+        const evaluationData = {
+            relevance: result.relevanceScore,
+            accuracy: result.accuracyScore,
+            completeness: result.completenessScore,
+            overall: result.finalScore,
+            isOffTopic: result.finalScore < offTopicThreshold,
+            alertSeverity: isPassing
+                ? "none"
+                : result.finalScore < highSeverityThreshold
+                    ? "high"
+                    : "medium",
+            reasoning: result.reasoning,
+            suggestedImprovements: result.suggestedImprovements,
+            evaluationModel: result.evaluationModel,
+            evaluationTime: result.evaluationTime,
+            evaluationAttempt: result.attemptNumber,
+            responseContent: evalContext.aiResponse,
+            queryContent: evalContext.userQuery,
+        };
+        span.durationMs = Date.now() - startTime;
+        const endedSpan = SpanSerializer.endSpan(span, SpanStatus.OK);
+        getMetricsAggregator().recordSpan(endedSpan);
+        return evaluationData;
+    }
+    catch (error) {
+        span.durationMs = Date.now() - startTime;
+        const endedSpan = SpanSerializer.endSpan(span, SpanStatus.ERROR);
+        endedSpan.statusMessage =
+            error instanceof Error ? error.message : String(error);
+        getMetricsAggregator().recordSpan(endedSpan);
+        throw error;
+    }
+}

package/dist/client/factories/providerFactory.js

@@ -0,0 +1,166 @@
+import { logger } from "../utils/logger.js";
+/**
+ * True Factory Pattern implementation for AI Providers
+ * Uses registration-based approach to eliminate switch statements
+ * and enable dynamic provider registration
+ */
+export class ProviderFactory {
+    static providers = new Map();
+    static initialized = false;
+    /**
+     * Register a provider with the factory
+     */
+    static registerProvider(name, constructor, defaultModel, // Optional - provider can read from env
+    aliases = []) {
+        const registration = {
+            constructor,
+            defaultModel,
+            aliases,
+        };
+        // Register main name
+        ProviderFactory.providers.set(name.toLowerCase(), registration);
+        // Register aliases
+        aliases.forEach((alias) => {
+            ProviderFactory.providers.set(alias.toLowerCase(), registration);
+        });
+        logger.debug(`Registered provider: ${name} with model ${defaultModel || "from-env"}`);
+    }
+    /**
+     * Create a provider instance
+     * @param providerName - Provider name (optional, uses NEUROLINK_PROVIDER env var or 'vertex' as default)
+     * @param modelName - Model name (optional, uses provider-specific env var or registry default)
+     */
+    static async createProvider(providerName, modelName, sdk, region) {
+        // Note: Providers are registered explicitly by ProviderRegistry to avoid circular dependencies
+        // Use environment variable or default if not specified
+        const resolvedProviderName = providerName ||
+            process.env.NEUROLINK_PROVIDER ||
+            process.env.AI_PROVIDER ||
+            "vertex";
+        const normalizedName = resolvedProviderName.toLowerCase();
+        const registration = ProviderFactory.providers.get(normalizedName);
+        if (!registration) {
+            throw new Error(`Unknown provider: ${resolvedProviderName}. Available providers: ${ProviderFactory.getAvailableProviders().join(", ")}`);
+        }
+        // Respect environment variables before falling back to registry default
+        let model = modelName;
+        if (!model) {
+            // Check for provider-specific environment variables
+            if (resolvedProviderName.toLowerCase().includes("vertex")) {
+                // Use gemini-2.5-flash as default - latest GA model with best price-performance
+                model = process.env.VERTEX_MODEL || "gemini-2.5-flash";
+            }
+            else if (resolvedProviderName.toLowerCase().includes("bedrock")) {
+                model = process.env.BEDROCK_MODEL || process.env.BEDROCK_MODEL_ID;
+            }
+            // Fallback to registry default if no env var
+            model = model || registration.defaultModel;
+        }
+        try {
+            if (typeof registration.constructor !== "function") {
+                throw new Error(`Invalid constructor for provider ${providerName}: not a function`);
+            }
+            let result;
+            try {
+                const factoryResult = registration.constructor(model, resolvedProviderName, sdk, region);
+                // Handle both sync and async results
+                result =
+                    factoryResult instanceof Promise
+                        ? await factoryResult
+                        : factoryResult;
+            }
+            catch (factoryError) {
+                if (registration.constructor.prototype &&
+                    registration.constructor.prototype.constructor ===
+                        registration.constructor) {
+                    try {
+                        result = new registration.constructor(model, resolvedProviderName, sdk, region);
+                    }
+                    catch (constructorError) {
+                        throw new Error(`Both factory function and constructor failed. Factory error: ${factoryError}. Constructor error: ${constructorError}`, { cause: constructorError });
+                    }
+                }
+                else {
+                    throw factoryError;
+                }
+            }
+            return result;
+        }
+        catch (error) {
+            logger.error(`Failed to create provider ${resolvedProviderName}:`, error);
+            throw new Error(`Failed to create provider ${resolvedProviderName}: ${error}`, { cause: error });
+        }
+    }
+    /**
+     * Check if a provider is registered
+     */
+    static hasProvider(providerName) {
+        return ProviderFactory.providers.has(providerName.toLowerCase());
+    }
+    /**
+     * Get list of available providers
+     */
+    static getAvailableProviders() {
+        return Array.from(ProviderFactory.providers.keys()).filter((name, index, arr) => arr.indexOf(name) === index);
+    }
+    /**
+     * Get provider registration info
+     */
+    static getProviderInfo(providerName) {
+        return ProviderFactory.providers.get(providerName.toLowerCase());
+    }
+    /**
+     * Normalize provider names using aliases (PHASE 1: Factory Pattern)
+     */
+    static normalizeProviderName(providerName) {
+        const normalized = providerName.toLowerCase();
+        // Check direct registration
+        if (ProviderFactory.providers.has(normalized)) {
+            return normalized;
+        }
+        // Check aliases from all registrations
+        for (const [name, registration] of ProviderFactory.providers.entries()) {
+            if (registration.aliases?.includes(normalized)) {
+                return name;
+            }
+        }
+        return null;
+    }
+    /**
+     * Clear all registrations (mainly for testing)
+     */
+    static clearRegistrations() {
+        ProviderFactory.providers.clear();
+        ProviderFactory.initialized = false;
+    }
+    /**
+     * Ensure providers are initialized
+     */
+    static ensureInitialized() {
+        if (!ProviderFactory.initialized) {
+            ProviderFactory.initializeDefaultProviders();
+            ProviderFactory.initialized = true;
+        }
+    }
+    /**
+     * Initialize default providers
+     * NOTE: Providers are now registered by ProviderRegistry to avoid circular dependencies
+     */
+    static initializeDefaultProviders() {
+        logger.debug("BaseProvider factory pattern ready - providers registered by ProviderRegistry");
+        // No hardcoded registrations - all done dynamically by ProviderRegistry
+    }
+    /**
+     * Create the best available provider for the given name
+     * Used by NeuroLink SDK for streaming and generation
+     */
+    static async createBestProvider(providerName, modelName, enableMCP, sdk) {
+        return await ProviderFactory.createProvider(providerName, modelName, sdk);
+    }
+}
+/**
+ * Helper function to create providers with backward compatibility
+ */
+export async function createAIProvider(providerName, modelName) {
+    return await ProviderFactory.createProvider(providerName, modelName);
+}