@juspay/neurolink 7.44.0 → 7.46.0

package/dist/evaluation/retryManager.js

@@ -0,0 +1,78 @@
+/**
+ * @file Implements the RetryManager class for handling evaluation retries.
+ */
+/**
+ * Manages the retry logic for the auto-evaluation middleware. It decides if a
+ * retry is warranted based on the evaluation score and prepares the options
+ * for the next generation attempt by incorporating feedback into the prompt.
+ */
+export class RetryManager {
+    maxRetries;
+    constructor(maxRetries = 2) {
+        // Total 3 attempts: 1 initial + 2 retries
+        this.maxRetries = maxRetries;
+    }
+    /**
+     * Determines if a retry should be attempted based on the evaluation result.
+     *
+     * @param evaluation The `EvaluationResult` of the last attempt.
+     * @returns `true` if the response did not pass and the maximum number of retries has not been reached.
+     */
+    shouldRetry(evaluation) {
+        // Attempt number is 1-based. If attempt 1 fails, we can retry.
+        // If attempt 3 (maxRetries + 1) fails, we stop.
+        return !evaluation.isPassing && evaluation.attemptNumber <= this.maxRetries;
+    }
+    /**
+     * Prepares the options for the next generation attempt by creating a new,
+     * improved prompt that includes feedback from the failed evaluation.
+     *
+     * @param originalOptions The original `TextGenerationOptions` from the user request.
+     * @param evaluation The `EvaluationResult` of the failed attempt.
+     * @returns A new `TextGenerationOptions` object with an improved prompt.
+     */
+    prepareRetryOptions(originalOptions, evaluation) {
+        const originalPrompt = originalOptions.prompt || originalOptions.input?.text || "";
+        const newPrompt = this.buildRetryPrompt(originalPrompt, evaluation.suggestedImprovements, evaluation.attemptNumber + 1);
+        // Return a new options object with the updated prompt
+        return {
+            ...originalOptions,
+            prompt: newPrompt,
+            // Ensure input is not carried over if prompt is now the source of truth
+            input: undefined,
+            // Carry over the original prompt for context in subsequent retries if needed
+            originalPrompt: originalOptions.originalPrompt || originalPrompt,
+        };
+    }
+    /**
+     * Builds a new prompt for a retry attempt by incorporating feedback from the
+     * evaluation. The instructions become progressively more direct with each attempt.
+     *
+     * @param originalPrompt The user's original prompt.
+     * @param feedback The constructive feedback from the evaluation.
+     * @param attemptNumber The upcoming attempt number (e.g., 2 for the first retry).
+     * @returns A new, enhanced prompt string.
+     */
+    buildRetryPrompt(originalPrompt, feedback, attemptNumber) {
+        let instruction = "";
+        switch (attemptNumber) {
+            case 2: // First retry
+                instruction = `The previous response was not satisfactory. Please improve it based on the following feedback: "${feedback}".`;
+                break;
+            case 3: // Second retry
+                instruction = `The last response still requires improvement. Pay close attention to this feedback: "${feedback}". You MUST address these points.`;
+                break;
+            default: // Final retry or unexpected attempt number
+                instruction = `This is the final attempt. You MUST address the following feedback to generate a satisfactory response: "${feedback}".`;
+                break;
+        }
+        return `
+    Original Request: ${originalPrompt}
+
+    **Correction Instructions:**
+    ${instruction}
+
+    Generate a new, complete response that incorporates this feedback.
+  `;
+    }
+}

package/dist/evaluation/scoring.d.ts

@@ -0,0 +1,16 @@
+/**
+ * @file Contains the logic for mapping raw evaluation results to the structured EvaluationData type.
+ */
+import type { EnhancedEvaluationContext, EvaluationResult } from "../types/evaluationTypes.js";
+import type { EvaluationData } from "../types/evaluation.js";
+/**
+ * Maps a raw `EvaluationResult` to the structured `EvaluationData` format.
+ * This includes calculating derived fields like `isOffTopic` and `alertSeverity`.
+ *
+ * @param result The raw `EvaluationResult` from the evaluator.
+ * @param threshold The score threshold to determine if the evaluation is passing.
+ * @param offTopicThreshold The score below which a response is considered off-topic.
+ * @param highSeverityThreshold The score below which a failing response is high severity.
+ * @returns A structured `EvaluationData` object.
+ */
+export declare function mapToEvaluationData(evalContext: EnhancedEvaluationContext, result: EvaluationResult, threshold: number, offTopicThreshold?: number, highSeverityThreshold?: number): EvaluationData;

package/dist/evaluation/scoring.js

@@ -0,0 +1,35 @@
+/**
+ * @file Contains the logic for mapping raw evaluation results to the structured EvaluationData type.
+ */
+/**
+ * Maps a raw `EvaluationResult` to the structured `EvaluationData` format.
+ * This includes calculating derived fields like `isOffTopic` and `alertSeverity`.
+ *
+ * @param result The raw `EvaluationResult` from the evaluator.
+ * @param threshold The score threshold to determine if the evaluation is passing.
+ * @param offTopicThreshold The score below which a response is considered off-topic.
+ * @param highSeverityThreshold The score below which a failing response is high severity.
+ * @returns A structured `EvaluationData` object.
+ */
+export function mapToEvaluationData(evalContext, result, threshold, offTopicThreshold = 5, highSeverityThreshold = 4) {
+    const isPassing = result.finalScore >= threshold;
+    return {
+        relevance: result.relevanceScore,
+        accuracy: result.accuracyScore,
+        completeness: result.completenessScore,
+        overall: result.finalScore,
+        isOffTopic: result.finalScore < offTopicThreshold,
+        alertSeverity: isPassing
+            ? "none"
+            : result.finalScore < highSeverityThreshold
+                ? "high"
+                : "medium",
+        reasoning: result.reasoning,
+        suggestedImprovements: result.suggestedImprovements,
+        evaluationModel: result.evaluationModel,
+        evaluationTime: result.evaluationTime,
+        evaluationAttempt: result.attemptNumber,
+        responseContent: evalContext.aiResponse,
+        queryContent: evalContext.userQuery,
+    };
+}

package/dist/factories/providerFactory.d.ts

@@ -4,8 +4,8 @@ import type { UnknownRecord } from "../types/common.js";
  * Provider constructor interface - supports both sync constructors and async factory functions
  */
 type ProviderConstructor = {
-    new (modelName?: string, providerName?: string, sdk?: UnknownRecord): AIProvider;
-} | ((modelName?: string, providerName?: string, sdk?: UnknownRecord) => Promise<AIProvider>);
+    new (modelName?: string, providerName?: string, sdk?: UnknownRecord, region?: string): AIProvider;
+} | ((modelName?: string, providerName?: string, sdk?: UnknownRecord, region?: string) => Promise<AIProvider>);
 /**
  * Provider registration entry
  */
@@ -30,7 +30,7 @@ export declare class ProviderFactory {
     /**
      * Create a provider instance
      */
-    static createProvider(providerName: AIProviderName | string, modelName?: string, sdk?: UnknownRecord): Promise<AIProvider>;
+    static createProvider(providerName: AIProviderName | string, modelName?: string, sdk?: UnknownRecord, region?: string): Promise<AIProvider>;
     /**
      * Check if a provider is registered
      */

package/dist/factories/providerFactory.js

@@ -28,7 +28,7 @@ export class ProviderFactory {
     /**
      * Create a provider instance
      */
-    static async createProvider(providerName, modelName, sdk) {
+    static async createProvider(providerName, modelName, sdk, region) {
         // Note: Providers are registered explicitly by ProviderRegistry to avoid circular dependencies
         const normalizedName = providerName.toLowerCase();
         const registration = this.providers.get(normalizedName);
@@ -54,7 +54,7 @@ export class ProviderFactory {
             }
             let result;
             try {
-                const factoryResult = registration.constructor(model, providerName, sdk);
+                const factoryResult = registration.constructor(model, providerName, sdk, region);
                 // Handle both sync and async results
                 result =
                     factoryResult instanceof Promise
@@ -66,7 +66,7 @@ export class ProviderFactory {
                     registration.constructor.prototype.constructor ===
                         registration.constructor) {
                     try {
-                        result = new registration.constructor(model, providerName, sdk);
+                        result = new registration.constructor(model, providerName, sdk, region);
                     }
                     catch (constructorError) {
                         throw new Error(`Both factory function and constructor failed. Factory error: ${factoryError}. Constructor error: ${constructorError}`);

package/dist/factories/providerRegistry.js

@@ -39,9 +39,9 @@ export class ProviderRegistry {
                 return new AnthropicProvider(modelName, sdk);
             }, "claude-3-5-sonnet-20241022", ["claude", "anthropic"]);
             // Register Amazon Bedrock provider
-            ProviderFactory.registerProvider(AIProviderName.BEDROCK, async (modelName, _providerName, sdk) => {
+            ProviderFactory.registerProvider(AIProviderName.BEDROCK, async (modelName, _providerName, sdk, region) => {
                 const { AmazonBedrockProvider } = await import("../providers/amazonBedrock.js");
-                return new AmazonBedrockProvider(modelName, sdk);
+                return new AmazonBedrockProvider(modelName, sdk, region);
             }, undefined, // Let provider read BEDROCK_MODEL from .env
             ["bedrock", "aws"]);
             // Register Azure OpenAI provider
@@ -54,9 +54,9 @@ export class ProviderRegistry {
                 process.env.AZURE_OPENAI_DEPLOYMENT_ID ||
                 "gpt-4o-mini", ["azure", "azureOpenai"]);
             // Register Google Vertex AI provider
-            ProviderFactory.registerProvider(AIProviderName.VERTEX, async (modelName, providerName, sdk) => {
+            ProviderFactory.registerProvider(AIProviderName.VERTEX, async (modelName, providerName, sdk, region) => {
                 const { GoogleVertexProvider } = await import("../providers/googleVertex.js");
-                return new GoogleVertexProvider(modelName, providerName, sdk);
+                return new GoogleVertexProvider(modelName, providerName, sdk, region);
             }, "claude-sonnet-4@20250514", ["vertex", "googleVertex"]);
             // Register Hugging Face provider (Unified Router implementation)
             ProviderFactory.registerProvider(AIProviderName.HUGGINGFACE, async (modelName) => {
@@ -85,9 +85,9 @@ export class ProviderRegistry {
             }, process.env.OPENAI_COMPATIBLE_MODEL || undefined, // Enable auto-discovery when no model specified
             ["openai-compatible", "openrouter", "vllm", "compatible"]);
             // Register Amazon SageMaker provider
-            ProviderFactory.registerProvider(AIProviderName.SAGEMAKER, async (modelName, _providerName, _sdk) => {
+            ProviderFactory.registerProvider(AIProviderName.SAGEMAKER, async (modelName, _providerName, _sdk, region) => {
                 const { AmazonSageMakerProvider } = await import("../providers/amazonSagemaker.js");
-                return new AmazonSageMakerProvider(modelName);
+                return new AmazonSageMakerProvider(modelName, region);
             }, process.env.SAGEMAKER_MODEL || "sagemaker-model", ["sagemaker", "aws-sagemaker"]);
             logger.debug("All providers registered successfully");
             this.registered = true;

package/dist/lib/core/factory.d.ts

@@ -19,9 +19,11 @@ export declare class AIProviderFactory {
      * @param providerName - Name of the provider ('vertex', 'bedrock', 'openai')
      * @param modelName - Optional model name override
      * @param enableMCP - Optional flag to enable MCP integration (default: true)
+     * @param sdk - SDK instance
+     * @param region - Optional region override for cloud providers
      * @returns AIProvider instance
      */
-    static createProvider(providerName: string, modelName?: string | null, enableMCP?: boolean, sdk?: UnknownRecord): Promise<AIProvider>;
+    static createProvider(providerName: string, modelName?: string | null, enableMCP?: boolean, sdk?: UnknownRecord, region?: string): Promise<AIProvider>;
     /**
      * Create a provider instance with specific provider enum and model
      * @param provider - Provider enum value

package/dist/lib/core/factory.js

@@ -52,9 +52,11 @@ export class AIProviderFactory {
      * @param providerName - Name of the provider ('vertex', 'bedrock', 'openai')
      * @param modelName - Optional model name override
      * @param enableMCP - Optional flag to enable MCP integration (default: true)
+     * @param sdk - SDK instance
+     * @param region - Optional region override for cloud providers
      * @returns AIProvider instance
      */
-    static async createProvider(providerName, modelName, enableMCP = true, sdk) {
+    static async createProvider(providerName, modelName, enableMCP = true, sdk, region) {
         const functionTag = "AIProviderFactory.createProvider";
         // Providers are registered via ProviderFactory.initialize() on first use
         logger.debug(`[${functionTag}] Provider creation started`, {
@@ -198,8 +200,8 @@ export class AIProviderFactory {
                 resolvedModelName: resolvedModelName || "not resolved",
                 finalModelName: finalModelName || "using provider default",
             });
-            // Create provider with enhanced SDK
-            const provider = await ProviderFactory.createProvider(normalizedName, finalModelName, sdk);
+            // Create provider with enhanced SDK and region support
+            const provider = await ProviderFactory.createProvider(normalizedName, finalModelName, sdk, region);
             // Summary logging in format expected by debugging tools
             logger.debug(`[AIProviderFactory] Provider creation completed { providerName: '${normalizedName}', modelName: '${finalModelName}' }`);
             logger.debug(`[AIProviderFactory] Resolved model: ${finalModelName}`);

package/dist/lib/evaluation/contextBuilder.d.ts

@@ -0,0 +1,48 @@
+/**
+ * @file Implements the ContextBuilder class for creating rich evaluation context.
+ */
+import type { EnhancedEvaluationContext, EvaluationResult } from "../types/evaluationTypes.js";
+import type { GenerateResult } from "../types/generateTypes.js";
+import type { LanguageModelV1CallOptions } from "ai";
+/**
+ * Builds the enhanced context required for a RAGAS-style evaluation.
+ * This class gathers data from the generation options and results to create a
+ * rich snapshot of the interaction, which is then used by the evaluator.
+ */
+export declare class ContextBuilder {
+    private attemptNumber;
+    private previousEvaluations;
+    private extractTextFromContent;
+    /**
+     * Builds the full evaluation context for a single evaluation attempt.
+     *
+     * @param options The original `TextGenerationOptions` used for the request.
+     * @param result The `GenerateResult` from the provider.
+     * @returns An `EnhancedEvaluationContext` object ready for evaluation.
+     */
+    buildContext(options: LanguageModelV1CallOptions, result: GenerateResult): EnhancedEvaluationContext;
+    /**
+     * Records the result of an evaluation and increments the internal attempt counter.
+     * This is used to build up the `previousEvaluations` array for subsequent retries.
+     *
+     * @param evaluation The `EvaluationResult` from the last attempt.
+     */
+    recordEvaluation(evaluation: EvaluationResult): void;
+    /**
+     * Resets the internal state of the context builder. This should be called
+     * before starting a new, independent evaluation sequence.
+     */
+    reset(): void;
+    /**
+     * Analyzes the user's query to determine intent and complexity.
+     * @param query The user's input query.
+     * @returns A QueryIntentAnalysis object.
+     */
+    private analyzeQuery;
+    /**
+     * Maps the tool execution format from GenerateResult to the canonical ToolExecution type.
+     * @param result The result from the generate call.
+     * @returns An array of ToolExecution objects.
+     */
+    private mapToolExecutions;
+}

package/dist/lib/evaluation/contextBuilder.js

@@ -0,0 +1,134 @@
+/**
+ * @file Implements the ContextBuilder class for creating rich evaluation context.
+ */
+import { logger } from "../utils/logger.js";
+/**
+ * Builds the enhanced context required for a RAGAS-style evaluation.
+ * This class gathers data from the generation options and results to create a
+ * rich snapshot of the interaction, which is then used by the evaluator.
+ */
+export class ContextBuilder {
+    attemptNumber = 1;
+    previousEvaluations = [];
+    extractTextFromContent(content) {
+        if (typeof content === "string") {
+            return content;
+        }
+        if (Array.isArray(content)) {
+            return content
+                .filter((part) => part.type === "text" && "text" in part)
+                .map((part) => part.text)
+                .join("");
+        }
+        return "";
+    }
+    /**
+     * Builds the full evaluation context for a single evaluation attempt.
+     *
+     * @param options The original `TextGenerationOptions` used for the request.
+     * @param result The `GenerateResult` from the provider.
+     * @returns An `EnhancedEvaluationContext` object ready for evaluation.
+     */
+    buildContext(options, result) {
+        const userMessages = options.prompt.filter((p) => p.role === "user");
+        const lastUserMessage = userMessages[userMessages.length - 1];
+        const userQuery = this.extractTextFromContent(lastUserMessage?.content ?? "");
+        const systemPromptMessage = options.prompt.find((p) => p.role === "system");
+        const systemPrompt = this.extractTextFromContent(systemPromptMessage?.content ?? "");
+        const queryAnalysis = this.analyzeQuery(userQuery);
+        const toolExecutions = this.mapToolExecutions(result);
+        const context = {
+            userQuery,
+            queryAnalysis,
+            aiResponse: result.content,
+            provider: result.provider || "unknown",
+            model: result.model || "unknown",
+            generationParams: {
+                temperature: options.temperature,
+                maxTokens: options.maxTokens,
+                systemPrompt: systemPrompt || undefined,
+            },
+            toolExecutions,
+            conversationHistory: (options.prompt || [])
+                .filter((p) => p.role !== "system")
+                .map((turn) => ({
+                role: turn.role,
+                content: this.extractTextFromContent(turn.content),
+                timestamp: new Date().toISOString(),
+            })),
+            responseTime: result.responseTime || 0,
+            tokenUsage: result.usage || { input: 0, output: 0, total: 0 },
+            previousEvaluations: this.previousEvaluations,
+            attemptNumber: this.attemptNumber,
+        };
+        logger.debug("Built Evaluation Context:", context);
+        return context;
+    }
+    /**
+     * Records the result of an evaluation and increments the internal attempt counter.
+     * This is used to build up the `previousEvaluations` array for subsequent retries.
+     *
+     * @param evaluation The `EvaluationResult` from the last attempt.
+     */
+    recordEvaluation(evaluation) {
+        this.previousEvaluations.push(evaluation);
+        this.attemptNumber++;
+    }
+    /**
+     * Resets the internal state of the context builder. This should be called
+     * before starting a new, independent evaluation sequence.
+     */
+    reset() {
+        this.attemptNumber = 1;
+        this.previousEvaluations = [];
+    }
+    /**
+     * Analyzes the user's query to determine intent and complexity.
+     * @param query The user's input query.
+     * @returns A QueryIntentAnalysis object.
+     */
+    analyzeQuery(query) {
+        const lowerCaseQuery = query.toLowerCase();
+        let type = "unknown";
+        if (lowerCaseQuery.startsWith("what") ||
+            lowerCaseQuery.startsWith("how") ||
+            lowerCaseQuery.startsWith("why")) {
+            type = "question";
+        }
+        else if (lowerCaseQuery.length < 20) {
+            type = "greeting";
+        }
+        else {
+            type = "command";
+        }
+        const complexity = query.length > 100 ? "high" : query.length > 40 ? "medium" : "low";
+        return {
+            type,
+            complexity,
+            shouldHaveUsedTools: false, // This would require deeper analysis
+        };
+    }
+    /**
+     * Maps the tool execution format from GenerateResult to the canonical ToolExecution type.
+     * @param result The result from the generate call.
+     * @returns An array of ToolExecution objects.
+     */
+    mapToolExecutions(result) {
+        if (!result.toolExecutions) {
+            return [];
+        }
+        return result.toolExecutions.map((exec) => {
+            const toolResult = {
+                success: true,
+                data: exec.output,
+            };
+            return {
+                toolName: exec.name,
+                params: exec.input,
+                result: toolResult,
+                executionTime: 0,
+                timestamp: Date.now(),
+            };
+        });
+    }
+}

package/dist/lib/evaluation/index.d.ts

@@ -0,0 +1,36 @@
+/**
+ * @file This file exports the main Evaluator class, which serves as the central entry point for the evaluation system.
+ */
+import type { GenerateResult } from "../types/generateTypes.js";
+import type { EvaluationConfig } from "../types/evaluationTypes.js";
+import type { LanguageModelV1CallOptions } from "ai";
+import type { AutoEvaluationConfig } from "../types/middlewareTypes.js";
+import type { EvaluationData } from "../types/evaluation.js";
+/**
+ * A centralized class for performing response evaluations. It supports different
+ * evaluation strategies, with RAGAS-style model-based evaluation as the default.
+ * This class orchestrates the context building and evaluation process.
+ */
+export declare class Evaluator {
+    private contextBuilder;
+    private config;
+    private ragasEvaluator;
+    constructor(config?: EvaluationConfig);
+    /**
+     * The main entry point for performing an evaluation. It selects the evaluation
+     * strategy based on the configuration and executes it.
+     *
+     * @param options The original `TextGenerationOptions` from the user request.
+     * @param result The `GenerateResult` from the provider.
+     * @returns A promise that resolves to the `EvaluationResult`.
+     */
+    evaluate(options: LanguageModelV1CallOptions, result: GenerateResult, threshold: number, config: AutoEvaluationConfig): Promise<EvaluationData>;
+    /**
+     * Performs evaluation using the RAGAS-style model-based evaluator.
+     *
+     * @param options The original `TextGenerationOptions`.
+     * @param result The `GenerateResult` to be evaluated.
+     * @returns A promise that resolves to the `EvaluationResult`.
+     */
+    private evaluateWithRAGAS;
+}

package/dist/lib/evaluation/index.js

@@ -0,0 +1,61 @@
+/**
+ * @file This file exports the main Evaluator class, which serves as the central entry point for the evaluation system.
+ */
+import { ContextBuilder } from "./contextBuilder.js";
+import { RAGASEvaluator } from "./ragasEvaluator.js";
+import { mapToEvaluationData } from "./scoring.js";
+/**
+ * A centralized class for performing response evaluations. It supports different
+ * evaluation strategies, with RAGAS-style model-based evaluation as the default.
+ * This class orchestrates the context building and evaluation process.
+ */
+export class Evaluator {
+    contextBuilder;
+    config;
+    ragasEvaluator;
+    constructor(config = {}) {
+        this.config = config;
+        this.contextBuilder = new ContextBuilder();
+        this.ragasEvaluator = new RAGASEvaluator(this.config.evaluationModel, this.config.provider, this.config.threshold, this.config.promptGenerator);
+    }
+    /**
+     * The main entry point for performing an evaluation. It selects the evaluation
+     * strategy based on the configuration and executes it.
+     *
+     * @param options The original `TextGenerationOptions` from the user request.
+     * @param result The `GenerateResult` from the provider.
+     * @returns A promise that resolves to the `EvaluationResult`.
+     */
+    async evaluate(options, result, threshold, config) {
+        const evaluationStrategy = this.config.evaluationStrategy || "ragas";
+        const customEvaluator = this.config.customEvaluator;
+        switch (evaluationStrategy) {
+            case "ragas": {
+                const { evaluationResult, evalContext } = await this.evaluateWithRAGAS(options, result);
+                const evaluationData = mapToEvaluationData(evalContext, evaluationResult, threshold, config.offTopicThreshold, config.highSeverityThreshold);
+                return evaluationData;
+            }
+            case "custom": {
+                if (customEvaluator) {
+                    const { evaluationResult, evalContext } = await customEvaluator(options, result);
+                    return mapToEvaluationData(evalContext, evaluationResult, threshold, config.offTopicThreshold, config.highSeverityThreshold);
+                }
+                throw new Error("Custom evaluator function not provided in config.");
+            }
+            default:
+                throw new Error(`Unsupported evaluation strategy: ${evaluationStrategy} `);
+        }
+    }
+    /**
+     * Performs evaluation using the RAGAS-style model-based evaluator.
+     *
+     * @param options The original `TextGenerationOptions`.
+     * @param result The `GenerateResult` to be evaluated.
+     * @returns A promise that resolves to the `EvaluationResult`.
+     */
+    async evaluateWithRAGAS(options, result) {
+        const evalContext = this.contextBuilder.buildContext(options, result);
+        const evaluationResult = await this.ragasEvaluator.evaluate(evalContext);
+        return { evaluationResult, evalContext };
+    }
+}

package/dist/lib/evaluation/prompts.d.ts

@@ -0,0 +1,22 @@
+import type { EnhancedEvaluationContext, GetPromptFunction } from "../types/evaluationTypes.js";
+/**
+ * A flexible class for building evaluation prompts. It allows for custom prompt
+ * generation logic to be injected while ensuring a consistent output format.
+ */
+export declare class PromptBuilder {
+    /**
+     * Builds the final evaluation prompt.
+     *
+     * @param context The rich context for the evaluation.
+     * @param getPrompt An optional function to generate the main body of the prompt.
+     *                  If not provided, a default prompt is used.
+     * @returns The complete prompt string to be sent to the judge LLM.
+     */
+    buildEvaluationPrompt(context: EnhancedEvaluationContext, getPrompt?: GetPromptFunction): string;
+    /**
+     * The default prompt generation logic.
+     * @param context The prepared context strings.
+     * @returns The default prompt body.
+     */
+    private getDefaultPrompt;
+}

package/dist/lib/evaluation/prompts.js

@@ -0,0 +1,73 @@
+/**
+ * A flexible class for building evaluation prompts. It allows for custom prompt
+ * generation logic to be injected while ensuring a consistent output format.
+ */
+export class PromptBuilder {
+    /**
+     * Builds the final evaluation prompt.
+     *
+     * @param context The rich context for the evaluation.
+     * @param getPrompt An optional function to generate the main body of the prompt.
+     *                  If not provided, a default prompt is used.
+     * @returns The complete prompt string to be sent to the judge LLM.
+     */
+    buildEvaluationPrompt(context, getPrompt) {
+        const { userQuery, aiResponse, conversationHistory, toolExecutions, previousEvaluations, } = context;
+        const historyStr = conversationHistory
+            .map((turn) => `${turn.role}: ${turn.content}`)
+            .join("\n");
+        const toolsStr = toolExecutions.length > 0
+            ? `Tools were used: ${toolExecutions.map((t) => t.toolName).join(", ")}`
+            : "No tools were used.";
+        const retryStr = previousEvaluations && previousEvaluations.length > 0
+            ? `This is attempt #${context.attemptNumber}. Previous reasoning: ${previousEvaluations
+                .map((e) => e.reasoning)
+                .join("; ")} Previous suggested Improvements: ${previousEvaluations.map((e) => e.suggestedImprovements).join("; ")}`
+            : "This is the first attempt.";
+        const promptContext = {
+            userQuery,
+            history: historyStr,
+            tools: toolsStr,
+            retryInfo: retryStr,
+            aiResponse,
+        };
+        const mainPrompt = getPrompt
+            ? getPrompt(promptContext)
+            : this.getDefaultPrompt(promptContext);
+        return `
+      ${mainPrompt}
+
+      **Output Format (JSON):**
+      {
+        "relevanceScore": <1-10>,
+        "accuracyScore": <1-10>,
+        "completenessScore": <1-10>,
+        "finalScore": <1-10>,
+        "reasoning": "<Your constructive reasoning here>",
+        "suggestedImprovements": "<How the response can be improved>"
+      }
+    `;
+    }
+    /**
+     * The default prompt generation logic.
+     * @param context The prepared context strings.
+     * @returns The default prompt body.
+     */
+    getDefaultPrompt(context) {
+        return `
+      You are an expert AI quality evaluator. Your task is to evaluate the AI assistant's response based on the provided context.
+      Provide a score from 1 to 10 for each of the following criteria: Relevance, Accuracy, and Completeness.
+      Finally, provide an overall finalScore and constructive feedback for improvement.
+
+      **Evaluation Context:**
+      - User Query: ${context.userQuery}
+      - Conversation History:
+      ${context.history}
+      - Tools Executed: ${context.tools}
+      - Retry Information: ${context.retryInfo}
+
+      **AI Assistant's Response to Evaluate:**
+      ${context.aiResponse}
+    `;
+    }
+}

package/dist/lib/evaluation/ragasEvaluator.d.ts

@@ -0,0 +1,28 @@
+import type { EnhancedEvaluationContext, EvaluationResult, GetPromptFunction } from "../types/evaluationTypes.js";
+/**
+ * Implements a RAGAS-style evaluator that uses a "judge" LLM to score the
+ * quality of an AI response based on rich, contextual information.
+ */
+export declare class RAGASEvaluator {
+    private evaluationModel;
+    private providerName;
+    private threshold;
+    private promptBuilder;
+    private promptGenerator?;
+    constructor(evaluationModel?: string, providerName?: string, threshold?: number, promptGenerator?: GetPromptFunction);
+    /**
+     * Evaluates an AI-generated response using a model-based approach.
+     *
+     * @param context The rich, contextual information for the evaluation.
+     * @returns A promise that resolves to a detailed `EvaluationResult`.
+     */
+    evaluate(context: EnhancedEvaluationContext): Promise<EvaluationResult>;
+    /**
+     * Parses the raw JSON string from the judge LLM into a structured `EvaluationResult` object.
+     * It includes error handling to gracefully manage malformed JSON.
+     *
+     * @param rawResponse The raw string response from the evaluation model.
+     * @returns A structured object containing the evaluation scores and feedback.
+     */
+    private parseEvaluationResponse;
+}