@juspay/neurolink 7.44.0 → 7.46.0

package/CHANGELOG.md

@@ -1,3 +1,15 @@
+## [7.46.0](https://github.com/juspay/neurolink/compare/v7.45.0...v7.46.0) (2025-09-24)
+
+### Features
+
+- **(auto-evaluation):** added auto evaluation for LLM response ([6f23fae](https://github.com/juspay/neurolink/commit/6f23fae5cacb1c0686257cc7ed547be675b68b23))
+
+## [7.45.0](https://github.com/juspay/neurolink/compare/v7.44.0...v7.45.0) (2025-09-24)
+
+### Features
+
+- **(provider):** Add support to provide region while streaming or generating for few providers ([a0a5bed](https://github.com/juspay/neurolink/commit/a0a5bed2bba4118dde149713708e36d4d29e1aae))
+
 ## [7.44.0](https://github.com/juspay/neurolink/compare/v7.43.0...v7.44.0) (2025-09-24)
 
 ### Features

package/dist/cli/commands/config.d.ts

@@ -256,8 +256,8 @@ declare const ConfigSchema: z.ZodObject<{
         maxTokens?: number | undefined;
         defaultEvaluationDomain?: string | undefined;
     }, {
-        maxTokens?: number | undefined;
         temperature?: number | undefined;
+        maxTokens?: number | undefined;
         outputFormat?: "text" | "json" | "yaml" | undefined;
         enableLogging?: boolean | undefined;
         enableCaching?: boolean | undefined;
@@ -606,8 +606,8 @@ declare const ConfigSchema: z.ZodObject<{
     defaultProvider?: "openai" | "anthropic" | "vertex" | "google-ai" | "auto" | "bedrock" | "azure" | "ollama" | "huggingface" | "mistral" | undefined;
     profiles?: Record<string, any> | undefined;
     preferences?: {
-        maxTokens?: number | undefined;
         temperature?: number | undefined;
+        maxTokens?: number | undefined;
         outputFormat?: "text" | "json" | "yaml" | undefined;
         enableLogging?: boolean | undefined;
         enableCaching?: boolean | undefined;

package/dist/cli/loop/optionsSchema.d.ts

@@ -12,4 +12,4 @@ export interface OptionSchema {
  * This object provides metadata for validation and help text in the CLI loop.
  * It is derived from the main TextGenerationOptions interface to ensure consistency.
  */
-export declare const textGenerationOptionsSchema: Record<keyof Omit<TextGenerationOptions, "prompt" | "input" | "schema" | "tools" | "context" | "conversationHistory" | "conversationMessages" | "conversationMemoryConfig" | "originalPrompt" | "middleware" | "expectedOutcome" | "evaluationCriteria">, OptionSchema>;
+export declare const textGenerationOptionsSchema: Record<keyof Omit<TextGenerationOptions, "prompt" | "input" | "schema" | "tools" | "context" | "conversationHistory" | "conversationMessages" | "conversationMemoryConfig" | "originalPrompt" | "middleware" | "expectedOutcome" | "evaluationCriteria" | "region">, OptionSchema>;

package/dist/core/factory.d.ts

@@ -19,9 +19,11 @@ export declare class AIProviderFactory {
      * @param providerName - Name of the provider ('vertex', 'bedrock', 'openai')
      * @param modelName - Optional model name override
      * @param enableMCP - Optional flag to enable MCP integration (default: true)
+     * @param sdk - SDK instance
+     * @param region - Optional region override for cloud providers
      * @returns AIProvider instance
      */
-    static createProvider(providerName: string, modelName?: string | null, enableMCP?: boolean, sdk?: UnknownRecord): Promise<AIProvider>;
+    static createProvider(providerName: string, modelName?: string | null, enableMCP?: boolean, sdk?: UnknownRecord, region?: string): Promise<AIProvider>;
     /**
      * Create a provider instance with specific provider enum and model
      * @param provider - Provider enum value

package/dist/core/factory.js

@@ -52,9 +52,11 @@ export class AIProviderFactory {
      * @param providerName - Name of the provider ('vertex', 'bedrock', 'openai')
      * @param modelName - Optional model name override
      * @param enableMCP - Optional flag to enable MCP integration (default: true)
+     * @param sdk - SDK instance
+     * @param region - Optional region override for cloud providers
      * @returns AIProvider instance
      */
-    static async createProvider(providerName, modelName, enableMCP = true, sdk) {
+    static async createProvider(providerName, modelName, enableMCP = true, sdk, region) {
         const functionTag = "AIProviderFactory.createProvider";
         // Providers are registered via ProviderFactory.initialize() on first use
         logger.debug(`[${functionTag}] Provider creation started`, {
@@ -198,8 +200,8 @@ export class AIProviderFactory {
                 resolvedModelName: resolvedModelName || "not resolved",
                 finalModelName: finalModelName || "using provider default",
             });
-            // Create provider with enhanced SDK
-            const provider = await ProviderFactory.createProvider(normalizedName, finalModelName, sdk);
+            // Create provider with enhanced SDK and region support
+            const provider = await ProviderFactory.createProvider(normalizedName, finalModelName, sdk, region);
             // Summary logging in format expected by debugging tools
             logger.debug(`[AIProviderFactory] Provider creation completed { providerName: '${normalizedName}', modelName: '${finalModelName}' }`);
             logger.debug(`[AIProviderFactory] Resolved model: ${finalModelName}`);

package/dist/evaluation/contextBuilder.d.ts

@@ -0,0 +1,48 @@
+/**
+ * @file Implements the ContextBuilder class for creating rich evaluation context.
+ */
+import type { EnhancedEvaluationContext, EvaluationResult } from "../types/evaluationTypes.js";
+import type { GenerateResult } from "../types/generateTypes.js";
+import type { LanguageModelV1CallOptions } from "ai";
+/**
+ * Builds the enhanced context required for a RAGAS-style evaluation.
+ * This class gathers data from the generation options and results to create a
+ * rich snapshot of the interaction, which is then used by the evaluator.
+ */
+export declare class ContextBuilder {
+    private attemptNumber;
+    private previousEvaluations;
+    private extractTextFromContent;
+    /**
+     * Builds the full evaluation context for a single evaluation attempt.
+     *
+     * @param options The original `TextGenerationOptions` used for the request.
+     * @param result The `GenerateResult` from the provider.
+     * @returns An `EnhancedEvaluationContext` object ready for evaluation.
+     */
+    buildContext(options: LanguageModelV1CallOptions, result: GenerateResult): EnhancedEvaluationContext;
+    /**
+     * Records the result of an evaluation and increments the internal attempt counter.
+     * This is used to build up the `previousEvaluations` array for subsequent retries.
+     *
+     * @param evaluation The `EvaluationResult` from the last attempt.
+     */
+    recordEvaluation(evaluation: EvaluationResult): void;
+    /**
+     * Resets the internal state of the context builder. This should be called
+     * before starting a new, independent evaluation sequence.
+     */
+    reset(): void;
+    /**
+     * Analyzes the user's query to determine intent and complexity.
+     * @param query The user's input query.
+     * @returns A QueryIntentAnalysis object.
+     */
+    private analyzeQuery;
+    /**
+     * Maps the tool execution format from GenerateResult to the canonical ToolExecution type.
+     * @param result The result from the generate call.
+     * @returns An array of ToolExecution objects.
+     */
+    private mapToolExecutions;
+}

package/dist/evaluation/contextBuilder.js

@@ -0,0 +1,134 @@
+/**
+ * @file Implements the ContextBuilder class for creating rich evaluation context.
+ */
+import { logger } from "../utils/logger.js";
+/**
+ * Builds the enhanced context required for a RAGAS-style evaluation.
+ * This class gathers data from the generation options and results to create a
+ * rich snapshot of the interaction, which is then used by the evaluator.
+ */
+export class ContextBuilder {
+    attemptNumber = 1;
+    previousEvaluations = [];
+    extractTextFromContent(content) {
+        if (typeof content === "string") {
+            return content;
+        }
+        if (Array.isArray(content)) {
+            return content
+                .filter((part) => part.type === "text" && "text" in part)
+                .map((part) => part.text)
+                .join("");
+        }
+        return "";
+    }
+    /**
+     * Builds the full evaluation context for a single evaluation attempt.
+     *
+     * @param options The original `TextGenerationOptions` used for the request.
+     * @param result The `GenerateResult` from the provider.
+     * @returns An `EnhancedEvaluationContext` object ready for evaluation.
+     */
+    buildContext(options, result) {
+        const userMessages = options.prompt.filter((p) => p.role === "user");
+        const lastUserMessage = userMessages[userMessages.length - 1];
+        const userQuery = this.extractTextFromContent(lastUserMessage?.content ?? "");
+        const systemPromptMessage = options.prompt.find((p) => p.role === "system");
+        const systemPrompt = this.extractTextFromContent(systemPromptMessage?.content ?? "");
+        const queryAnalysis = this.analyzeQuery(userQuery);
+        const toolExecutions = this.mapToolExecutions(result);
+        const context = {
+            userQuery,
+            queryAnalysis,
+            aiResponse: result.content,
+            provider: result.provider || "unknown",
+            model: result.model || "unknown",
+            generationParams: {
+                temperature: options.temperature,
+                maxTokens: options.maxTokens,
+                systemPrompt: systemPrompt || undefined,
+            },
+            toolExecutions,
+            conversationHistory: (options.prompt || [])
+                .filter((p) => p.role !== "system")
+                .map((turn) => ({
+                role: turn.role,
+                content: this.extractTextFromContent(turn.content),
+                timestamp: new Date().toISOString(),
+            })),
+            responseTime: result.responseTime || 0,
+            tokenUsage: result.usage || { input: 0, output: 0, total: 0 },
+            previousEvaluations: this.previousEvaluations,
+            attemptNumber: this.attemptNumber,
+        };
+        logger.debug("Built Evaluation Context:", context);
+        return context;
+    }
+    /**
+     * Records the result of an evaluation and increments the internal attempt counter.
+     * This is used to build up the `previousEvaluations` array for subsequent retries.
+     *
+     * @param evaluation The `EvaluationResult` from the last attempt.
+     */
+    recordEvaluation(evaluation) {
+        this.previousEvaluations.push(evaluation);
+        this.attemptNumber++;
+    }
+    /**
+     * Resets the internal state of the context builder. This should be called
+     * before starting a new, independent evaluation sequence.
+     */
+    reset() {
+        this.attemptNumber = 1;
+        this.previousEvaluations = [];
+    }
+    /**
+     * Analyzes the user's query to determine intent and complexity.
+     * @param query The user's input query.
+     * @returns A QueryIntentAnalysis object.
+     */
+    analyzeQuery(query) {
+        const lowerCaseQuery = query.toLowerCase();
+        let type = "unknown";
+        if (lowerCaseQuery.startsWith("what") ||
+            lowerCaseQuery.startsWith("how") ||
+            lowerCaseQuery.startsWith("why")) {
+            type = "question";
+        }
+        else if (lowerCaseQuery.length < 20) {
+            type = "greeting";
+        }
+        else {
+            type = "command";
+        }
+        const complexity = query.length > 100 ? "high" : query.length > 40 ? "medium" : "low";
+        return {
+            type,
+            complexity,
+            shouldHaveUsedTools: false, // This would require deeper analysis
+        };
+    }
+    /**
+     * Maps the tool execution format from GenerateResult to the canonical ToolExecution type.
+     * @param result The result from the generate call.
+     * @returns An array of ToolExecution objects.
+     */
+    mapToolExecutions(result) {
+        if (!result.toolExecutions) {
+            return [];
+        }
+        return result.toolExecutions.map((exec) => {
+            const toolResult = {
+                success: true,
+                data: exec.output,
+            };
+            return {
+                toolName: exec.name,
+                params: exec.input,
+                result: toolResult,
+                executionTime: 0,
+                timestamp: Date.now(),
+            };
+        });
+    }
+}

package/dist/evaluation/index.d.ts

@@ -0,0 +1,36 @@
+/**
+ * @file This file exports the main Evaluator class, which serves as the central entry point for the evaluation system.
+ */
+import type { GenerateResult } from "../types/generateTypes.js";
+import type { EvaluationConfig } from "../types/evaluationTypes.js";
+import type { LanguageModelV1CallOptions } from "ai";
+import type { AutoEvaluationConfig } from "../types/middlewareTypes.js";
+import type { EvaluationData } from "../types/evaluation.js";
+/**
+ * A centralized class for performing response evaluations. It supports different
+ * evaluation strategies, with RAGAS-style model-based evaluation as the default.
+ * This class orchestrates the context building and evaluation process.
+ */
+export declare class Evaluator {
+    private contextBuilder;
+    private config;
+    private ragasEvaluator;
+    constructor(config?: EvaluationConfig);
+    /**
+     * The main entry point for performing an evaluation. It selects the evaluation
+     * strategy based on the configuration and executes it.
+     *
+     * @param options The original `TextGenerationOptions` from the user request.
+     * @param result The `GenerateResult` from the provider.
+     * @returns A promise that resolves to the `EvaluationResult`.
+     */
+    evaluate(options: LanguageModelV1CallOptions, result: GenerateResult, threshold: number, config: AutoEvaluationConfig): Promise<EvaluationData>;
+    /**
+     * Performs evaluation using the RAGAS-style model-based evaluator.
+     *
+     * @param options The original `TextGenerationOptions`.
+     * @param result The `GenerateResult` to be evaluated.
+     * @returns A promise that resolves to the `EvaluationResult`.
+     */
+    private evaluateWithRAGAS;
+}

package/dist/evaluation/index.js

@@ -0,0 +1,61 @@
+/**
+ * @file This file exports the main Evaluator class, which serves as the central entry point for the evaluation system.
+ */
+import { ContextBuilder } from "./contextBuilder.js";
+import { RAGASEvaluator } from "./ragasEvaluator.js";
+import { mapToEvaluationData } from "./scoring.js";
+/**
+ * A centralized class for performing response evaluations. It supports different
+ * evaluation strategies, with RAGAS-style model-based evaluation as the default.
+ * This class orchestrates the context building and evaluation process.
+ */
+export class Evaluator {
+    contextBuilder;
+    config;
+    ragasEvaluator;
+    constructor(config = {}) {
+        this.config = config;
+        this.contextBuilder = new ContextBuilder();
+        this.ragasEvaluator = new RAGASEvaluator(this.config.evaluationModel, this.config.provider, this.config.threshold, this.config.promptGenerator);
+    }
+    /**
+     * The main entry point for performing an evaluation. It selects the evaluation
+     * strategy based on the configuration and executes it.
+     *
+     * @param options The original `TextGenerationOptions` from the user request.
+     * @param result The `GenerateResult` from the provider.
+     * @returns A promise that resolves to the `EvaluationResult`.
+     */
+    async evaluate(options, result, threshold, config) {
+        const evaluationStrategy = this.config.evaluationStrategy || "ragas";
+        const customEvaluator = this.config.customEvaluator;
+        switch (evaluationStrategy) {
+            case "ragas": {
+                const { evaluationResult, evalContext } = await this.evaluateWithRAGAS(options, result);
+                const evaluationData = mapToEvaluationData(evalContext, evaluationResult, threshold, config.offTopicThreshold, config.highSeverityThreshold);
+                return evaluationData;
+            }
+            case "custom": {
+                if (customEvaluator) {
+                    const { evaluationResult, evalContext } = await customEvaluator(options, result);
+                    return mapToEvaluationData(evalContext, evaluationResult, threshold, config.offTopicThreshold, config.highSeverityThreshold);
+                }
+                throw new Error("Custom evaluator function not provided in config.");
+            }
+            default:
+                throw new Error(`Unsupported evaluation strategy: ${evaluationStrategy} `);
+        }
+    }
+    /**
+     * Performs evaluation using the RAGAS-style model-based evaluator.
+     *
+     * @param options The original `TextGenerationOptions`.
+     * @param result The `GenerateResult` to be evaluated.
+     * @returns A promise that resolves to the `EvaluationResult`.
+     */
+    async evaluateWithRAGAS(options, result) {
+        const evalContext = this.contextBuilder.buildContext(options, result);
+        const evaluationResult = await this.ragasEvaluator.evaluate(evalContext);
+        return { evaluationResult, evalContext };
+    }
+}

package/dist/evaluation/prompts.d.ts

@@ -0,0 +1,22 @@
+import type { EnhancedEvaluationContext, GetPromptFunction } from "../types/evaluationTypes.js";
+/**
+ * A flexible class for building evaluation prompts. It allows for custom prompt
+ * generation logic to be injected while ensuring a consistent output format.
+ */
+export declare class PromptBuilder {
+    /**
+     * Builds the final evaluation prompt.
+     *
+     * @param context The rich context for the evaluation.
+     * @param getPrompt An optional function to generate the main body of the prompt.
+     *                  If not provided, a default prompt is used.
+     * @returns The complete prompt string to be sent to the judge LLM.
+     */
+    buildEvaluationPrompt(context: EnhancedEvaluationContext, getPrompt?: GetPromptFunction): string;
+    /**
+     * The default prompt generation logic.
+     * @param context The prepared context strings.
+     * @returns The default prompt body.
+     */
+    private getDefaultPrompt;
+}

package/dist/evaluation/prompts.js

@@ -0,0 +1,73 @@
+/**
+ * A flexible class for building evaluation prompts. It allows for custom prompt
+ * generation logic to be injected while ensuring a consistent output format.
+ */
+export class PromptBuilder {
+    /**
+     * Builds the final evaluation prompt.
+     *
+     * @param context The rich context for the evaluation.
+     * @param getPrompt An optional function to generate the main body of the prompt.
+     *                  If not provided, a default prompt is used.
+     * @returns The complete prompt string to be sent to the judge LLM.
+     */
+    buildEvaluationPrompt(context, getPrompt) {
+        const { userQuery, aiResponse, conversationHistory, toolExecutions, previousEvaluations, } = context;
+        const historyStr = conversationHistory
+            .map((turn) => `${turn.role}: ${turn.content}`)
+            .join("\n");
+        const toolsStr = toolExecutions.length > 0
+            ? `Tools were used: ${toolExecutions.map((t) => t.toolName).join(", ")}`
+            : "No tools were used.";
+        const retryStr = previousEvaluations && previousEvaluations.length > 0
+            ? `This is attempt #${context.attemptNumber}. Previous reasoning: ${previousEvaluations
+                .map((e) => e.reasoning)
+                .join("; ")} Previous suggested Improvements: ${previousEvaluations.map((e) => e.suggestedImprovements).join("; ")}`
+            : "This is the first attempt.";
+        const promptContext = {
+            userQuery,
+            history: historyStr,
+            tools: toolsStr,
+            retryInfo: retryStr,
+            aiResponse,
+        };
+        const mainPrompt = getPrompt
+            ? getPrompt(promptContext)
+            : this.getDefaultPrompt(promptContext);
+        return `
+      ${mainPrompt}
+
+      **Output Format (JSON):**
+      {
+        "relevanceScore": <1-10>,
+        "accuracyScore": <1-10>,
+        "completenessScore": <1-10>,
+        "finalScore": <1-10>,
+        "reasoning": "<Your constructive reasoning here>",
+        "suggestedImprovements": "<How the response can be improved>"
+      }
+    `;
+    }
+    /**
+     * The default prompt generation logic.
+     * @param context The prepared context strings.
+     * @returns The default prompt body.
+     */
+    getDefaultPrompt(context) {
+        return `
+      You are an expert AI quality evaluator. Your task is to evaluate the AI assistant's response based on the provided context.
+      Provide a score from 1 to 10 for each of the following criteria: Relevance, Accuracy, and Completeness.
+      Finally, provide an overall finalScore and constructive feedback for improvement.
+
+      **Evaluation Context:**
+      - User Query: ${context.userQuery}
+      - Conversation History:
+      ${context.history}
+      - Tools Executed: ${context.tools}
+      - Retry Information: ${context.retryInfo}
+
+      **AI Assistant's Response to Evaluate:**
+      ${context.aiResponse}
+    `;
+    }
+}

package/dist/evaluation/ragasEvaluator.d.ts

@@ -0,0 +1,28 @@
+import type { EnhancedEvaluationContext, EvaluationResult, GetPromptFunction } from "../types/evaluationTypes.js";
+/**
+ * Implements a RAGAS-style evaluator that uses a "judge" LLM to score the
+ * quality of an AI response based on rich, contextual information.
+ */
+export declare class RAGASEvaluator {
+    private evaluationModel;
+    private providerName;
+    private threshold;
+    private promptBuilder;
+    private promptGenerator?;
+    constructor(evaluationModel?: string, providerName?: string, threshold?: number, promptGenerator?: GetPromptFunction);
+    /**
+     * Evaluates an AI-generated response using a model-based approach.
+     *
+     * @param context The rich, contextual information for the evaluation.
+     * @returns A promise that resolves to a detailed `EvaluationResult`.
+     */
+    evaluate(context: EnhancedEvaluationContext): Promise<EvaluationResult>;
+    /**
+     * Parses the raw JSON string from the judge LLM into a structured `EvaluationResult` object.
+     * It includes error handling to gracefully manage malformed JSON.
+     *
+     * @param rawResponse The raw string response from the evaluation model.
+     * @returns A structured object containing the evaluation scores and feedback.
+     */
+    private parseEvaluationResponse;
+}

package/dist/evaluation/ragasEvaluator.js

@@ -0,0 +1,90 @@
+import { AIProviderFactory } from "../core/factory.js";
+import { PromptBuilder } from "./prompts.js";
+import { logger } from "../utils/logger.js";
+/**
+ * Implements a RAGAS-style evaluator that uses a "judge" LLM to score the
+ * quality of an AI response based on rich, contextual information.
+ */
+export class RAGASEvaluator {
+    evaluationModel;
+    providerName;
+    threshold;
+    promptBuilder;
+    promptGenerator;
+    constructor(evaluationModel, providerName, threshold, promptGenerator) {
+        this.evaluationModel =
+            evaluationModel ||
+                process.env.NEUROLINK_RAGAS_EVALUATION_MODEL ||
+                "gemini-1.5-flash";
+        this.providerName =
+            providerName ||
+                process.env.NEUROLINK_RAGAS_EVALUATION_PROVIDER ||
+                "vertex";
+        this.threshold =
+            threshold || Number(process.env.NEUROLINK_EVALUATION_THRESHOLD) || 7;
+        this.promptBuilder = new PromptBuilder();
+        this.promptGenerator = promptGenerator;
+    }
+    /**
+     * Evaluates an AI-generated response using a model-based approach.
+     *
+     * @param context The rich, contextual information for the evaluation.
+     * @returns A promise that resolves to a detailed `EvaluationResult`.
+     */
+    async evaluate(context) {
+        const startTime = Date.now();
+        const prompt = this.promptBuilder.buildEvaluationPrompt(context, this.promptGenerator);
+        const provider = await AIProviderFactory.createProvider(this.providerName, this.evaluationModel);
+        const result = await provider.generate({
+            input: { text: prompt },
+        });
+        if (!result) {
+            throw new Error("Evaluation generation failed to return a result.");
+        }
+        const rawEvaluationResponse = result.content;
+        const parsedResult = this.parseEvaluationResponse(rawEvaluationResponse);
+        const evaluationTime = Date.now() - startTime;
+        const finalResult = {
+            ...parsedResult,
+            isPassing: parsedResult.finalScore >= this.threshold, // This will be recalculated, but is needed for the type
+            evaluationModel: this.evaluationModel,
+            evaluationTime,
+            attemptNumber: context.attemptNumber,
+            rawEvaluationResponse,
+        };
+        return finalResult;
+    }
+    /**
+     * Parses the raw JSON string from the judge LLM into a structured `EvaluationResult` object.
+     * It includes error handling to gracefully manage malformed JSON.
+     *
+     * @param rawResponse The raw string response from the evaluation model.
+     * @returns A structured object containing the evaluation scores and feedback.
+     */
+    parseEvaluationResponse(rawResponse) {
+        try {
+            const cleanedResponse = rawResponse.replace(/```json\n|```/g, "").trim();
+            const parsed = JSON.parse(cleanedResponse);
+            logger.debug("Parsed evaluation response for RAGAS Evaluator:", parsed);
+            return {
+                relevanceScore: Number(parsed.relevanceScore) || 0,
+                accuracyScore: Number(parsed.accuracyScore) || 0,
+                completenessScore: Number(parsed.completenessScore) || 0,
+                finalScore: Number(parsed.finalScore) || 0,
+                suggestedImprovements: parsed.suggestedImprovements || "No suggestions provided.",
+                reasoning: parsed.reasoning || "No reasoning provided.",
+            };
+        }
+        catch (error) {
+            logger.error("Failed to parse evaluation response:", error);
+            return {
+                relevanceScore: 0,
+                accuracyScore: 0,
+                completenessScore: 0,
+                finalScore: 0,
+                reasoning: "Error parsing evaluation response.",
+                suggestedImprovements: "Error parsing evaluation response.",
+            };
+        }
+    }
+}

package/dist/evaluation/retryManager.d.ts

@@ -0,0 +1,40 @@
+/**
+ * @file Implements the RetryManager class for handling evaluation retries.
+ */
+import type { EvaluationResult } from "../types/evaluationTypes.js";
+import type { TextGenerationOptions } from "../types/generateTypes.js";
+/**
+ * Manages the retry logic for the auto-evaluation middleware. It decides if a
+ * retry is warranted based on the evaluation score and prepares the options
+ * for the next generation attempt by incorporating feedback into the prompt.
+ */
+export declare class RetryManager {
+    private maxRetries;
+    constructor(maxRetries?: number);
+    /**
+     * Determines if a retry should be attempted based on the evaluation result.
+     *
+     * @param evaluation The `EvaluationResult` of the last attempt.
+     * @returns `true` if the response did not pass and the maximum number of retries has not been reached.
+     */
+    shouldRetry(evaluation: EvaluationResult): boolean;
+    /**
+     * Prepares the options for the next generation attempt by creating a new,
+     * improved prompt that includes feedback from the failed evaluation.
+     *
+     * @param originalOptions The original `TextGenerationOptions` from the user request.
+     * @param evaluation The `EvaluationResult` of the failed attempt.
+     * @returns A new `TextGenerationOptions` object with an improved prompt.
+     */
+    prepareRetryOptions(originalOptions: TextGenerationOptions, evaluation: EvaluationResult): TextGenerationOptions;
+    /**
+     * Builds a new prompt for a retry attempt by incorporating feedback from the
+     * evaluation. The instructions become progressively more direct with each attempt.
+     *
+     * @param originalPrompt The user's original prompt.
+     * @param feedback The constructive feedback from the evaluation.
+     * @param attemptNumber The upcoming attempt number (e.g., 2 for the first retry).
+     * @returns A new, enhanced prompt string.
+     */
+    private buildRetryPrompt;
+}