@juspay/neurolink 7.45.0 → 7.47.0

package/dist/lib/evaluation/retryManager.d.ts

@@ -0,0 +1,40 @@
+/**
+ * @file Implements the RetryManager class for handling evaluation retries.
+ */
+import type { EvaluationResult } from "../types/evaluationTypes.js";
+import type { TextGenerationOptions } from "../types/generateTypes.js";
+/**
+ * Manages the retry logic for the auto-evaluation middleware. It decides if a
+ * retry is warranted based on the evaluation score and prepares the options
+ * for the next generation attempt by incorporating feedback into the prompt.
+ */
+export declare class RetryManager {
+    private maxRetries;
+    constructor(maxRetries?: number);
+    /**
+     * Determines if a retry should be attempted based on the evaluation result.
+     *
+     * @param evaluation The `EvaluationResult` of the last attempt.
+     * @returns `true` if the response did not pass and the maximum number of retries has not been reached.
+     */
+    shouldRetry(evaluation: EvaluationResult): boolean;
+    /**
+     * Prepares the options for the next generation attempt by creating a new,
+     * improved prompt that includes feedback from the failed evaluation.
+     *
+     * @param originalOptions The original `TextGenerationOptions` from the user request.
+     * @param evaluation The `EvaluationResult` of the failed attempt.
+     * @returns A new `TextGenerationOptions` object with an improved prompt.
+     */
+    prepareRetryOptions(originalOptions: TextGenerationOptions, evaluation: EvaluationResult): TextGenerationOptions;
+    /**
+     * Builds a new prompt for a retry attempt by incorporating feedback from the
+     * evaluation. The instructions become progressively more direct with each attempt.
+     *
+     * @param originalPrompt The user's original prompt.
+     * @param feedback The constructive feedback from the evaluation.
+     * @param attemptNumber The upcoming attempt number (e.g., 2 for the first retry).
+     * @returns A new, enhanced prompt string.
+     */
+    private buildRetryPrompt;
+}

package/dist/lib/evaluation/retryManager.js

@@ -0,0 +1,78 @@
+/**
+ * @file Implements the RetryManager class for handling evaluation retries.
+ */
+/**
+ * Manages the retry logic for the auto-evaluation middleware. It decides if a
+ * retry is warranted based on the evaluation score and prepares the options
+ * for the next generation attempt by incorporating feedback into the prompt.
+ */
+export class RetryManager {
+    maxRetries;
+    constructor(maxRetries = 2) {
+        // Total 3 attempts: 1 initial + 2 retries
+        this.maxRetries = maxRetries;
+    }
+    /**
+     * Determines if a retry should be attempted based on the evaluation result.
+     *
+     * @param evaluation The `EvaluationResult` of the last attempt.
+     * @returns `true` if the response did not pass and the maximum number of retries has not been reached.
+     */
+    shouldRetry(evaluation) {
+        // Attempt number is 1-based. If attempt 1 fails, we can retry.
+        // If attempt 3 (maxRetries + 1) fails, we stop.
+        return !evaluation.isPassing && evaluation.attemptNumber <= this.maxRetries;
+    }
+    /**
+     * Prepares the options for the next generation attempt by creating a new,
+     * improved prompt that includes feedback from the failed evaluation.
+     *
+     * @param originalOptions The original `TextGenerationOptions` from the user request.
+     * @param evaluation The `EvaluationResult` of the failed attempt.
+     * @returns A new `TextGenerationOptions` object with an improved prompt.
+     */
+    prepareRetryOptions(originalOptions, evaluation) {
+        const originalPrompt = originalOptions.prompt || originalOptions.input?.text || "";
+        const newPrompt = this.buildRetryPrompt(originalPrompt, evaluation.suggestedImprovements, evaluation.attemptNumber + 1);
+        // Return a new options object with the updated prompt
+        return {
+            ...originalOptions,
+            prompt: newPrompt,
+            // Ensure input is not carried over if prompt is now the source of truth
+            input: undefined,
+            // Carry over the original prompt for context in subsequent retries if needed
+            originalPrompt: originalOptions.originalPrompt || originalPrompt,
+        };
+    }
+    /**
+     * Builds a new prompt for a retry attempt by incorporating feedback from the
+     * evaluation. The instructions become progressively more direct with each attempt.
+     *
+     * @param originalPrompt The user's original prompt.
+     * @param feedback The constructive feedback from the evaluation.
+     * @param attemptNumber The upcoming attempt number (e.g., 2 for the first retry).
+     * @returns A new, enhanced prompt string.
+     */
+    buildRetryPrompt(originalPrompt, feedback, attemptNumber) {
+        let instruction = "";
+        switch (attemptNumber) {
+            case 2: // First retry
+                instruction = `The previous response was not satisfactory. Please improve it based on the following feedback: "${feedback}".`;
+                break;
+            case 3: // Second retry
+                instruction = `The last response still requires improvement. Pay close attention to this feedback: "${feedback}". You MUST address these points.`;
+                break;
+            default: // Final retry or unexpected attempt number
+                instruction = `This is the final attempt. You MUST address the following feedback to generate a satisfactory response: "${feedback}".`;
+                break;
+        }
+        return `
+    Original Request: ${originalPrompt}
+
+    **Correction Instructions:**
+    ${instruction}
+
+    Generate a new, complete response that incorporates this feedback.
+  `;
+    }
+}

package/dist/lib/evaluation/scoring.d.ts

@@ -0,0 +1,16 @@
+/**
+ * @file Contains the logic for mapping raw evaluation results to the structured EvaluationData type.
+ */
+import type { EnhancedEvaluationContext, EvaluationResult } from "../types/evaluationTypes.js";
+import type { EvaluationData } from "../types/evaluation.js";
+/**
+ * Maps a raw `EvaluationResult` to the structured `EvaluationData` format.
+ * This includes calculating derived fields like `isOffTopic` and `alertSeverity`.
+ *
+ * @param result The raw `EvaluationResult` from the evaluator.
+ * @param threshold The score threshold to determine if the evaluation is passing.
+ * @param offTopicThreshold The score below which a response is considered off-topic.
+ * @param highSeverityThreshold The score below which a failing response is high severity.
+ * @returns A structured `EvaluationData` object.
+ */
+export declare function mapToEvaluationData(evalContext: EnhancedEvaluationContext, result: EvaluationResult, threshold: number, offTopicThreshold?: number, highSeverityThreshold?: number): EvaluationData;

package/dist/lib/evaluation/scoring.js

@@ -0,0 +1,35 @@
+/**
+ * @file Contains the logic for mapping raw evaluation results to the structured EvaluationData type.
+ */
+/**
+ * Maps a raw `EvaluationResult` to the structured `EvaluationData` format.
+ * This includes calculating derived fields like `isOffTopic` and `alertSeverity`.
+ *
+ * @param result The raw `EvaluationResult` from the evaluator.
+ * @param threshold The score threshold to determine if the evaluation is passing.
+ * @param offTopicThreshold The score below which a response is considered off-topic.
+ * @param highSeverityThreshold The score below which a failing response is high severity.
+ * @returns A structured `EvaluationData` object.
+ */
+export function mapToEvaluationData(evalContext, result, threshold, offTopicThreshold = 5, highSeverityThreshold = 4) {
+    const isPassing = result.finalScore >= threshold;
+    return {
+        relevance: result.relevanceScore,
+        accuracy: result.accuracyScore,
+        completeness: result.completenessScore,
+        overall: result.finalScore,
+        isOffTopic: result.finalScore < offTopicThreshold,
+        alertSeverity: isPassing
+            ? "none"
+            : result.finalScore < highSeverityThreshold
+                ? "high"
+                : "medium",
+        reasoning: result.reasoning,
+        suggestedImprovements: result.suggestedImprovements,
+        evaluationModel: result.evaluationModel,
+        evaluationTime: result.evaluationTime,
+        evaluationAttempt: result.attemptNumber,
+        responseContent: evalContext.aiResponse,
+        queryContent: evalContext.userQuery,
+    };
+}

package/dist/lib/middleware/builtin/autoEvaluation.d.ts

@@ -0,0 +1,14 @@
+/**
+ * @file Implements the Auto-Evaluation Middleware for ensuring response quality.
+ */
+import type { NeuroLinkMiddleware, AutoEvaluationConfig } from "../../types/middlewareTypes.js";
+/**
+ * Creates the Auto-Evaluation middleware, which intercepts generation requests
+ * to evaluate the quality of the response. If the response quality is below a
+ * configured threshold, it can trigger retries with feedback.
+ *
+ * @param config - Configuration for the auto-evaluation middleware.
+ * @returns A `NeuroLinkMiddleware` object.
+ */
+export declare function createAutoEvaluationMiddleware(config?: AutoEvaluationConfig): NeuroLinkMiddleware;
+export default createAutoEvaluationMiddleware;

package/dist/lib/middleware/builtin/autoEvaluation.js

@@ -0,0 +1,181 @@
+/**
+ * @file Implements the Auto-Evaluation Middleware for ensuring response quality.
+ */
+import { Evaluator } from "../../evaluation/index.js";
+import { logger } from "../../utils/logger.js";
+/**
+ * Creates the Auto-Evaluation middleware, which intercepts generation requests
+ * to evaluate the quality of the response. If the response quality is below a
+ * configured threshold, it can trigger retries with feedback.
+ *
+ * @param config - Configuration for the auto-evaluation middleware.
+ * @returns A `NeuroLinkMiddleware` object.
+ */
+export function createAutoEvaluationMiddleware(config = {}) {
+    const metadata = {
+        id: "autoEvaluation",
+        name: "Auto Evaluation",
+        description: "Automatically evaluates response quality and retries if needed.",
+        priority: 90,
+        defaultEnabled: false, // Should be explicitly enabled
+    };
+    logger.debug("Auto-Evaluation Middleware Config:", config);
+    const middleware = {
+        wrapGenerate: async ({ doGenerate, params }) => {
+            const options = params;
+            const rawResult = await doGenerate();
+            const result = {
+                ...rawResult,
+                content: rawResult.text ?? "",
+                usage: {
+                    input: rawResult.usage.promptTokens,
+                    output: rawResult.usage.completionTokens,
+                    total: rawResult.usage.promptTokens + rawResult.usage.completionTokens,
+                },
+                toolCalls: rawResult.toolCalls?.map((tc) => {
+                    let parsedArgs = tc.args;
+                    if (typeof tc.args === "string") {
+                        try {
+                            parsedArgs = JSON.parse(tc.args);
+                        }
+                        catch (e) {
+                            logger.warn(`Failed to parse tool call args for tool ${tc.toolName}:`, e);
+                            parsedArgs = tc.args; // Fallback to original string if parsing fails
+                        }
+                    }
+                    return {
+                        ...tc,
+                        args: parsedArgs,
+                    };
+                }),
+            };
+            const isBlocking = config.blocking !== false;
+            if (isBlocking) {
+                const evaluationResult = await performEvaluation(config, options, result);
+                return {
+                    ...rawResult,
+                    evaluationResult,
+                };
+            }
+            else {
+                performEvaluation(config, options, result).catch((err) => {
+                    logger.error("Non-blocking auto-evaluation error:", err);
+                });
+                return rawResult;
+            }
+        },
+        wrapStream: async ({ doStream, params }) => {
+            const options = params;
+            const rawResult = await doStream();
+            const [streamForUser, streamForEvaluation] = rawResult.stream.tee();
+            // Non-blocking evaluation for streams
+            consumeAndEvaluateStream(config, options, streamForEvaluation).catch((err) => {
+                logger.error("Non-blocking stream auto-evaluation error:", err);
+            });
+            return {
+                ...rawResult,
+                stream: streamForUser,
+            };
+        },
+    };
+    return {
+        ...middleware,
+        metadata,
+    };
+}
+/**
+ * A common function to perform the evaluation logic.
+ * @param config The middleware configuration.
+ * @param options The text generation options.
+ * @param result The generation result.
+ */
+async function performEvaluation(config, options, result) {
+    const isBlocking = config.blocking !== false;
+    const threshold = config.threshold ??
+        (Number(process.env.NEUROLINK_EVALUATION_THRESHOLD) || 7);
+    try {
+        const evaluator = new Evaluator({
+            threshold,
+            provider: config.provider,
+            promptGenerator: config.promptGenerator,
+            evaluationModel: config.evaluationModel,
+        });
+        const evaluationResult = await evaluator.evaluate(options, result, threshold, config);
+        if (config.onEvaluationComplete) {
+            await config.onEvaluationComplete(evaluationResult);
+        }
+    }
+    catch (error) {
+        logger.error("Error during auto-evaluation:", error);
+        if (isBlocking) {
+            throw error;
+        }
+    }
+}
+/**
+ * Consumes a stream to build the full response and then evaluates it.
+ * @param config The middleware configuration.
+ * @param options The generation options.
+ * @param stream The stream to consume.
+ */
+async function consumeAndEvaluateStream(config, options, stream) {
+    let fullText = "";
+    let usage;
+    const toolCalls = [];
+    const reader = stream.getReader();
+    try {
+        while (true) {
+            const { done, value } = await reader.read();
+            if (done) {
+                break;
+            }
+            switch (value.type) {
+                case "text-delta":
+                    fullText += value.textDelta;
+                    break;
+                case "tool-call":
+                    {
+                        let parsedArgs;
+                        try {
+                            parsedArgs = JSON.parse(value.args);
+                        }
+                        catch (e) {
+                            logger.warn(`Failed to parse tool call args for tool ${value.toolName}:`, e);
+                            // In case of parsing failure, we can't assign a string.
+                            // Let's use an object with the raw string to maintain type safety.
+                            parsedArgs = { raw: value.args };
+                        }
+                        toolCalls.push({
+                            toolCallId: value.toolCallId,
+                            toolName: value.toolName,
+                            args: parsedArgs,
+                        });
+                    }
+                    break;
+                case "finish":
+                    usage = {
+                        input: value.usage.promptTokens,
+                        output: value.usage.completionTokens,
+                        total: value.usage.promptTokens + value.usage.completionTokens,
+                    };
+                    break;
+            }
+        }
+    }
+    finally {
+        reader.releaseLock();
+    }
+    const result = {
+        content: fullText,
+        usage,
+        toolCalls: toolCalls.length > 0 ? toolCalls : undefined,
+    };
+    // For streams, evaluation is always non-blocking from the user's perspective.
+    if (config.blocking) {
+        logger.warn("Auto-evaluation 'blocking' mode is not supported for streaming responses. Evaluation will proceed non-blockingly.");
+    }
+    // Create a new config object to force non-blocking behavior for the evaluation function
+    const nonBlockingConfig = { ...config, blocking: false };
+    await performEvaluation(nonBlockingConfig, options, result);
+}
+export default createAutoEvaluationMiddleware;

package/dist/lib/middleware/factory.js

@@ -2,6 +2,7 @@ import { wrapLanguageModel } from "ai";
 import { MiddlewareRegistry } from "./registry.js";
 import { createAnalyticsMiddleware } from "./builtin/analytics.js";
 import { createGuardrailsMiddleware } from "./builtin/guardrails.js";
+import { createAutoEvaluationMiddleware } from "./builtin/autoEvaluation.js";
 import { logger } from "../utils/logger.js";
 /**
  * Middleware factory for creating and applying middleware chains.
@@ -24,6 +25,7 @@ export class MiddlewareFactory {
         const builtInMiddlewareCreators = {
             analytics: createAnalyticsMiddleware,
             guardrails: createGuardrailsMiddleware,
+            autoEvaluation: createAutoEvaluationMiddleware,
         };
         // Register built-in presets
         this.registerPreset({
@@ -54,6 +56,7 @@ export class MiddlewareFactory {
             if (!this.registry.has(middlewareId)) {
                 const creator = builtInMiddlewareCreators[middlewareId];
                 const config = options.middlewareConfig?.[middlewareId]?.config;
+                logger.debug(`Registering built-in middleware '${middlewareId}'`, config);
                 this.registry.register(creator(config));
             }
         }
@@ -92,6 +95,7 @@ export class MiddlewareFactory {
             const middlewareConfig = this.buildMiddlewareConfig(mergedOptions);
             // Re-register middleware with the correct configuration for this call
             for (const [id, config] of Object.entries(middlewareConfig)) {
+                logger.debug(`Configuring middleware '${id}'`, { config });
                 if (config.enabled && this.registry.has(id)) {
                     const creator = this.getCreator(id);
                     if (creator) {
@@ -137,7 +141,9 @@ export class MiddlewareFactory {
         const builtInMiddlewareCreators = {
             analytics: createAnalyticsMiddleware,
             guardrails: createGuardrailsMiddleware,
+            autoEvaluation: createAutoEvaluationMiddleware,
         };
+        logger.debug("Getting creator for middleware ID:", id);
         return builtInMiddlewareCreators[id];
     }
     /**

package/dist/lib/providers/azureOpenai.js

@@ -4,7 +4,7 @@ import { BaseProvider } from "../core/baseProvider.js";
 import { APIVersions } from "../types/providers.js";
 import { validateApiKey, createAzureAPIKeyConfig, createAzureEndpointConfig, } from "../utils/providerConfig.js";
 import { logger } from "../utils/logger.js";
-import { buildMessagesArray } from "../utils/messageBuilder.js";
+import { buildMessagesArray, buildMultimodalMessagesArray, convertToCoreMessages, } from "../utils/messageBuilder.js";
 import { createProxyFetch } from "../proxy/proxyFetch.js";
 import { DEFAULT_MAX_STEPS } from "../core/constants.js";
 export class AzureOpenAIProvider extends BaseProvider {
@@ -109,8 +109,41 @@ export class AzureOpenAIProvider extends BaseProvider {
                     })),
                 });
             }
-            // Build message array from options
-            const messages = buildMessagesArray(options);
+            // Build message array from options with multimodal support
+            const hasMultimodalInput = !!(options.input?.images?.length || options.input?.content?.length);
+            let messages;
+            if (hasMultimodalInput) {
+                logger.debug(`Azure OpenAI: Detected multimodal input, using multimodal message builder`, {
+                    hasImages: !!options.input?.images?.length,
+                    imageCount: options.input?.images?.length || 0,
+                    hasContent: !!options.input?.content?.length,
+                    contentCount: options.input?.content?.length || 0,
+                });
+                // Create multimodal options for buildMultimodalMessagesArray
+                const multimodalOptions = {
+                    input: {
+                        text: options.input?.text || "",
+                        images: options.input?.images,
+                        content: options.input?.content,
+                    },
+                    systemPrompt: options.systemPrompt,
+                    conversationHistory: options.conversationMessages,
+                    provider: this.providerName,
+                    model: this.modelName,
+                    temperature: options.temperature,
+                    maxTokens: options.maxTokens,
+                    enableAnalytics: options.enableAnalytics,
+                    enableEvaluation: options.enableEvaluation,
+                    context: options.context,
+                };
+                const mm = await buildMultimodalMessagesArray(multimodalOptions, this.providerName, this.modelName);
+                // Convert multimodal messages to Vercel AI SDK format (CoreMessage[])
+                messages = convertToCoreMessages(mm);
+            }
+            else {
+                logger.debug(`Azure OpenAI: Text-only input, using standard message builder`);
+                messages = buildMessagesArray(options);
+            }
             const model = await this.getAISDKModelWithMiddleware(options);
             const stream = await streamText({
                 model,

package/dist/lib/providers/googleAiStudio.js

@@ -7,8 +7,9 @@ import { createTimeoutController, TimeoutError } from "../utils/timeout.js";
 import { AuthenticationError, NetworkError, ProviderError, RateLimitError, } from "../types/errors.js";
 import { DEFAULT_MAX_STEPS } from "../core/constants.js";
 import { streamAnalyticsCollector } from "../core/streamAnalytics.js";
-import { buildMessagesArray } from "../utils/messageBuilder.js";
+import { buildMessagesArray, buildMultimodalMessagesArray, convertToCoreMessages, } from "../utils/messageBuilder.js";
 // Google AI Live API types now imported from ../types/providerSpecific.js
+// Import proper types for multimodal message handling
 // Create Google GenAI client
 async function createGoogleGenAIClient(apiKey) {
     const mod = await import("@google/genai");
@@ -90,8 +91,41 @@ export class GoogleAIStudioProvider extends BaseProvider {
             // Get tools consistently with generate method
             const shouldUseTools = !options.disableTools && this.supportsTools();
             const tools = shouldUseTools ? await this.getAllTools() : {};
-            // Build message array from options
-            const messages = buildMessagesArray(options);
+            // Build message array from options with multimodal support
+            const hasMultimodalInput = !!(options.input?.images?.length || options.input?.content?.length);
+            let messages;
+            if (hasMultimodalInput) {
+                logger.debug(`Google AI Studio: Detected multimodal input, using multimodal message builder`, {
+                    hasImages: !!options.input?.images?.length,
+                    imageCount: options.input?.images?.length || 0,
+                    hasContent: !!options.input?.content?.length,
+                    contentCount: options.input?.content?.length || 0,
+                });
+                // Create multimodal options for buildMultimodalMessagesArray
+                const multimodalOptions = {
+                    input: {
+                        text: options.input?.text || "",
+                        images: options.input?.images,
+                        content: options.input?.content,
+                    },
+                    systemPrompt: options.systemPrompt,
+                    conversationHistory: options.conversationMessages,
+                    provider: this.providerName,
+                    model: this.modelName,
+                    temperature: options.temperature,
+                    maxTokens: options.maxTokens,
+                    enableAnalytics: options.enableAnalytics,
+                    enableEvaluation: options.enableEvaluation,
+                    context: options.context,
+                };
+                const mm = await buildMultimodalMessagesArray(multimodalOptions, this.providerName, this.modelName);
+                // Convert multimodal messages to Vercel AI SDK format (CoreMessage[])
+                messages = convertToCoreMessages(mm);
+            }
+            else {
+                logger.debug(`Google AI Studio: Text-only input, using standard message builder`);
+                messages = buildMessagesArray(options);
+            }
             const result = await streamText({
                 model,
                 messages: messages,

package/dist/lib/providers/googleVertex.js

@@ -11,8 +11,9 @@ import fs from "fs";
 import path from "path";
 import os from "os";
 import dns from "dns";
-import { buildMessagesArray } from "../utils/messageBuilder.js";
+import { buildMessagesArray, buildMultimodalMessagesArray, convertToCoreMessages, } from "../utils/messageBuilder.js";
 import { createProxyFetch } from "../proxy/proxyFetch.js";
+// Import proper types for multimodal message handling
 // Enhanced Anthropic support with direct imports
 // Using the dual provider architecture from Vercel AI SDK
 const hasAnthropicSupport = () => {
@@ -594,8 +595,41 @@ export class GoogleVertexProvider extends BaseProvider {
         try {
             // Validate stream options
             this.validateStreamOptionsOnly(options);
-            // Build message array from options
-            const messages = buildMessagesArray(options);
+            // Build message array from options with multimodal support
+            const hasMultimodalInput = !!(options.input?.images?.length || options.input?.content?.length);
+            let messages;
+            if (hasMultimodalInput) {
+                logger.debug(`${functionTag}: Detected multimodal input, using multimodal message builder`, {
+                    hasImages: !!options.input?.images?.length,
+                    imageCount: options.input?.images?.length || 0,
+                    hasContent: !!options.input?.content?.length,
+                    contentCount: options.input?.content?.length || 0,
+                });
+                // Create multimodal options for buildMultimodalMessagesArray
+                const multimodalOptions = {
+                    input: {
+                        text: options.input?.text || "",
+                        images: options.input?.images,
+                        content: options.input?.content,
+                    },
+                    systemPrompt: options.systemPrompt,
+                    conversationHistory: options.conversationMessages,
+                    provider: this.providerName,
+                    model: this.modelName,
+                    temperature: options.temperature,
+                    maxTokens: options.maxTokens,
+                    enableAnalytics: options.enableAnalytics,
+                    enableEvaluation: options.enableEvaluation,
+                    context: options.context,
+                };
+                const mm = await buildMultimodalMessagesArray(multimodalOptions, this.providerName, this.modelName);
+                // Convert multimodal messages to Vercel AI SDK format (CoreMessage[])
+                messages = convertToCoreMessages(mm);
+            }
+            else {
+                logger.debug(`${functionTag}: Text-only input, using standard message builder`);
+                messages = buildMessagesArray(options);
+            }
             const model = await this.getAISDKModelWithMiddleware(options); // This is where network connection happens!
             // Get all available tools (direct + MCP + external) for streaming
             const shouldUseTools = !options.disableTools && this.supportsTools();

package/dist/lib/types/evaluation.d.ts

@@ -25,6 +25,8 @@ export type EvaluationData = {
     domainAlignment?: number;
     terminologyAccuracy?: number;
     toolEffectiveness?: number;
+    responseContent?: string;
+    queryContent?: string;
     isOffTopic: boolean;
     alertSeverity: AlertSeverity;
     reasoning: string;