@juspay/neurolink 7.45.0 → 7.46.0

package/dist/middleware/factory.js

@@ -2,6 +2,7 @@ import { wrapLanguageModel } from "ai";
 import { MiddlewareRegistry } from "./registry.js";
 import { createAnalyticsMiddleware } from "./builtin/analytics.js";
 import { createGuardrailsMiddleware } from "./builtin/guardrails.js";
+import { createAutoEvaluationMiddleware } from "./builtin/autoEvaluation.js";
 import { logger } from "../utils/logger.js";
 /**
  * Middleware factory for creating and applying middleware chains.
@@ -24,6 +25,7 @@ export class MiddlewareFactory {
         const builtInMiddlewareCreators = {
             analytics: createAnalyticsMiddleware,
             guardrails: createGuardrailsMiddleware,
+            autoEvaluation: createAutoEvaluationMiddleware,
         };
         // Register built-in presets
         this.registerPreset({
@@ -54,6 +56,7 @@ export class MiddlewareFactory {
             if (!this.registry.has(middlewareId)) {
                 const creator = builtInMiddlewareCreators[middlewareId];
                 const config = options.middlewareConfig?.[middlewareId]?.config;
+                logger.debug(`Registering built-in middleware '${middlewareId}'`, config);
                 this.registry.register(creator(config));
             }
         }
@@ -92,6 +95,7 @@ export class MiddlewareFactory {
             const middlewareConfig = this.buildMiddlewareConfig(mergedOptions);
             // Re-register middleware with the correct configuration for this call
             for (const [id, config] of Object.entries(middlewareConfig)) {
+                logger.debug(`Configuring middleware '${id}'`, { config });
                 if (config.enabled && this.registry.has(id)) {
                     const creator = this.getCreator(id);
                     if (creator) {
@@ -137,7 +141,9 @@ export class MiddlewareFactory {
         const builtInMiddlewareCreators = {
             analytics: createAnalyticsMiddleware,
             guardrails: createGuardrailsMiddleware,
+            autoEvaluation: createAutoEvaluationMiddleware,
         };
+        logger.debug("Getting creator for middleware ID:", id);
         return builtInMiddlewareCreators[id];
     }
     /**

package/dist/types/evaluation.d.ts

@@ -25,6 +25,8 @@ export type EvaluationData = {
     domainAlignment?: number;
     terminologyAccuracy?: number;
     toolEffectiveness?: number;
+    responseContent?: string;
+    queryContent?: string;
     isOffTopic: boolean;
     alertSeverity: AlertSeverity;
     reasoning: string;

package/dist/types/evaluationTypes.d.ts

@@ -0,0 +1,142 @@
+import type { LanguageModelV1CallOptions } from "ai";
+import type { TokenUsage } from "./analytics.js";
+import type { GenerateResult } from "./generateTypes.js";
+import type { ToolExecution } from "./tools.js";
+/**
+ * Represents the analysis of the user's query intent.
+ * This provides a basic understanding of what the user is trying to achieve.
+ */
+export interface QueryIntentAnalysis {
+    /** The type of query, e.g., asking a question or giving a command. */
+    type: "question" | "command" | "greeting" | "unknown";
+    /** The estimated complexity of the query. */
+    complexity: "low" | "medium" | "high";
+    /** Whether the query likely required the use of tools to be answered correctly. */
+    shouldHaveUsedTools: boolean;
+}
+/**
+ * Represents a single turn in an enhanced conversation history,
+ * including tool executions and evaluations for richer context.
+ */
+export interface EnhancedConversationTurn {
+    /** The role of the speaker, either 'user' or 'assistant'. */
+    role: "user" | "assistant";
+    /** The content of the message. */
+    content: string;
+    /** The timestamp of the message. */
+    timestamp: string;
+    /** Any tools that were executed as part of this turn. */
+    toolExecutions?: ToolExecution[];
+    /** The evaluation result for this turn, if applicable. */
+    evaluation?: EvaluationResult;
+}
+/**
+ * Contains all the rich context needed for a thorough, RAGAS-style evaluation.
+ * This object is constructed by the `ContextBuilder` and used by the `RAGASEvaluator`.
+ */
+export interface EnhancedEvaluationContext {
+    /** The original user query. */
+    userQuery: string;
+    /** An analysis of the user's query intent. */
+    queryAnalysis: QueryIntentAnalysis;
+    /** The AI's response that is being evaluated. */
+    aiResponse: string;
+    /** The AI provider that generated the response. */
+    provider: string;
+    /** The specific model that generated the response. */
+    model: string;
+    /** The parameters used for the generation call. */
+    generationParams: {
+        temperature?: number;
+        maxTokens?: number;
+        systemPrompt?: string;
+    };
+    /** A list of tools that were executed. */
+    toolExecutions: ToolExecution[];
+    /** The history of the conversation leading up to this turn. */
+    conversationHistory: EnhancedConversationTurn[];
+    /** The response time of the AI in milliseconds. */
+    responseTime: number;
+    /** The token usage for the generation. */
+    tokenUsage: TokenUsage;
+    /** The results of any previous evaluation attempts for this response. */
+    previousEvaluations?: EvaluationResult[];
+    /** The current attempt number for this evaluation (1-based). */
+    attemptNumber: number;
+}
+/**
+ * Represents the result of a single evaluation attempt, based on RAGAS principles.
+ */
+export interface EvaluationResult {
+    /** The final, overall score for the response, typically from 1 to 10. */
+    finalScore: number;
+    /** How well the response addresses the user's query. */
+    relevanceScore: number;
+    /** The factual accuracy of the information in the response. */
+    accuracyScore: number;
+    /** How completely the response answers the user's query. */
+    completenessScore: number;
+    /** Whether the final score meets the passing threshold. */
+    isPassing: boolean;
+    /** Constructive response from the judge LLM on how to improve the response. */
+    reasoning: string;
+    /** Specific suggestions for improving the response. */
+    suggestedImprovements: string;
+    /** The raw, unparsed response from the judge LLM. */
+    rawEvaluationResponse: string;
+    /** The model used to perform the evaluation. */
+    evaluationModel: string;
+    /** The time taken for the evaluation in milliseconds. */
+    evaluationTime: number;
+    /** The attempt number for this evaluation. */
+    attemptNumber: number;
+}
+/**
+ * Provides detailed information when a response fails quality assurance checks.
+ */
+export interface QualityErrorDetails {
+    /** The history of all evaluation attempts for this response. */
+    evaluationHistory: EvaluationResult[];
+    /** The final score of the last attempt. */
+    finalScore: number;
+    /** The total number of evaluation attempts made. */
+    attempts: number;
+    /** A summary message of the failure. */
+    message: string;
+}
+/**
+ * Configuration for the main `Evaluator` class.
+ */
+export interface EvaluationConfig {
+    /** The minimum score (1-10) for a response to be considered passing. */
+    threshold?: number;
+    /** The evaluation strategy to use. Currently only 'ragas' is supported. */
+    evaluationStrategy?: "ragas" | "custom";
+    /** The model to use for the LLM-as-judge evaluation. */
+    evaluationModel?: string;
+    /** The maximum number of evaluation attempts before failing. */
+    maxAttempts?: number;
+    /** The provider to use for the evaluation model. */
+    provider?: string;
+    /** A custom evaluator function to override the default behavior. */
+    customEvaluator?: (options: LanguageModelV1CallOptions, result: GenerateResult) => Promise<{
+        evaluationResult: EvaluationResult;
+        evalContext: EnhancedEvaluationContext;
+    }>;
+    /** The score below which a response is considered off-topic. */
+    offTopicThreshold?: number;
+    /** The score below which a failing response is considered a high severity alert. */
+    highSeverityThreshold?: number;
+    /** An optional function to generate custom evaluation prompts. */
+    promptGenerator?: GetPromptFunction;
+}
+/**
+ * A function that generates the main body of an evaluation prompt.
+ */
+export type GetPromptFunction = (context: {
+    userQuery: string;
+    history: string;
+    tools: string;
+    retryInfo: string;
+    aiResponse: string;
+}) => string;

package/dist/types/evaluationTypes.js

@@ -0,0 +1 @@
+export {};

package/dist/types/middlewareTypes.d.ts

@@ -1,5 +1,7 @@
 import type { LanguageModelV1Middleware } from "ai";
 import type { JsonValue } from "../types/common.js";
+import type { EvaluationData } from "./evaluation.js";
+import type { GetPromptFunction } from "./evaluationTypes.js";
 /**
  * Metadata interface for NeuroLink middleware
  * Provides additional information about middleware without affecting execution
@@ -33,7 +35,7 @@ export interface MiddlewareConfig {
     /** Whether the middleware is enabled */
     enabled?: boolean;
     /** Middleware-specific configuration */
-    config?: Record<string, JsonValue>;
+    config?: Record<string, unknown>;
     /** Conditions under which to apply this middleware */
     conditions?: MiddlewareConditions;
 }
@@ -108,7 +110,7 @@ export interface MiddlewareChainStats {
 /**
  * Built-in middleware types
  */
-export type BuiltInMiddlewareType = "analytics" | "guardrails" | "logging" | "caching" | "rateLimit" | "retry" | "timeout";
+export type BuiltInMiddlewareType = "analytics" | "guardrails" | "logging" | "caching" | "rateLimit" | "retry" | "timeout" | "autoEvaluation";
 /**
  * Middleware preset configurations
  */
@@ -144,3 +146,27 @@ export interface MiddlewareFactoryOptions {
         collectStats?: boolean;
     };
 }
+/**
+ * Configuration for the Auto-Evaluation Middleware.
+ */
+export interface AutoEvaluationConfig {
+    /** The minimum score (1-10) for a response to be considered passing. */
+    threshold?: number;
+    /** The maximum number of retry attempts before failing. */
+    maxRetries?: number;
+    /** The model to use for the LLM-as-judge evaluation. */
+    evaluationModel?: string;
+    /**
+     * If true, the middleware will wait for the evaluation to complete before returning.
+     * If the evaluation fails, it will throw an error. Defaults to true.
+     */
+    blocking?: boolean;
+    /** A callback function to be invoked with the evaluation result. */
+    onEvaluationComplete?: (evaluation: EvaluationData) => void | Promise<void>;
+    /** The score below which a response is considered off-topic. */
+    offTopicThreshold?: number;
+    /** The score below which a failing response is considered a high severity alert. */
+    highSeverityThreshold?: number;
+    promptGenerator?: GetPromptFunction;
+    provider?: string;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@juspay/neurolink",
-  "version": "7.45.0",
+  "version": "7.46.0",
   "description": "Universal AI Development Platform with working MCP integration, multi-provider support, and professional CLI. Built-in tools operational, 58+ external MCP servers discoverable. Connect to filesystem, GitHub, database operations, and more. Build, test, and deploy AI applications with 9 major providers: OpenAI, Anthropic, Google AI, AWS Bedrock, Azure, Hugging Face, Ollama, and Mistral AI.",
   "author": {
     "name": "Juspay Technologies",