@juspay/neurolink 9.36.1 → 9.37.0

package/dist/lib/evaluation/EvaluatorFactory.d.ts

@@ -0,0 +1,113 @@
+/**
+ * @file EvaluatorFactory - Factory for creating evaluator instances.
+ * Extends BaseFactory to provide dynamic evaluator creation with configuration support.
+ */
+import { BaseFactory } from "../core/infrastructure/index.js";
+import type { EvaluationConfig } from "../types/evaluationTypes.js";
+import { Evaluator } from "./index.js";
+/**
+ * Configuration presets for common evaluation scenarios.
+ */
+export interface EvaluatorPreset {
+    /** Preset name for identification */
+    name: string;
+    /** Description of the preset use case */
+    description: string;
+    /** The underlying evaluation configuration (optional for built-in presets) */
+    config?: EvaluationConfig;
+}
+/**
+ * Factory for creating Evaluator instances with various configurations.
+ * Supports presets for common use cases and custom configurations.
+ *
+ * @example
+ * ```typescript
+ * const factory = EvaluatorFactory.getInstance();
+ *
+ * // Create with default configuration
+ * const evaluator = await factory.create('default');
+ *
+ * // Create with a preset
+ * const strictEvaluator = await factory.create('strict');
+ *
+ * // Create with custom config
+ * const customEvaluator = await factory.create('default', {
+ *   threshold: 9,
+ *   evaluationModel: 'gpt-4',
+ *   provider: 'openai'
+ * });
+ * ```
+ */
+export declare class EvaluatorFactory extends BaseFactory<Evaluator, EvaluationConfig> {
+    private static instance;
+    private constructor();
+    /**
+     * Gets the singleton instance of the EvaluatorFactory.
+     */
+    static getInstance(): EvaluatorFactory;
+    /**
+     * Resets the singleton instance (useful for testing).
+     */
+    static resetInstance(): void;
+    /**
+     * Registers all built-in evaluator configurations.
+     * This is called automatically on first access.
+     */
+    protected registerAll(): Promise<void>;
+    /**
+     * Creates an evaluator instance with the specified preset and optional config overrides.
+     *
+     * @param presetOrName - The preset name or alias
+     * @param config - Optional configuration overrides
+     * @returns A configured Evaluator instance
+     */
+    createEvaluator(presetOrName?: string, config?: EvaluationConfig): Promise<Evaluator>;
+    /**
+     * Creates an evaluator with a fully custom configuration (not based on a preset).
+     *
+     * @param config - The evaluation configuration
+     * @returns A configured Evaluator instance
+     */
+    createCustomEvaluator(config: EvaluationConfig): Evaluator;
+    /**
+     * Gets information about a preset by name or alias.
+     *
+     * @param presetOrName - The preset name or alias
+     * @returns The preset information or undefined if not found
+     */
+    getPresetInfo(presetOrName: string): Promise<EvaluatorPreset | undefined>;
+    /**
+     * Lists all available presets with their descriptions.
+     *
+     * @returns Array of preset information
+     */
+    listPresets(): Promise<Array<{
+        name: string;
+        aliases: string[];
+        preset: EvaluatorPreset;
+    }>>;
+    /**
+     * Validates an evaluation configuration.
+     *
+     * @param config - The configuration to validate
+     * @throws {NeuroLinkFeatureError} If the configuration is invalid
+     */
+    validateConfig(config: EvaluationConfig): void;
+    /**
+     * Registers a custom evaluator preset.
+     *
+     * @param name - Unique name for the preset
+     * @param config - The evaluation configuration for this preset
+     * @param aliases - Alternative names for the preset
+     * @param description - Human-readable description
+     */
+    registerPreset(name: string, config: EvaluationConfig, aliases?: string[], description?: string): void;
+    /**
+     * Unregisters a preset from the factory.
+     *
+     * @param name - The preset name to remove
+     * @returns true if the preset was removed, false if it didn't exist
+     */
+    unregisterPreset(name: string): boolean;
+}
+export declare const getEvaluatorFactory: () => EvaluatorFactory;

package/dist/lib/evaluation/EvaluatorFactory.js

@@ -0,0 +1,281 @@
+/**
+ * @file EvaluatorFactory - Factory for creating evaluator instances.
+ * Extends BaseFactory to provide dynamic evaluator creation with configuration support.
+ */
+import { BaseFactory } from "../core/infrastructure/index.js";
+import { Evaluator } from "./index.js";
+import { createConfigurationError } from "./errors/EvaluationError.js";
+/**
+ * Factory for creating Evaluator instances with various configurations.
+ * Supports presets for common use cases and custom configurations.
+ *
+ * @example
+ * ```typescript
+ * const factory = EvaluatorFactory.getInstance();
+ *
+ * // Create with default configuration
+ * const evaluator = await factory.create('default');
+ *
+ * // Create with a preset
+ * const strictEvaluator = await factory.create('strict');
+ *
+ * // Create with custom config
+ * const customEvaluator = await factory.create('default', {
+ *   threshold: 9,
+ *   evaluationModel: 'gpt-4',
+ *   provider: 'openai'
+ * });
+ * ```
+ */
+export class EvaluatorFactory extends BaseFactory {
+    static instance = null;
+    constructor() {
+        super();
+    }
+    /**
+     * Gets the singleton instance of the EvaluatorFactory.
+     */
+    static getInstance() {
+        if (!EvaluatorFactory.instance) {
+            EvaluatorFactory.instance = new EvaluatorFactory();
+        }
+        return EvaluatorFactory.instance;
+    }
+    /**
+     * Resets the singleton instance (useful for testing).
+     */
+    static resetInstance() {
+        EvaluatorFactory.instance = null;
+    }
+    /**
+     * Registers all built-in evaluator configurations.
+     * This is called automatically on first access.
+     */
+    async registerAll() {
+        // Register default configuration
+        this.register("default", async (config) => {
+            const mergedConfig = {
+                threshold: 7,
+                evaluationStrategy: "ragas",
+                evaluationModel: process.env.NEUROLINK_RAGAS_EVALUATION_MODEL || "gemini-1.5-flash",
+                provider: process.env.NEUROLINK_RAGAS_EVALUATION_PROVIDER || "vertex",
+                ...config,
+            };
+            if (config) {
+                this.validateConfig(mergedConfig);
+            }
+            return new Evaluator(mergedConfig);
+        }, ["standard", "basic"], {
+            preset: {
+                name: "Default",
+                description: "Standard evaluation with balanced settings",
+            },
+        });
+        // Register strict configuration (higher threshold)
+        this.register("strict", async (config) => {
+            const mergedConfig = {
+                threshold: 8,
+                evaluationStrategy: "ragas",
+                evaluationModel: "gpt-4",
+                provider: "openai",
+                ...config,
+            };
+            if (config) {
+                this.validateConfig(mergedConfig);
+            }
+            return new Evaluator(mergedConfig);
+        }, ["high-quality", "production"], {
+            preset: {
+                name: "Strict",
+                description: "Strict evaluation with higher quality threshold (8/10)",
+            },
+        });
+        // Register lenient configuration (lower threshold)
+        this.register("lenient", async (config) => {
+            const mergedConfig = {
+                threshold: 5,
+                evaluationStrategy: "ragas",
+                evaluationModel: process.env.NEUROLINK_RAGAS_EVALUATION_MODEL || "gemini-1.5-flash",
+                provider: process.env.NEUROLINK_RAGAS_EVALUATION_PROVIDER || "vertex",
+                ...config,
+            };
+            if (config) {
+                this.validateConfig(mergedConfig);
+            }
+            return new Evaluator(mergedConfig);
+        }, ["relaxed", "development"], {
+            preset: {
+                name: "Lenient",
+                description: "Lenient evaluation with lower threshold for development (5/10)",
+            },
+        });
+        // Register fast configuration (optimized for speed)
+        this.register("fast", async (config) => {
+            const mergedConfig = {
+                threshold: 6,
+                evaluationStrategy: "ragas",
+                evaluationModel: "gemini-1.5-flash",
+                provider: "vertex",
+                ...config,
+            };
+            if (config) {
+                this.validateConfig(mergedConfig);
+            }
+            return new Evaluator(mergedConfig);
+        }, ["quick", "speed"], {
+            preset: {
+                name: "Fast",
+                description: "Fast evaluation optimized for speed with lighter model",
+            },
+        });
+        // Register premium configuration (highest quality)
+        this.register("premium", async (config) => {
+            const mergedConfig = {
+                threshold: 9,
+                evaluationStrategy: "ragas",
+                evaluationModel: "gpt-4-turbo",
+                provider: "openai",
+                ...config,
+            };
+            if (config) {
+                this.validateConfig(mergedConfig);
+            }
+            return new Evaluator(mergedConfig);
+        }, ["enterprise", "highest-quality"], {
+            preset: {
+                name: "Premium",
+                description: "Premium evaluation with highest quality model and strictest threshold (9/10)",
+            },
+        });
+    }
+    /**
+     * Creates an evaluator instance with the specified preset and optional config overrides.
+     *
+     * @param presetOrName - The preset name or alias
+     * @param config - Optional configuration overrides
+     * @returns A configured Evaluator instance
+     */
+    async createEvaluator(presetOrName = "default", config) {
+        return this.create(presetOrName, config);
+    }
+    /**
+     * Creates an evaluator with a fully custom configuration (not based on a preset).
+     *
+     * @param config - The evaluation configuration
+     * @returns A configured Evaluator instance
+     */
+    createCustomEvaluator(config) {
+        this.validateConfig(config);
+        return new Evaluator(config);
+    }
+    /**
+     * Gets information about a preset by name or alias.
+     *
+     * @param presetOrName - The preset name or alias
+     * @returns The preset information or undefined if not found
+     */
+    async getPresetInfo(presetOrName) {
+        await this.ensureInitialized();
+        const name = this.resolveName(presetOrName);
+        const registration = this.items.get(name);
+        if (!registration?.metadata?.preset) {
+            return undefined;
+        }
+        return registration.metadata.preset;
+    }
+    /**
+     * Lists all available presets with their descriptions.
+     *
+     * @returns Array of preset information
+     */
+    async listPresets() {
+        await this.ensureInitialized();
+        const presets = [];
+        const entries = Array.from(this.items.entries());
+        for (const [name, registration] of entries) {
+            if (registration.metadata?.preset) {
+                presets.push({
+                    name,
+                    aliases: registration.aliases,
+                    preset: registration.metadata.preset,
+                });
+            }
+        }
+        return presets;
+    }
+    /**
+     * Validates an evaluation configuration.
+     *
+     * @param config - The configuration to validate
+     * @throws {NeuroLinkFeatureError} If the configuration is invalid
+     */
+    validateConfig(config) {
+        if (config.threshold !== undefined) {
+            if (config.threshold < 1 || config.threshold > 10) {
+                throw createConfigurationError("Evaluation threshold must be between 1 and 10", "threshold out of range");
+            }
+        }
+        if (config.evaluationStrategy === "custom" && !config.customEvaluator) {
+            throw createConfigurationError("Custom evaluation strategy requires a customEvaluator function", "missing customEvaluator");
+        }
+        if (config.offTopicThreshold !== undefined) {
+            if (config.offTopicThreshold < 1 || config.offTopicThreshold > 10) {
+                throw createConfigurationError("Off-topic threshold must be between 1 and 10", "offTopicThreshold out of range");
+            }
+        }
+        if (config.highSeverityThreshold !== undefined) {
+            if (config.highSeverityThreshold < 1 ||
+                config.highSeverityThreshold > 10) {
+                throw createConfigurationError("High severity threshold must be between 1 and 10", "highSeverityThreshold out of range");
+            }
+        }
+    }
+    /**
+     * Registers a custom evaluator preset.
+     *
+     * @param name - Unique name for the preset
+     * @param config - The evaluation configuration for this preset
+     * @param aliases - Alternative names for the preset
+     * @param description - Human-readable description
+     */
+    registerPreset(name, config, aliases = [], description = "") {
+        this.validateConfig(config);
+        this.register(name, async (overrides) => {
+            const mergedConfig = {
+                ...config,
+                ...overrides,
+            };
+            if (overrides) {
+                this.validateConfig(mergedConfig);
+            }
+            return new Evaluator(mergedConfig);
+        }, aliases, {
+            preset: {
+                name,
+                description,
+                config,
+            },
+        });
+    }
+    /**
+     * Unregisters a preset from the factory.
+     *
+     * @param name - The preset name to remove
+     * @returns true if the preset was removed, false if it didn't exist
+     */
+    unregisterPreset(name) {
+        const registration = this.items.get(name);
+        if (registration) {
+            // Remove aliases
+            for (const alias of registration.aliases) {
+                this.aliasMap.delete(alias.toLowerCase());
+            }
+            this.items.delete(name);
+            return true;
+        }
+        return false;
+    }
+}
+// Export singleton instance getter for convenience
+export const getEvaluatorFactory = () => EvaluatorFactory.getInstance();
+//# sourceMappingURL=EvaluatorFactory.js.map

package/dist/lib/evaluation/EvaluatorRegistry.d.ts

@@ -0,0 +1,160 @@
+/**
+ * @file EvaluatorRegistry - Registry for evaluation strategies.
+ * Extends BaseRegistry to provide dynamic strategy registration and lookup.
+ */
+import { BaseRegistry } from "../core/infrastructure/index.js";
+import type { LanguageModelV3CallOptions } from "@ai-sdk/provider";
+import type { GenerateResult } from "../types/generateTypes.js";
+import type { EvaluationResult, EnhancedEvaluationContext } from "../types/evaluationTypes.js";
+/**
+ * A function that performs evaluation and returns results.
+ */
+export type EvaluationStrategyFunction = (options: LanguageModelV3CallOptions, result: GenerateResult, config?: EvaluationStrategyConfig) => Promise<{
+    evaluationResult: EvaluationResult;
+    evalContext: EnhancedEvaluationContext;
+}>;
+/**
+ * Configuration for evaluation strategies.
+ */
+export interface EvaluationStrategyConfig {
+    /** The model to use for evaluation */
+    evaluationModel?: string;
+    /** The provider to use for evaluation */
+    provider?: string;
+    /** The passing threshold (1-10) */
+    threshold?: number;
+    /** Custom prompt generator */
+    promptGenerator?: (context: {
+        userQuery: string;
+        history: string;
+        tools: string;
+        retryInfo: string;
+        aiResponse: string;
+    }) => string;
+    /** Additional strategy-specific options */
+    options?: Record<string, unknown>;
+}
+/**
+ * Metadata for registered evaluation strategies.
+ */
+export interface EvaluationStrategyMetadata {
+    /** Human-readable name for the strategy */
+    name: string;
+    /** Description of what the strategy does */
+    description: string;
+    /** Whether the strategy requires an external LLM */
+    requiresLLM: boolean;
+    /** Default model for the strategy (if requiresLLM is true) */
+    defaultModel?: string;
+    /** Default provider for the strategy (if requiresLLM is true) */
+    defaultProvider?: string;
+    /** Version of the strategy */
+    version: string;
+    /** Supported features */
+    features: string[];
+}
+/**
+ * Registry for evaluation strategies.
+ * Allows dynamic registration and retrieval of evaluation strategies.
+ *
+ * @example
+ * ```typescript
+ * // Register a custom strategy
+ * EvaluatorRegistry.getInstance().registerStrategy(
+ *   'custom-ragas',
+ *   async () => ({
+ *     evaluate: async (options, result) => { ... }
+ *   }),
+ *   {
+ *     name: 'Custom RAGAS',
+ *     description: 'Custom RAGAS implementation',
+ *     requiresLLM: true,
+ *     defaultModel: 'gpt-4',
+ *     defaultProvider: 'openai',
+ *     version: '1.0.0',
+ *     features: ['custom-metrics']
+ *   }
+ * );
+ *
+ * // Get a strategy
+ * const strategy = await EvaluatorRegistry.getInstance().getStrategy('ragas');
+ * ```
+ */
+export declare class EvaluatorRegistry extends BaseRegistry<EvaluationStrategyFunction, EvaluationStrategyMetadata> {
+    private static instance;
+    private constructor();
+    /**
+     * Gets the singleton instance of the EvaluatorRegistry.
+     */
+    static getInstance(): EvaluatorRegistry;
+    /**
+     * Resets the singleton instance (useful for testing).
+     */
+    static resetInstance(): void;
+    /**
+     * Registers all built-in evaluation strategies.
+     * This is called automatically on first access.
+     */
+    protected registerAll(): Promise<void>;
+    /**
+     * Registers an evaluation strategy with the registry.
+     *
+     * @param id - Unique identifier for the strategy
+     * @param factory - Factory function that creates the strategy
+     * @param metadata - Metadata about the strategy
+     */
+    registerStrategy(id: string, factory: () => Promise<EvaluationStrategyFunction>, metadata: EvaluationStrategyMetadata): void;
+    /**
+     * Gets an evaluation strategy by ID.
+     *
+     * @param id - The strategy identifier
+     * @returns The evaluation strategy function
+     * @throws {NeuroLinkFeatureError} If the strategy is not found
+     */
+    getStrategy(id: string): Promise<EvaluationStrategyFunction>;
+    /**
+     * Checks if a strategy exists in the registry.
+     *
+     * @param id - The strategy identifier
+     * @returns true if the strategy exists
+     */
+    hasStrategy(id: string): Promise<boolean>;
+    /**
+     * Lists all registered strategies with their metadata.
+     *
+     * @returns Array of strategy IDs and their metadata
+     */
+    listStrategies(): Promise<Array<{
+        id: string;
+        metadata: EvaluationStrategyMetadata;
+    }>>;
+    /**
+     * Gets the metadata for a specific strategy.
+     *
+     * @param id - The strategy identifier
+     * @returns The strategy metadata or undefined if not found
+     */
+    getStrategyMetadata(id: string): Promise<EvaluationStrategyMetadata | undefined>;
+    /**
+     * Unregisters a strategy from the registry.
+     *
+     * @param id - The strategy identifier
+     * @returns true if the strategy was removed, false if it didn't exist
+     */
+    unregisterStrategy(id: string): Promise<boolean>;
+    /**
+     * Gets strategies that support a specific feature.
+     *
+     * @param feature - The feature to filter by
+     * @returns Array of strategy IDs that support the feature
+     */
+    getStrategiesWithFeature(feature: string): Promise<string[]>;
+    /**
+     * Gets strategies that use a specific provider.
+     *
+     * @param provider - The provider to filter by
+     * @returns Array of strategy IDs that use the provider
+     */
+    getStrategiesByProvider(provider: string): Promise<string[]>;
+}
+export declare const getEvaluatorRegistry: () => EvaluatorRegistry;