@juspay/neurolink 9.36.0 → 9.37.0

package/dist/evaluation/pipeline/evaluationPipeline.d.ts

@@ -0,0 +1,114 @@
+/**
+ * @file Evaluation Pipeline
+ * Multi-scorer orchestration with configurable execution
+ */
+import type { JsonObject } from "../../types/common.js";
+import type { AggregatedScores, PipelineConfig, Scorer, ScorerInput } from "../../types/scorerTypes.js";
+/**
+ * Pipeline execution options
+ */
+export type PipelineExecutionOptions = {
+    /** Correlation ID for tracing */
+    correlationId?: string;
+    /** Custom timeout override */
+    timeout?: number;
+    /** Skip specific scorers */
+    skipScorers?: string[];
+    /** Only run specific scorers */
+    onlyScorers?: string[];
+    /** Additional metadata to attach */
+    metadata?: JsonObject;
+};
+/**
+ * Pipeline execution result
+ */
+export type PipelineResult = AggregatedScores & {
+    /** Pipeline configuration used */
+    pipelineConfig: PipelineConfig;
+    /** Execution options used */
+    executionOptions?: PipelineExecutionOptions;
+    /** Errors that occurred during execution */
+    errors: Array<{
+        scorerId: string;
+        error: string;
+    }>;
+    /** Scorers that were skipped */
+    skippedScorers: string[];
+};
+/**
+ * Evaluation Pipeline for running multiple scorers
+ */
+export declare class EvaluationPipeline {
+    private _config;
+    private _scorers;
+    private _initialized;
+    constructor(config: PipelineConfig);
+    /**
+     * Get pipeline configuration
+     */
+    get config(): PipelineConfig;
+    /**
+     * Check if pipeline is initialized
+     */
+    get initialized(): boolean;
+    /**
+     * Initialize the pipeline by loading all scorers
+     */
+    initialize(): Promise<void>;
+    /**
+     * Execute the pipeline on input
+     */
+    execute(input: ScorerInput, options?: PipelineExecutionOptions): Promise<PipelineResult>;
+    /**
+     * Get scorers to run based on options
+     */
+    private _getScorersToRun;
+    /**
+     * Get list of skipped scorers
+     */
+    private _getSkippedScorers;
+    /**
+     * Execute a single scorer with timeout
+     */
+    private _executeScorer;
+    /**
+     * Aggregate scores based on configuration
+     */
+    /**
+     * Rescale a result's score to the default 0-MAX scale using its own scale info
+     */
+    private _rescaleToDefault;
+    private _aggregateScores;
+    /**
+     * Add a scorer to the pipeline
+     */
+    addScorer(id: string, scorer: Scorer): void;
+    /**
+     * Remove a scorer from the pipeline
+     */
+    removeScorer(id: string): boolean;
+    /**
+     * Get a scorer by ID
+     */
+    getScorer(id: string): Scorer | undefined;
+    /**
+     * Get all scorer IDs
+     */
+    getScorerIds(): string[];
+    /**
+     * Update pipeline configuration
+     */
+    configure(config: Partial<PipelineConfig>): void;
+    /**
+     * Create a clone of this pipeline
+     */
+    clone(): EvaluationPipeline;
+}
+/**
+ * Create a new evaluation pipeline
+ */
+export declare function createPipeline(config: PipelineConfig): EvaluationPipeline;
+/**
+ * Create and initialize a pipeline
+ */
+export declare function createAndInitializePipeline(config: PipelineConfig): Promise<EvaluationPipeline>;

package/dist/evaluation/pipeline/evaluationPipeline.js

@@ -0,0 +1,381 @@
+/**
+ * @file Evaluation Pipeline
+ * Multi-scorer orchestration with configurable execution
+ */
+import { logger } from "../../utils/logger.js";
+import { withTimeout } from "../../utils/errorHandling.js";
+import { DEFAULT_SCORE_SCALE } from "../scorers/baseScorer.js";
+import { ScorerRegistry } from "../scorers/scorerRegistry.js";
+/**
+ * Evaluation Pipeline for running multiple scorers
+ */
+export class EvaluationPipeline {
+    _config;
+    _scorers = new Map();
+    _initialized = false;
+    constructor(config) {
+        this._config = {
+            executionMode: "parallel",
+            stopOnFailure: false,
+            passThreshold: 0.7,
+            ...config,
+        };
+    }
+    /**
+     * Get pipeline configuration
+     */
+    get config() {
+        return this._config;
+    }
+    /**
+     * Check if pipeline is initialized
+     */
+    get initialized() {
+        return this._initialized;
+    }
+    /**
+     * Initialize the pipeline by loading all scorers
+     */
+    async initialize() {
+        if (this._initialized) {
+            return;
+        }
+        logger.debug(`Initializing evaluation pipeline: ${this._config.name ?? "unnamed"}`);
+        // Initialize registry
+        await ScorerRegistry.registerBuiltInScorers();
+        // Load all configured scorers using canonical IDs as map keys
+        for (const scorerDef of this._config.scorers) {
+            try {
+                const scorer = await ScorerRegistry.getScorer(scorerDef.id, scorerDef.config);
+                if (scorer) {
+                    const canonicalId = scorer.metadata.id;
+                    this._scorers.set(canonicalId, scorer);
+                    logger.debug(`Loaded scorer: ${scorerDef.id} (canonical: ${canonicalId})`);
+                }
+                else {
+                    logger.warn(`Scorer not found: ${scorerDef.id}`);
+                }
+            }
+            catch (error) {
+                logger.error(`Failed to load scorer: ${scorerDef.id}`, {
+                    error: error instanceof Error ? error.message : String(error),
+                });
+            }
+        }
+        // Normalize requiredScorers to canonical IDs
+        if (this._config.requiredScorers) {
+            this._config.requiredScorers = this._config.requiredScorers.map((id) => {
+                // Look up by alias first, then try canonical
+                for (const [canonicalId, scorer] of this._scorers) {
+                    if (scorer.metadata.id === id || canonicalId === id) {
+                        return canonicalId;
+                    }
+                }
+                return id;
+            });
+        }
+        // Validate required scorers are actually loaded
+        if (this._config.requiredScorers) {
+            const missing = this._config.requiredScorers.filter((id) => !this._scorers.has(id));
+            if (missing.length > 0) {
+                throw new Error(`Required scorers could not be loaded: ${missing.join(", ")}`);
+            }
+        }
+        this._initialized = true;
+        logger.debug(`Pipeline initialized with ${this._scorers.size} scorers`);
+    }
+    /**
+     * Execute the pipeline on input
+     */
+    async execute(input, options) {
+        if (!this._initialized) {
+            await this.initialize();
+        }
+        const startTime = Date.now();
+        const correlationId = options?.correlationId ?? `pipeline-${Date.now()}`;
+        logger.debug(`Executing pipeline: ${this._config.name ?? "unnamed"}`, {
+            correlationId,
+            scorerCount: this._scorers.size,
+        });
+        // Determine which scorers to run
+        const scorersToRun = this._getScorersToRun(options);
+        const skippedScorers = this._getSkippedScorers(options);
+        // Execute scorers
+        const results = [];
+        const errors = [];
+        if (this._config.executionMode === "parallel") {
+            // Parallel execution
+            const promises = scorersToRun.map(([id, scorer]) => this._executeScorer(id, scorer, input, options?.timeout));
+            const settledResults = await Promise.allSettled(promises);
+            for (let i = 0; i < settledResults.length; i++) {
+                const result = settledResults[i];
+                const [id] = scorersToRun[i];
+                if (result.status === "fulfilled") {
+                    results.push(result.value);
+                    if (result.value.error) {
+                        errors.push({ scorerId: id, error: result.value.error });
+                    }
+                }
+                else {
+                    errors.push({
+                        scorerId: id,
+                        error: result.reason?.message ?? "Unknown error",
+                    });
+                }
+            }
+        }
+        else {
+            // Sequential execution
+            for (const [id, scorer] of scorersToRun) {
+                try {
+                    const result = await this._executeScorer(id, scorer, input, options?.timeout);
+                    results.push(result);
+                    if (result.error) {
+                        errors.push({ scorerId: id, error: result.error });
+                    }
+                    // Check for stop on failure
+                    if (this._config.stopOnFailure && !result.passed) {
+                        logger.debug(`Stopping pipeline execution: scorer ${id} failed`);
+                        break;
+                    }
+                }
+                catch (error) {
+                    const errorMessage = error instanceof Error ? error.message : String(error);
+                    errors.push({ scorerId: id, error: errorMessage });
+                    if (this._config.stopOnFailure) {
+                        break;
+                    }
+                }
+            }
+        }
+        // Aggregate results
+        const aggregated = this._aggregateScores(results);
+        const totalComputeTime = Date.now() - startTime;
+        // Check required scorers
+        const requiredScorers = this._config.requiredScorers ?? [];
+        const allRequiredPassed = requiredScorers.every((id) => {
+            const result = results.find((r) => r.scorerId === id);
+            return result?.passed ?? false;
+        });
+        const overallPassed = aggregated.normalizedScore >= (this._config.passThreshold ?? 0.7) &&
+            allRequiredPassed;
+        return {
+            scores: results,
+            overallScore: aggregated.score,
+            aggregationMethod: this._config.aggregation?.method ?? "average",
+            passed: overallPassed,
+            totalComputeTime,
+            timestamp: Date.now(),
+            correlationId,
+            pipelineConfig: this._config,
+            executionOptions: options,
+            errors,
+            skippedScorers,
+        };
+    }
+    /**
+     * Get scorers to run based on options
+     */
+    _getScorersToRun(options) {
+        const allScorers = Array.from(this._scorers.entries());
+        if (options?.onlyScorers && options.onlyScorers.length > 0) {
+            return allScorers.filter(([id]) => options.onlyScorers.includes(id));
+        }
+        if (options?.skipScorers && options.skipScorers.length > 0) {
+            return allScorers.filter(([id]) => !options.skipScorers.includes(id));
+        }
+        return allScorers;
+    }
+    /**
+     * Get list of skipped scorers
+     */
+    _getSkippedScorers(options) {
+        const allIds = Array.from(this._scorers.keys());
+        if (options?.onlyScorers && options.onlyScorers.length > 0) {
+            return allIds.filter((id) => !options.onlyScorers.includes(id));
+        }
+        if (options?.skipScorers && options.skipScorers.length > 0) {
+            return options.skipScorers.filter((id) => allIds.includes(id));
+        }
+        return [];
+    }
+    /**
+     * Execute a single scorer with timeout
+     */
+    async _executeScorer(id, scorer, input, timeout) {
+        const scorerTimeout = timeout ?? scorer.config.timeout ?? this._config.timeout ?? 30000;
+        try {
+            const result = await withTimeout(scorer.score(input), scorerTimeout, new Error(`Scorer ${id} timed out after ${scorerTimeout}ms`));
+            return result;
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            return {
+                scorerId: id,
+                scorerName: scorer.metadata.name,
+                score: 0,
+                normalizedScore: 0,
+                scale: DEFAULT_SCORE_SCALE,
+                reasoning: `Scorer execution failed: ${errorMessage}`,
+                passed: false,
+                threshold: scorer.config.threshold ?? 0.7,
+                computeTime: 0,
+                error: errorMessage,
+            };
+        }
+    }
+    /**
+     * Aggregate scores based on configuration
+     */
+    /**
+     * Rescale a result's score to the default 0-MAX scale using its own scale info
+     */
+    _rescaleToDefault(result) {
+        const scale = result.scale ?? DEFAULT_SCORE_SCALE;
+        if (scale.max === scale.min) {
+            return 0;
+        }
+        // Normalize to 0-1 then rescale to default
+        const normalized = (result.score - scale.min) / (scale.max - scale.min);
+        return normalized * DEFAULT_SCORE_SCALE.max;
+    }
+    _aggregateScores(results) {
+        if (results.length === 0) {
+            return { score: 0, normalizedScore: 0 };
+        }
+        const aggregation = this._config.aggregation ?? { method: "average" };
+        const weights = aggregation.weights ?? {};
+        // Rescale all results to the common default scale before aggregation
+        const rescaled = results.map((r) => this._rescaleToDefault(r));
+        let score;
+        switch (aggregation.method) {
+            case "minimum":
+                score = Math.min(...rescaled);
+                break;
+            case "maximum":
+                score = Math.max(...rescaled);
+                break;
+            case "weighted": {
+                let totalWeight = 0;
+                let weightedSum = 0;
+                // Build a reverse map from canonical scorer ID to configured key
+                const configuredKeyMap = new Map();
+                for (const scorerDef of this._config.scorers) {
+                    const scorer = this._scorers.get(scorerDef.id);
+                    if (scorer) {
+                        configuredKeyMap.set(scorer.metadata.id, scorerDef.id);
+                    }
+                }
+                for (let i = 0; i < results.length; i++) {
+                    const result = results[i];
+                    const configuredKey = configuredKeyMap.get(result.scorerId);
+                    const weight = weights[result.scorerId] ??
+                        (configuredKey ? weights[configuredKey] : undefined) ??
+                        1.0;
+                    totalWeight += weight;
+                    weightedSum += rescaled[i] * weight;
+                }
+                score = totalWeight > 0 ? weightedSum / totalWeight : 0;
+                break;
+            }
+            case "custom":
+                if (aggregation.customFn) {
+                    score = aggregation.customFn(results);
+                    // Clamp custom output to valid range
+                    score = Math.max(0, Math.min(DEFAULT_SCORE_SCALE.max, score));
+                }
+                else {
+                    score = rescaled.reduce((sum, s) => sum + s, 0) / rescaled.length;
+                }
+                break;
+            case "average":
+            default:
+                score = rescaled.reduce((sum, s) => sum + s, 0) / rescaled.length;
+                break;
+        }
+        const normalizedScore = score / DEFAULT_SCORE_SCALE.max;
+        return { score, normalizedScore };
+    }
+    /**
+     * Add a scorer to the pipeline
+     */
+    addScorer(id, scorer) {
+        this._scorers.set(id, scorer);
+        // Update config
+        if (!this._config.scorers.some((s) => s.id === id)) {
+            this._config.scorers.push({ id });
+        }
+    }
+    /**
+     * Remove a scorer from the pipeline
+     */
+    removeScorer(id) {
+        const removed = this._scorers.delete(id);
+        if (removed) {
+            this._config.scorers = this._config.scorers.filter((s) => s.id !== id);
+            this._config.requiredScorers = this._config.requiredScorers?.filter((requiredId) => requiredId !== id);
+        }
+        return removed;
+    }
+    /**
+     * Get a scorer by ID
+     */
+    getScorer(id) {
+        return this._scorers.get(id);
+    }
+    /**
+     * Get all scorer IDs
+     */
+    getScorerIds() {
+        return Array.from(this._scorers.keys());
+    }
+    /**
+     * Update pipeline configuration
+     */
+    configure(config) {
+        this._config = { ...this._config, ...config };
+    }
+    /**
+     * Create a clone of this pipeline
+     */
+    clone() {
+        const clonedConfig = {
+            ...this._config,
+            scorers: this._config.scorers.map((s) => ({
+                id: s.id,
+                config: s.config ? { ...s.config } : undefined,
+            })),
+            aggregation: this._config.aggregation
+                ? {
+                    ...this._config.aggregation,
+                    weights: this._config.aggregation.weights
+                        ? { ...this._config.aggregation.weights }
+                        : undefined,
+                }
+                : undefined,
+            requiredScorers: this._config.requiredScorers
+                ? [...this._config.requiredScorers]
+                : undefined,
+        };
+        const cloned = new EvaluationPipeline(clonedConfig);
+        // Do not copy scorer instances to avoid shared mutable state
+        // (e.g., BaseLLMScorer.provider, initializationPromise).
+        // The cloned pipeline will create fresh scorers on initialize().
+        cloned._initialized = false;
+        return cloned;
+    }
+}
+/**
+ * Create a new evaluation pipeline
+ */
+export function createPipeline(config) {
+    return new EvaluationPipeline(config);
+}
+/**
+ * Create and initialize a pipeline
+ */
+export async function createAndInitializePipeline(config) {
+    const pipeline = new EvaluationPipeline(config);
+    await pipeline.initialize();
+    return pipeline;
+}

package/dist/evaluation/pipeline/index.d.ts

@@ -0,0 +1,8 @@
+/**
+ * @file Pipeline Index
+ * Export all pipeline components
+ */
+export { createAndInitializePipeline, createPipeline, EvaluationPipeline, type PipelineExecutionOptions, type PipelineResult, } from "./evaluationPipeline.js";
+export { PipelineBuilder, Pipelines } from "./pipelineBuilder.js";
+export { CODE_GENERATION_PIPELINE, COMPREHENSIVE_PIPELINE, CUSTOMER_SUPPORT_PIPELINE, getPreset, getPresetNames, MINIMAL_PIPELINE, PipelinePresets, QUALITY_PIPELINE, RAG_PIPELINE, SAFETY_PIPELINE, SUMMARIZATION_PIPELINE, } from "./presets.js";
+export * from "./strategies/index.js";

package/dist/evaluation/pipeline/index.js

@@ -0,0 +1,8 @@
+/**
+ * @file Pipeline Index
+ * Export all pipeline components
+ */
+export { createAndInitializePipeline, createPipeline, EvaluationPipeline, } from "./evaluationPipeline.js";
+export { PipelineBuilder, Pipelines } from "./pipelineBuilder.js";
+export { CODE_GENERATION_PIPELINE, COMPREHENSIVE_PIPELINE, CUSTOMER_SUPPORT_PIPELINE, getPreset, getPresetNames, MINIMAL_PIPELINE, PipelinePresets, QUALITY_PIPELINE, RAG_PIPELINE, SAFETY_PIPELINE, SUMMARIZATION_PIPELINE, } from "./presets.js";
+export * from "./strategies/index.js";

package/dist/evaluation/pipeline/pipelineBuilder.d.ts

@@ -0,0 +1,126 @@
+/**
+ * @file Pipeline Builder
+ * Fluent builder API for creating evaluation pipelines
+ */
+import type { AggregationMethod, PipelineConfig, ScoreResult, ScorerConfig } from "../../types/scorerTypes.js";
+import { EvaluationPipeline } from "./evaluationPipeline.js";
+/**
+ * Fluent builder for creating evaluation pipelines
+ */
+export declare class PipelineBuilder {
+    private _name?;
+    private _description?;
+    private _scorers;
+    private _aggregation;
+    private _passThreshold;
+    private _executionMode;
+    private _stopOnFailure;
+    private _timeout?;
+    private _requiredScorers;
+    constructor(name?: string);
+    /**
+     * Create a new pipeline builder
+     */
+    static create(name?: string): PipelineBuilder;
+    /**
+     * Set pipeline name
+     */
+    name(name: string): this;
+    /**
+     * Set pipeline description
+     */
+    description(desc: string): this;
+    /**
+     * Add a scorer by ID
+     */
+    addScorer(id: string, config?: ScorerConfig): this;
+    /**
+     * Add multiple scorers
+     */
+    addScorers(...ids: string[]): this;
+    /**
+     * Add a scorer and mark it as required
+     */
+    requireScorer(id: string, config?: ScorerConfig): this;
+    /**
+     * Set aggregation method
+     */
+    aggregateWith(method: AggregationMethod): this;
+    /**
+     * Set weights for weighted aggregation
+     */
+    withWeights(weights: Record<string, number>): this;
+    /**
+     * Set custom aggregation function
+     */
+    customAggregation(fn: (scores: ScoreResult[]) => number): this;
+    /**
+     * Set pass/fail threshold
+     */
+    passThreshold(threshold: number): this;
+    /**
+     * Run scorers in parallel (default)
+     */
+    parallel(): this;
+    /**
+     * Run scorers sequentially
+     */
+    sequential(): this;
+    /**
+     * Stop pipeline on first failure
+     */
+    stopOnFailure(): this;
+    /**
+     * Continue pipeline on failures (default)
+     */
+    continueOnFailure(): this;
+    /**
+     * Set pipeline timeout
+     */
+    timeout(ms: number): this;
+    /**
+     * Build the pipeline configuration
+     */
+    buildConfig(): PipelineConfig;
+    /**
+     * Build the pipeline (not initialized)
+     */
+    build(): EvaluationPipeline;
+    /**
+     * Build and initialize the pipeline
+     */
+    buildAndInitialize(): Promise<EvaluationPipeline>;
+}
+/**
+ * Quick pipeline builder factory
+ */
+export declare const Pipelines: {
+    /**
+     * Create a new pipeline builder
+     */
+    create: (name?: string) => PipelineBuilder;
+    /**
+     * Create a safety-focused pipeline
+     */
+    safety: () => PipelineBuilder;
+    /**
+     * Create a RAG evaluation pipeline
+     */
+    rag: () => PipelineBuilder;
+    /**
+     * Create a quality-focused pipeline
+     */
+    quality: () => PipelineBuilder;
+    /**
+     * Create a comprehensive pipeline with all scorers
+     */
+    comprehensive: () => PipelineBuilder;
+    /**
+     * Create a minimal fast pipeline
+     */
+    minimal: () => PipelineBuilder;
+    /**
+     * Create a summarization evaluation pipeline
+     */
+    summarization: () => PipelineBuilder;
+};