@juspay/neurolink 9.36.1 → 9.38.0

package/dist/evaluation/pipeline/pipelineBuilder.js

@@ -0,0 +1,260 @@
+/**
+ * @file Pipeline Builder
+ * Fluent builder API for creating evaluation pipelines
+ */
+import { EvaluationPipeline } from "./evaluationPipeline.js";
+/**
+ * Fluent builder for creating evaluation pipelines
+ */
+export class PipelineBuilder {
+    _name;
+    _description;
+    _scorers = [];
+    _aggregation = { method: "average" };
+    _passThreshold = 0.7;
+    _executionMode = "parallel";
+    _stopOnFailure = false;
+    _timeout;
+    _requiredScorers = [];
+    constructor(name) {
+        this._name = name;
+    }
+    /**
+     * Create a new pipeline builder
+     */
+    static create(name) {
+        return new PipelineBuilder(name);
+    }
+    /**
+     * Set pipeline name
+     */
+    name(name) {
+        this._name = name;
+        return this;
+    }
+    /**
+     * Set pipeline description
+     */
+    description(desc) {
+        this._description = desc;
+        return this;
+    }
+    /**
+     * Add a scorer by ID
+     */
+    addScorer(id, config) {
+        this._scorers.push({ id, config });
+        return this;
+    }
+    /**
+     * Add multiple scorers
+     */
+    addScorers(...ids) {
+        for (const id of ids) {
+            this._scorers.push({ id });
+        }
+        return this;
+    }
+    /**
+     * Add a scorer and mark it as required
+     */
+    requireScorer(id, config) {
+        const existing = this._scorers.find((scorer) => scorer.id === id);
+        if (existing) {
+            existing.config = {
+                ...existing.config,
+                ...config,
+            };
+        }
+        else {
+            this._scorers.push({ id, config });
+        }
+        if (!this._requiredScorers.includes(id)) {
+            this._requiredScorers.push(id);
+        }
+        return this;
+    }
+    /**
+     * Set aggregation method
+     */
+    aggregateWith(method) {
+        this._aggregation.method = method;
+        return this;
+    }
+    /**
+     * Set weights for weighted aggregation
+     */
+    withWeights(weights) {
+        this._aggregation.method = "weighted";
+        this._aggregation.weights = weights;
+        return this;
+    }
+    /**
+     * Set custom aggregation function
+     */
+    customAggregation(fn) {
+        this._aggregation.method = "custom";
+        this._aggregation.customFn = fn;
+        return this;
+    }
+    /**
+     * Set pass/fail threshold
+     */
+    passThreshold(threshold) {
+        this._passThreshold = threshold;
+        return this;
+    }
+    /**
+     * Run scorers in parallel (default)
+     */
+    parallel() {
+        this._executionMode = "parallel";
+        return this;
+    }
+    /**
+     * Run scorers sequentially
+     */
+    sequential() {
+        this._executionMode = "sequential";
+        return this;
+    }
+    /**
+     * Stop pipeline on first failure
+     */
+    stopOnFailure() {
+        this._stopOnFailure = true;
+        return this;
+    }
+    /**
+     * Continue pipeline on failures (default)
+     */
+    continueOnFailure() {
+        this._stopOnFailure = false;
+        return this;
+    }
+    /**
+     * Set pipeline timeout
+     */
+    timeout(ms) {
+        this._timeout = ms;
+        return this;
+    }
+    /**
+     * Build the pipeline configuration
+     */
+    buildConfig() {
+        return {
+            name: this._name,
+            description: this._description,
+            scorers: this._scorers.map((scorer) => ({
+                id: scorer.id,
+                config: scorer.config ? { ...scorer.config } : undefined,
+            })),
+            aggregation: {
+                ...this._aggregation,
+                weights: this._aggregation.weights
+                    ? { ...this._aggregation.weights }
+                    : undefined,
+            },
+            passThreshold: this._passThreshold,
+            executionMode: this._executionMode,
+            stopOnFailure: this._stopOnFailure,
+            timeout: this._timeout,
+            requiredScorers: this._requiredScorers.length > 0
+                ? [...this._requiredScorers]
+                : undefined,
+        };
+    }
+    /**
+     * Build the pipeline (not initialized)
+     */
+    build() {
+        return new EvaluationPipeline(this.buildConfig());
+    }
+    /**
+     * Build and initialize the pipeline
+     */
+    async buildAndInitialize() {
+        const pipeline = this.build();
+        await pipeline.initialize();
+        return pipeline;
+    }
+}
+/**
+ * Quick pipeline builder factory
+ */
+export const Pipelines = {
+    /**
+     * Create a new pipeline builder
+     */
+    create: (name) => PipelineBuilder.create(name),
+    /**
+     * Create a safety-focused pipeline
+     */
+    safety: () => PipelineBuilder.create("safety")
+        .description("Safety evaluation pipeline")
+        .addScorers("toxicity", "bias-detection")
+        .requireScorer("toxicity")
+        .aggregateWith("minimum")
+        .passThreshold(0.8),
+    /**
+     * Create a RAG evaluation pipeline
+     */
+    rag: () => PipelineBuilder.create("rag")
+        .description("RAG evaluation pipeline")
+        .addScorers("faithfulness", "context-relevancy", "answer-relevancy", "hallucination")
+        .withWeights({
+        faithfulness: 1.5,
+        "context-relevancy": 1.0,
+        "answer-relevancy": 1.0,
+        hallucination: 1.5,
+    })
+        .passThreshold(0.7),
+    /**
+     * Create a quality-focused pipeline
+     */
+    quality: () => PipelineBuilder.create("quality")
+        .description("Quality evaluation pipeline")
+        .addScorers("tone-consistency", "prompt-alignment", "length", "format")
+        .aggregateWith("average")
+        .passThreshold(0.7),
+    /**
+     * Create a comprehensive pipeline with all scorers
+     */
+    comprehensive: () => PipelineBuilder.create("comprehensive")
+        .description("Comprehensive evaluation pipeline")
+        .addScorers("toxicity", "bias-detection", "hallucination", "faithfulness", "context-relevancy", "answer-relevancy", "tone-consistency", "prompt-alignment")
+        .requireScorer("toxicity")
+        .withWeights({
+        toxicity: 2.0,
+        "bias-detection": 1.5,
+        hallucination: 1.5,
+        faithfulness: 1.0,
+        "context-relevancy": 1.0,
+        "answer-relevancy": 1.0,
+        "tone-consistency": 0.8,
+        "prompt-alignment": 0.8,
+    })
+        .passThreshold(0.75),
+    /**
+     * Create a minimal fast pipeline
+     */
+    minimal: () => PipelineBuilder.create("minimal")
+        .description("Minimal fast evaluation pipeline")
+        .addScorers("toxicity", "hallucination")
+        .parallel()
+        .passThreshold(0.8),
+    /**
+     * Create a summarization evaluation pipeline
+     */
+    summarization: () => PipelineBuilder.create("summarization")
+        .description("Summarization quality evaluation pipeline")
+        .addScorers("summarization", "faithfulness", "content-similarity", "length")
+        .withWeights({
+        summarization: 1.5,
+        faithfulness: 1.2,
+        "content-similarity": 1.0,
+        length: 0.8,
+    })
+        .passThreshold(0.7),
+};

package/dist/evaluation/pipeline/presets.d.ts

@@ -0,0 +1,66 @@
+/**
+ * @file Pipeline Presets
+ * Pre-configured evaluation pipelines for common use cases
+ */
+import type { PipelineConfig } from "../../types/scorerTypes.js";
+/**
+ * Safety evaluation preset
+ * Focuses on content safety: toxicity, bias, harmful content
+ */
+export declare const SAFETY_PIPELINE: PipelineConfig;
+/**
+ * RAG evaluation preset
+ * Evaluates Retrieval Augmented Generation quality
+ */
+export declare const RAG_PIPELINE: PipelineConfig;
+/**
+ * Quality evaluation preset
+ * Focuses on response quality: format, length, tone
+ */
+export declare const QUALITY_PIPELINE: PipelineConfig;
+/**
+ * Comprehensive evaluation preset
+ * Full evaluation across all dimensions
+ */
+export declare const COMPREHENSIVE_PIPELINE: PipelineConfig;
+/**
+ * Minimal/fast evaluation preset
+ * Quick checks for high-throughput scenarios
+ */
+export declare const MINIMAL_PIPELINE: PipelineConfig;
+/**
+ * Summarization evaluation preset
+ * Evaluates summarization quality
+ */
+export declare const SUMMARIZATION_PIPELINE: PipelineConfig;
+/**
+ * Customer support evaluation preset
+ * Tailored for customer service responses
+ */
+export declare const CUSTOMER_SUPPORT_PIPELINE: PipelineConfig;
+/**
+ * Code generation evaluation preset
+ * Evaluates generated code quality
+ */
+export declare const CODE_GENERATION_PIPELINE: PipelineConfig;
+/**
+ * All available presets
+ */
+export declare const PipelinePresets: {
+    readonly safety: PipelineConfig;
+    readonly rag: PipelineConfig;
+    readonly quality: PipelineConfig;
+    readonly comprehensive: PipelineConfig;
+    readonly minimal: PipelineConfig;
+    readonly summarization: PipelineConfig;
+    readonly customerSupport: PipelineConfig;
+    readonly codeGeneration: PipelineConfig;
+};
+/**
+ * Get a preset pipeline configuration by name
+ */
+export declare function getPreset(name: keyof typeof PipelinePresets): PipelineConfig;
+/**
+ * Get all available preset names
+ */
+export declare function getPresetNames(): string[];

package/dist/evaluation/pipeline/presets.js

@@ -0,0 +1,224 @@
+/**
+ * @file Pipeline Presets
+ * Pre-configured evaluation pipelines for common use cases
+ */
+/**
+ * Safety evaluation preset
+ * Focuses on content safety: toxicity, bias, harmful content
+ */
+export const SAFETY_PIPELINE = {
+    name: "safety",
+    description: "Safety evaluation pipeline for detecting harmful content",
+    scorers: [
+        { id: "toxicity", config: { threshold: 0.9, weight: 2.0 } },
+        { id: "bias-detection", config: { threshold: 0.8, weight: 1.5 } },
+    ],
+    aggregation: {
+        method: "minimum",
+    },
+    passThreshold: 0.8,
+    executionMode: "parallel",
+    requiredScorers: ["toxicity"],
+};
+/**
+ * RAG evaluation preset
+ * Evaluates Retrieval Augmented Generation quality
+ */
+export const RAG_PIPELINE = {
+    name: "rag",
+    description: "RAG evaluation pipeline for retrieval-augmented generation",
+    scorers: [
+        { id: "faithfulness", config: { weight: 1.5 } },
+        { id: "context-relevancy", config: { weight: 1.0 } },
+        { id: "context-precision", config: { weight: 1.0 } },
+        { id: "answer-relevancy", config: { weight: 1.2 } },
+        { id: "hallucination", config: { weight: 1.5 } },
+    ],
+    aggregation: {
+        method: "weighted",
+        weights: {
+            faithfulness: 1.5,
+            "context-relevancy": 1.0,
+            "context-precision": 1.0,
+            "answer-relevancy": 1.2,
+            hallucination: 1.5,
+        },
+    },
+    passThreshold: 0.7,
+    executionMode: "parallel",
+    requiredScorers: ["faithfulness", "hallucination"],
+};
+/**
+ * Quality evaluation preset
+ * Focuses on response quality: format, length, tone
+ */
+export const QUALITY_PIPELINE = {
+    name: "quality",
+    description: "Quality evaluation pipeline for response assessment",
+    scorers: [
+        { id: "tone-consistency", config: { weight: 1.0 } },
+        { id: "prompt-alignment", config: { weight: 1.2 } },
+        { id: "length", config: { weight: 0.8 } },
+        { id: "format", config: { weight: 0.8 } },
+    ],
+    aggregation: {
+        method: "average",
+    },
+    passThreshold: 0.7,
+    executionMode: "parallel",
+};
+/**
+ * Comprehensive evaluation preset
+ * Full evaluation across all dimensions
+ */
+export const COMPREHENSIVE_PIPELINE = {
+    name: "comprehensive",
+    description: "Comprehensive evaluation pipeline covering all aspects",
+    scorers: [
+        // Safety scorers
+        { id: "toxicity", config: { threshold: 0.9, weight: 2.0 } },
+        { id: "bias-detection", config: { threshold: 0.8, weight: 1.5 } },
+        // Accuracy scorers
+        { id: "hallucination", config: { weight: 1.5 } },
+        { id: "faithfulness", config: { weight: 1.2 } },
+        // Relevancy scorers
+        { id: "context-relevancy", config: { weight: 1.0 } },
+        { id: "answer-relevancy", config: { weight: 1.0 } },
+        // Quality scorers
+        { id: "tone-consistency", config: { weight: 0.8 } },
+        { id: "prompt-alignment", config: { weight: 1.0 } },
+    ],
+    aggregation: {
+        method: "weighted",
+        weights: {
+            toxicity: 2.0,
+            "bias-detection": 1.5,
+            hallucination: 1.5,
+            faithfulness: 1.2,
+            "context-relevancy": 1.0,
+            "answer-relevancy": 1.0,
+            "tone-consistency": 0.8,
+            "prompt-alignment": 1.0,
+        },
+    },
+    passThreshold: 0.75,
+    executionMode: "parallel",
+    requiredScorers: ["toxicity"],
+};
+/**
+ * Minimal/fast evaluation preset
+ * Quick checks for high-throughput scenarios
+ */
+export const MINIMAL_PIPELINE = {
+    name: "minimal",
+    description: "Minimal fast evaluation for high-throughput scenarios",
+    scorers: [
+        { id: "toxicity", config: { threshold: 0.9 } },
+        { id: "hallucination", config: { threshold: 0.8 } },
+    ],
+    aggregation: {
+        method: "minimum",
+    },
+    passThreshold: 0.8,
+    executionMode: "parallel",
+    timeout: 10000,
+};
+/**
+ * Summarization evaluation preset
+ * Evaluates summarization quality
+ */
+export const SUMMARIZATION_PIPELINE = {
+    name: "summarization",
+    description: "Summarization quality evaluation pipeline",
+    scorers: [
+        { id: "summarization", config: { weight: 1.5 } },
+        { id: "faithfulness", config: { weight: 1.2 } },
+        { id: "content-similarity", config: { weight: 1.0 } },
+        { id: "length", config: { weight: 0.8 } },
+    ],
+    aggregation: {
+        method: "weighted",
+        weights: {
+            summarization: 1.5,
+            faithfulness: 1.2,
+            "content-similarity": 1.0,
+            length: 0.8,
+        },
+    },
+    passThreshold: 0.7,
+    executionMode: "parallel",
+};
+/**
+ * Customer support evaluation preset
+ * Tailored for customer service responses
+ */
+export const CUSTOMER_SUPPORT_PIPELINE = {
+    name: "customerSupport",
+    description: "Customer support response evaluation pipeline",
+    scorers: [
+        { id: "toxicity", config: { threshold: 0.95, weight: 2.0 } },
+        { id: "tone-consistency", config: { weight: 1.5 } },
+        { id: "prompt-alignment", config: { weight: 1.2 } },
+        { id: "answer-relevancy", config: { weight: 1.0 } },
+    ],
+    aggregation: {
+        method: "weighted",
+        weights: {
+            toxicity: 2.0,
+            "tone-consistency": 1.5,
+            "prompt-alignment": 1.2,
+            "answer-relevancy": 1.0,
+        },
+    },
+    passThreshold: 0.8,
+    executionMode: "parallel",
+    requiredScorers: ["toxicity"],
+};
+/**
+ * Code generation evaluation preset
+ * Evaluates generated code quality
+ */
+export const CODE_GENERATION_PIPELINE = {
+    name: "codeGeneration",
+    description: "Code generation quality evaluation pipeline",
+    scorers: [
+        { id: "format", config: { weight: 1.0 } },
+        { id: "prompt-alignment", config: { weight: 1.5 } },
+        { id: "hallucination", config: { weight: 1.2 } },
+    ],
+    aggregation: {
+        method: "weighted",
+        weights: {
+            format: 1.0,
+            "prompt-alignment": 1.5,
+            hallucination: 1.2,
+        },
+    },
+    passThreshold: 0.75,
+    executionMode: "sequential",
+};
+/**
+ * All available presets
+ */
+export const PipelinePresets = {
+    safety: SAFETY_PIPELINE,
+    rag: RAG_PIPELINE,
+    quality: QUALITY_PIPELINE,
+    comprehensive: COMPREHENSIVE_PIPELINE,
+    minimal: MINIMAL_PIPELINE,
+    summarization: SUMMARIZATION_PIPELINE,
+    customerSupport: CUSTOMER_SUPPORT_PIPELINE,
+    codeGeneration: CODE_GENERATION_PIPELINE,
+};
+/**
+ * Get a preset pipeline configuration by name
+ */
+export function getPreset(name) {
+    return PipelinePresets[name];
+}
+/**
+ * Get all available preset names
+ */
+export function getPresetNames() {
+    return Object.keys(PipelinePresets);
+}

package/dist/evaluation/pipeline/strategies/batchStrategy.d.ts

@@ -0,0 +1,99 @@
+/**
+ * @file Batch Strategy
+ * Batch processing for evaluation pipelines
+ */
+import type { ScorerInput } from "../../../types/scorerTypes.js";
+import type { EvaluationPipeline, PipelineExecutionOptions, PipelineResult } from "../evaluationPipeline.js";
+/**
+ * Batch processing configuration
+ */
+export type BatchConfig = {
+    /** Maximum concurrent evaluations */
+    concurrency?: number;
+    /** Delay between batches (ms) */
+    batchDelay?: number;
+    /** Continue on individual failures */
+    continueOnError?: boolean;
+    /** Progress callback */
+    onProgress?: (progress: BatchProgress) => void;
+    /** Individual result callback */
+    onResult?: (result: BatchItemResult) => void;
+};
+/**
+ * Batch progress information
+ */
+export type BatchProgress = {
+    total: number;
+    completed: number;
+    failed: number;
+    remaining: number;
+    percentComplete: number;
+    estimatedTimeRemaining?: number;
+};
+/**
+ * Individual batch item result
+ */
+export type BatchItemResult = {
+    index: number;
+    input: ScorerInput;
+    result?: PipelineResult;
+    error?: string;
+    duration: number;
+};
+/**
+ * Batch evaluation result
+ */
+export type BatchResult = {
+    /** All individual results */
+    results: BatchItemResult[];
+    /** Summary statistics */
+    summary: {
+        total: number;
+        successful: number;
+        failed: number;
+        averageScore: number;
+        passRate: number;
+        totalDuration: number;
+        averageDuration: number;
+    };
+};
+/**
+ * Batch evaluation strategy
+ */
+export declare class BatchStrategy {
+    private _pipeline;
+    private _config;
+    constructor(pipeline: EvaluationPipeline, config?: BatchConfig);
+    /**
+     * Evaluate a batch of inputs
+     */
+    evaluate(inputs: ScorerInput[], options?: PipelineExecutionOptions): Promise<BatchResult>;
+    /**
+     * Evaluate a single item
+     */
+    private _evaluateItem;
+    /**
+     * Estimate remaining time based on average duration
+     */
+    private _estimateRemainingTime;
+    /**
+     * Delay helper
+     */
+    private _delay;
+    /**
+     * Update configuration
+     */
+    configure(config: Partial<BatchConfig>): void;
+}
+/**
+ * Create a batch strategy for a pipeline
+ */
+export declare function createBatchStrategy(pipeline: EvaluationPipeline, config?: BatchConfig): BatchStrategy;
+/**
+ * Evaluate a batch of inputs using a pipeline
+ */
+export declare function evaluateBatch(pipeline: EvaluationPipeline, inputs: ScorerInput[], config?: BatchConfig): Promise<BatchResult>;
+/**
+ * Stream batch evaluation results
+ */
+export declare function streamBatchEvaluation(pipeline: EvaluationPipeline, inputs: ScorerInput[], config?: Omit<BatchConfig, "onResult" | "onProgress">): AsyncGenerator<BatchItemResult, BatchResult["summary"], void>;