@juspay/neurolink 9.36.0 → 9.37.0

package/dist/evaluation/pipeline/strategies/batchStrategy.js

@@ -0,0 +1,238 @@
+/**
+ * @file Batch Strategy
+ * Batch processing for evaluation pipelines
+ */
+/**
+ * Default batch configuration
+ */
+const DEFAULT_BATCH_CONFIG = {
+    concurrency: 5,
+    batchDelay: 0,
+    continueOnError: true,
+    onProgress: () => { },
+    onResult: () => { },
+};
+/**
+ * Batch evaluation strategy
+ */
+export class BatchStrategy {
+    _pipeline;
+    _config;
+    constructor(pipeline, config) {
+        this._pipeline = pipeline;
+        this._config = { ...DEFAULT_BATCH_CONFIG, ...config };
+    }
+    /**
+     * Evaluate a batch of inputs
+     */
+    async evaluate(inputs, options) {
+        const startTime = Date.now();
+        const results = [];
+        const durations = [];
+        // Process in batches based on concurrency
+        for (let i = 0; i < inputs.length; i += this._config.concurrency) {
+            const batch = inputs.slice(i, i + this._config.concurrency);
+            // Execute batch in parallel
+            const batchPromises = batch.map((input, batchIndex) => {
+                const globalIndex = i + batchIndex;
+                return this._evaluateItem(input, globalIndex, options);
+            });
+            const batchResults = await Promise.all(batchPromises);
+            // Record results
+            for (const result of batchResults) {
+                results.push(result);
+                durations.push(result.duration);
+                // Notify of individual result
+                this._config.onResult(result);
+                // Check for fatal error
+                if (result.error && !this._config.continueOnError) {
+                    throw new Error(`Batch evaluation failed at index ${result.index}: ${result.error}`);
+                }
+            }
+            // Report progress
+            this._config.onProgress({
+                total: inputs.length,
+                completed: results.length,
+                failed: results.filter((r) => r.error).length,
+                remaining: inputs.length - results.length,
+                percentComplete: (results.length / inputs.length) * 100,
+                estimatedTimeRemaining: this._estimateRemainingTime(durations, inputs.length - results.length),
+            });
+            // Apply batch delay if configured
+            if (this._config.batchDelay > 0 &&
+                i + this._config.concurrency < inputs.length) {
+                await this._delay(this._config.batchDelay);
+            }
+        }
+        // Calculate summary
+        const totalDuration = Date.now() - startTime;
+        const successfulResults = results.filter((r) => !r.error && r.result);
+        const scores = successfulResults.map((r) => r.result.overallScore);
+        const passed = successfulResults.filter((r) => r.result.passed);
+        return {
+            results,
+            summary: {
+                total: inputs.length,
+                successful: successfulResults.length,
+                failed: results.length - successfulResults.length,
+                averageScore: scores.length > 0
+                    ? scores.reduce((a, b) => a + b, 0) / scores.length
+                    : 0,
+                passRate: successfulResults.length > 0
+                    ? passed.length / successfulResults.length
+                    : 0,
+                totalDuration,
+                averageDuration: durations.length > 0
+                    ? durations.reduce((a, b) => a + b, 0) / durations.length
+                    : 0,
+            },
+        };
+    }
+    /**
+     * Evaluate a single item
+     */
+    async _evaluateItem(input, index, options) {
+        const startTime = Date.now();
+        try {
+            const result = await this._pipeline.execute(input, {
+                ...options,
+                correlationId: options?.correlationId
+                    ? `${options.correlationId}-${index}`
+                    : `batch-${index}`,
+            });
+            return {
+                index,
+                input,
+                result,
+                duration: Date.now() - startTime,
+            };
+        }
+        catch (error) {
+            return {
+                index,
+                input,
+                error: error instanceof Error ? error.message : String(error),
+                duration: Date.now() - startTime,
+            };
+        }
+    }
+    /**
+     * Estimate remaining time based on average duration
+     */
+    _estimateRemainingTime(durations, remaining) {
+        if (durations.length === 0 || remaining === 0) {
+            return 0;
+        }
+        const avgDuration = durations.reduce((a, b) => a + b, 0) / durations.length;
+        const batches = Math.ceil(remaining / this._config.concurrency);
+        return avgDuration * batches + this._config.batchDelay * (batches - 1);
+    }
+    /**
+     * Delay helper
+     */
+    _delay(ms) {
+        return new Promise((resolve) => setTimeout(resolve, ms));
+    }
+    /**
+     * Update configuration
+     */
+    configure(config) {
+        this._config = { ...this._config, ...config };
+    }
+}
+/**
+ * Create a batch strategy for a pipeline
+ */
+export function createBatchStrategy(pipeline, config) {
+    return new BatchStrategy(pipeline, config);
+}
+/**
+ * Evaluate a batch of inputs using a pipeline
+ */
+export async function evaluateBatch(pipeline, inputs, config) {
+    const strategy = new BatchStrategy(pipeline, config);
+    return strategy.evaluate(inputs);
+}
+/**
+ * Stream batch evaluation results
+ */
+export async function* streamBatchEvaluation(pipeline, inputs, config) {
+    const results = [];
+    const durations = [];
+    const startTime = Date.now();
+    const batchConfig = { ...DEFAULT_BATCH_CONFIG, ...config };
+    for (let i = 0; i < inputs.length; i += batchConfig.concurrency) {
+        const batch = inputs.slice(i, i + batchConfig.concurrency);
+        const batchPromises = batch.map(async (input, batchIndex) => {
+            const globalIndex = i + batchIndex;
+            const itemStart = Date.now();
+            try {
+                const result = await pipeline.execute(input, {
+                    correlationId: `stream-batch-${globalIndex}`,
+                });
+                return {
+                    index: globalIndex,
+                    input,
+                    result,
+                    duration: Date.now() - itemStart,
+                };
+            }
+            catch (error) {
+                return {
+                    index: globalIndex,
+                    input,
+                    error: error instanceof Error ? error.message : String(error),
+                    duration: Date.now() - itemStart,
+                };
+            }
+        });
+        const batchResults = await Promise.all(batchPromises);
+        for (const result of batchResults) {
+            results.push(result);
+            durations.push(result.duration);
+            // If continueOnError is false and this result has an error, abort and return summary
+            if (result.error && batchConfig.continueOnError === false) {
+                const successfulResults = results.filter((r) => !r.error && r.result);
+                const earlyScores = successfulResults.map((r) => r.result.overallScore);
+                const earlyPassed = successfulResults.filter((r) => r.result.passed);
+                return {
+                    total: inputs.length,
+                    successful: successfulResults.length,
+                    failed: results.length - successfulResults.length,
+                    averageScore: earlyScores.length > 0
+                        ? earlyScores.reduce((a, b) => a + b, 0) / earlyScores.length
+                        : 0,
+                    passRate: successfulResults.length > 0
+                        ? earlyPassed.length / successfulResults.length
+                        : 0,
+                    totalDuration: Date.now() - startTime,
+                    averageDuration: durations.length > 0
+                        ? durations.reduce((a, b) => a + b, 0) / durations.length
+                        : 0,
+                };
+            }
+            yield result;
+        }
+        if (batchConfig.batchDelay > 0 &&
+            i + batchConfig.concurrency < inputs.length) {
+            await new Promise((resolve) => setTimeout(resolve, batchConfig.batchDelay));
+        }
+    }
+    // Return summary
+    const successfulResults = results.filter((r) => !r.error && r.result);
+    const scores = successfulResults.map((r) => r.result.overallScore);
+    const passed = successfulResults.filter((r) => r.result.passed);
+    return {
+        total: inputs.length,
+        successful: successfulResults.length,
+        failed: results.length - successfulResults.length,
+        averageScore: scores.length > 0 ? scores.reduce((a, b) => a + b, 0) / scores.length : 0,
+        passRate: successfulResults.length > 0
+            ? passed.length / successfulResults.length
+            : 0,
+        totalDuration: Date.now() - startTime,
+        averageDuration: durations.length > 0
+            ? durations.reduce((a, b) => a + b, 0) / durations.length
+            : 0,
+    };
+}

package/dist/evaluation/pipeline/strategies/index.d.ts

@@ -0,0 +1,6 @@
+/**
+ * @file Pipeline Strategies Index
+ * Export all pipeline strategies
+ */
+export { type BatchConfig, type BatchItemResult, type BatchProgress, type BatchResult, BatchStrategy, createBatchStrategy, evaluateBatch, streamBatchEvaluation, } from "./batchStrategy.js";
+export { createSamplingStrategy, DEFAULT_SAMPLING_CONFIG, SamplingStrategies, SamplingStrategy, } from "./samplingStrategy.js";

package/dist/evaluation/pipeline/strategies/index.js

@@ -0,0 +1,6 @@
+/**
+ * @file Pipeline Strategies Index
+ * Export all pipeline strategies
+ */
+export { BatchStrategy, createBatchStrategy, evaluateBatch, streamBatchEvaluation, } from "./batchStrategy.js";
+export { createSamplingStrategy, DEFAULT_SAMPLING_CONFIG, SamplingStrategies, SamplingStrategy, } from "./samplingStrategy.js";

package/dist/evaluation/pipeline/strategies/samplingStrategy.d.ts

@@ -0,0 +1,76 @@
+/**
+ * @file Sampling Strategy
+ * Configurable sampling for cost-efficient evaluation
+ */
+import type { SamplingConfig, SamplingContext, SamplingDecision } from "../../../types/scorerTypes.js";
+/**
+ * Default sampling configuration
+ */
+export declare const DEFAULT_SAMPLING_CONFIG: SamplingConfig;
+/**
+ * Sampling strategy for evaluation
+ */
+export declare class SamplingStrategy {
+    private _config;
+    private _recentScores;
+    private _currentRate;
+    private _maxRecentScores;
+    constructor(config?: Partial<SamplingConfig>);
+    /**
+     * Get current sampling configuration
+     */
+    get config(): SamplingConfig;
+    /**
+     * Get current sampling rate
+     */
+    get currentRate(): number;
+    /**
+     * Decide whether to sample a request
+     */
+    shouldSample(context?: SamplingContext): SamplingDecision;
+    /**
+     * Record a score for adaptive sampling
+     */
+    recordScore(score: number): void;
+    /**
+     * Update sampling rate based on recent quality
+     */
+    private _updateAdaptiveRate;
+    /**
+     * Reset sampling state
+     */
+    reset(): void;
+    /**
+     * Update sampling configuration
+     */
+    configure(config: Partial<SamplingConfig>): void;
+    /**
+     * Get sampling statistics
+     */
+    getStats(): {
+        currentRate: number;
+        recentScoresCount: number;
+        averageScore: number | null;
+    };
+}
+/**
+ * Create a sampling strategy
+ */
+export declare function createSamplingStrategy(config?: Partial<SamplingConfig>): SamplingStrategy;
+/**
+ * Pre-configured sampling strategies
+ */
+export declare const SamplingStrategies: {
+    /** Evaluate everything (default) */
+    readonly all: () => SamplingStrategy;
+    /** Evaluate 50% of requests */
+    readonly half: () => SamplingStrategy;
+    /** Evaluate 10% of requests */
+    readonly light: () => SamplingStrategy;
+    /** Adaptive sampling based on quality */
+    readonly adaptive: () => SamplingStrategy;
+    /** Only evaluate errors and specific conditions */
+    readonly errorsOnly: () => SamplingStrategy;
+    /** VIP users always evaluated */
+    readonly vipUsers: (users: string[]) => SamplingStrategy;
+};

package/dist/evaluation/pipeline/strategies/samplingStrategy.js

@@ -0,0 +1,238 @@
+/**
+ * @file Sampling Strategy
+ * Configurable sampling for cost-efficient evaluation
+ */
+/**
+ * Default sampling configuration
+ */
+export const DEFAULT_SAMPLING_CONFIG = {
+    rate: 1.0, // 100% sampling by default
+    alwaysEvaluate: {
+        errors: true,
+    },
+    adaptive: {
+        enabled: false,
+        qualityThreshold: 0.7,
+        minRate: 0.1,
+        maxRate: 1.0,
+    },
+};
+/**
+ * Sampling strategy for evaluation
+ */
+export class SamplingStrategy {
+    _config;
+    _recentScores = [];
+    _currentRate;
+    _maxRecentScores = 100;
+    constructor(config = {}) {
+        const clamp01 = (v) => Math.max(0, Math.min(1, v));
+        const rawRate = config.rate ?? DEFAULT_SAMPLING_CONFIG.rate;
+        const rawMinRate = config.adaptive?.minRate ?? DEFAULT_SAMPLING_CONFIG.adaptive.minRate;
+        const rawMaxRate = config.adaptive?.maxRate ?? DEFAULT_SAMPLING_CONFIG.adaptive.maxRate;
+        const minRate = clamp01(Math.min(rawMinRate, rawMaxRate));
+        const maxRate = clamp01(Math.max(rawMinRate, rawMaxRate));
+        this._config = {
+            ...DEFAULT_SAMPLING_CONFIG,
+            ...config,
+            rate: clamp01(rawRate),
+            alwaysEvaluate: {
+                ...DEFAULT_SAMPLING_CONFIG.alwaysEvaluate,
+                ...(config.alwaysEvaluate ?? {}),
+            },
+            adaptive: {
+                enabled: config.adaptive?.enabled ?? DEFAULT_SAMPLING_CONFIG.adaptive.enabled,
+                qualityThreshold: clamp01(config.adaptive?.qualityThreshold ??
+                    DEFAULT_SAMPLING_CONFIG.adaptive.qualityThreshold),
+                minRate,
+                maxRate,
+            },
+        };
+        this._currentRate = this._config.rate;
+    }
+    /**
+     * Get current sampling configuration
+     */
+    get config() {
+        return this._config;
+    }
+    /**
+     * Get current sampling rate
+     */
+    get currentRate() {
+        return this._currentRate;
+    }
+    /**
+     * Decide whether to sample a request
+     */
+    shouldSample(context) {
+        // Always evaluate errors
+        if (context?.hasError && this._config.alwaysEvaluate?.errors) {
+            return {
+                shouldSample: true,
+                reason: "Always evaluate errors",
+                currentRate: this._currentRate,
+            };
+        }
+        // Always evaluate specific users
+        if (context?.userId &&
+            this._config.alwaysEvaluate?.users?.includes(context.userId)) {
+            return {
+                shouldSample: true,
+                reason: `Always evaluate user: ${context.userId}`,
+                currentRate: this._currentRate,
+            };
+        }
+        // Always evaluate specific tags
+        if (context?.tags && this._config.alwaysEvaluate?.tags) {
+            const matchingTags = context.tags.filter((tag) => this._config.alwaysEvaluate?.tags?.includes(tag));
+            if (matchingTags.length > 0) {
+                return {
+                    shouldSample: true,
+                    reason: `Always evaluate tags: ${matchingTags.join(", ")}`,
+                    currentRate: this._currentRate,
+                };
+            }
+        }
+        // Random sampling based on current rate
+        const random = Math.random();
+        const shouldSample = random < this._currentRate;
+        return {
+            shouldSample,
+            reason: shouldSample
+                ? `Sampled at ${(this._currentRate * 100).toFixed(1)}% rate`
+                : `Not sampled (rate: ${(this._currentRate * 100).toFixed(1)}%)`,
+            currentRate: this._currentRate,
+        };
+    }
+    /**
+     * Record a score for adaptive sampling
+     */
+    recordScore(score) {
+        if (!Number.isFinite(score)) {
+            return;
+        }
+        const boundedScore = Math.max(0, Math.min(10, score));
+        this._recentScores.push(boundedScore);
+        // Keep only recent scores
+        if (this._recentScores.length > this._maxRecentScores) {
+            this._recentScores.shift();
+        }
+        // Update adaptive rate if enabled
+        if (this._config.adaptive?.enabled) {
+            this._updateAdaptiveRate();
+        }
+    }
+    /**
+     * Update sampling rate based on recent quality
+     */
+    _updateAdaptiveRate() {
+        if (!this._config.adaptive?.enabled || this._recentScores.length < 10) {
+            return;
+        }
+        const adaptive = this._config.adaptive;
+        const avgScore = this._recentScores.reduce((sum, s) => sum + s, 0) /
+            this._recentScores.length;
+        // Normalize to 0-1 scale (assuming 0-10 scores)
+        const normalizedAvg = avgScore / 10;
+        if (normalizedAvg < adaptive.qualityThreshold) {
+            // Low quality - increase sampling rate
+            this._currentRate = Math.min(adaptive.maxRate, this._currentRate * 1.2);
+        }
+        else {
+            // High quality - can decrease sampling rate
+            this._currentRate = Math.max(adaptive.minRate, this._currentRate * 0.9);
+        }
+    }
+    /**
+     * Reset sampling state
+     */
+    reset() {
+        this._recentScores = [];
+        this._currentRate = this._config.rate;
+    }
+    /**
+     * Update sampling configuration
+     */
+    configure(config) {
+        const clamp01 = (v) => Math.max(0, Math.min(1, v));
+        const rawRate = config.rate ?? this._config.rate;
+        const rawMinRate = config.adaptive?.minRate ?? this._config.adaptive.minRate;
+        const rawMaxRate = config.adaptive?.maxRate ?? this._config.adaptive.maxRate;
+        const minRate = clamp01(Math.min(rawMinRate, rawMaxRate));
+        const maxRate = clamp01(Math.max(rawMinRate, rawMaxRate));
+        this._config = {
+            ...this._config,
+            ...config,
+            rate: clamp01(rawRate),
+            alwaysEvaluate: {
+                ...this._config.alwaysEvaluate,
+                ...(config.alwaysEvaluate ?? {}),
+            },
+            adaptive: {
+                enabled: config.adaptive?.enabled ?? this._config.adaptive.enabled,
+                qualityThreshold: clamp01(config.adaptive?.qualityThreshold ??
+                    this._config.adaptive.qualityThreshold),
+                minRate,
+                maxRate,
+            },
+        };
+        this._currentRate = this._config.rate;
+    }
+    /**
+     * Get sampling statistics
+     */
+    getStats() {
+        const avgScore = this._recentScores.length > 0
+            ? this._recentScores.reduce((sum, s) => sum + s, 0) /
+                this._recentScores.length
+            : null;
+        return {
+            currentRate: this._currentRate,
+            recentScoresCount: this._recentScores.length,
+            averageScore: avgScore,
+        };
+    }
+}
+/**
+ * Create a sampling strategy
+ */
+export function createSamplingStrategy(config) {
+    return new SamplingStrategy(config);
+}
+/**
+ * Pre-configured sampling strategies
+ */
+export const SamplingStrategies = {
+    /** Evaluate everything (default) */
+    all: () => new SamplingStrategy({ rate: 1.0 }),
+    /** Evaluate 50% of requests */
+    half: () => new SamplingStrategy({ rate: 0.5 }),
+    /** Evaluate 10% of requests */
+    light: () => new SamplingStrategy({ rate: 0.1 }),
+    /** Adaptive sampling based on quality */
+    adaptive: () => new SamplingStrategy({
+        rate: 0.5,
+        adaptive: {
+            enabled: true,
+            qualityThreshold: 0.7,
+            minRate: 0.1,
+            maxRate: 1.0,
+        },
+    }),
+    /** Only evaluate errors and specific conditions */
+    errorsOnly: () => new SamplingStrategy({
+        rate: 0.0,
+        alwaysEvaluate: {
+            errors: true,
+        },
+    }),
+    /** VIP users always evaluated */
+    vipUsers: (users) => new SamplingStrategy({
+        rate: 0.1,
+        alwaysEvaluate: {
+            errors: true,
+            users,
+        },
+    }),
+};

package/dist/evaluation/reporting/index.d.ts

@@ -0,0 +1,6 @@
+/**
+ * @file Reporting Index
+ * Export all reporting components
+ */
+export { type AggregatedMetrics, createMetricsCollector, globalMetricsCollector, MetricsCollector, type PipelineMetrics, type ScorerMetrics, } from "./metricsCollector.js";
+export { createReportGenerator, type GeneratedReport, type ReportData, ReportGenerator, Reports, } from "./reportGenerator.js";

package/dist/evaluation/reporting/index.js

@@ -0,0 +1,6 @@
+/**
+ * @file Reporting Index
+ * Export all reporting components
+ */
+export { createMetricsCollector, globalMetricsCollector, MetricsCollector, } from "./metricsCollector.js";
+export { createReportGenerator, ReportGenerator, Reports, } from "./reportGenerator.js";