@juspay/neurolink 9.36.1 → 9.37.0

package/dist/lib/evaluation/reporting/metricsCollector.d.ts

@@ -0,0 +1,147 @@
+/**
+ * @file Metrics Collector
+ * Collect and aggregate evaluation metrics
+ */
+import type { ScoreResult } from "../../types/scorerTypes.js";
+import type { PipelineResult } from "../pipeline/evaluationPipeline.js";
+/**
+ * Scorer metrics
+ */
+export type ScorerMetrics = {
+    scorerId: string;
+    scorerName: string;
+    totalExecutions: number;
+    successfulExecutions: number;
+    failedExecutions: number;
+    passedCount: number;
+    failedCount: number;
+    totalScore: number;
+    minScore: number;
+    maxScore: number;
+    totalDuration: number;
+    averageDuration: number;
+    averageScore: number;
+    passRate: number;
+    lastExecutionTime: number;
+};
+/**
+ * Pipeline metrics
+ */
+export type PipelineMetrics = {
+    pipelineName: string;
+    totalExecutions: number;
+    passedCount: number;
+    failedCount: number;
+    totalScore: number;
+    minScore: number;
+    maxScore: number;
+    totalDuration: number;
+    averageDuration: number;
+    averageScore: number;
+    passRate: number;
+    lastExecutionTime: number;
+    scorerMetrics: Map<string, ScorerMetrics>;
+};
+/**
+ * Aggregated metrics
+ */
+export type AggregatedMetrics = {
+    totalEvaluations: number;
+    overallPassRate: number;
+    averageScore: number;
+    averageDuration: number;
+    scoreDistribution: {
+        excellent: number;
+        good: number;
+        fair: number;
+        poor: number;
+        failing: number;
+    };
+    pipelineMetrics: Map<string, PipelineMetrics>;
+    scorerMetrics: Map<string, ScorerMetrics>;
+    collectionStartTime: number;
+    lastUpdateTime: number;
+};
+/**
+ * Metrics collector for evaluation data
+ */
+export declare class MetricsCollector {
+    private _pipelineMetrics;
+    private _scorerMetrics;
+    private _collectionStartTime;
+    private _lastUpdateTime;
+    private _totalEvaluations;
+    private _scoreDistribution;
+    /**
+     * Record a scorer execution
+     */
+    recordScorer(scorerId: string, scorerName: string, result: ScoreResult): void;
+    /**
+     * Record a pipeline execution
+     */
+    recordPipeline(result: PipelineResult): void;
+    /**
+     * Get aggregated metrics
+     */
+    getMetrics(): AggregatedMetrics;
+    /**
+     * Get metrics for a specific scorer
+     */
+    getScorerMetrics(scorerId: string): ScorerMetrics | undefined;
+    /**
+     * Get metrics for a specific pipeline
+     */
+    getPipelineMetrics(pipelineName: string): PipelineMetrics | undefined;
+    /**
+     * Get summary statistics
+     */
+    getSummary(): {
+        totalEvaluations: number;
+        passRate: number;
+        averageScore: number;
+        topScorers: Array<{
+            id: string;
+            passRate: number;
+        }>;
+        bottomScorers: Array<{
+            id: string;
+            passRate: number;
+        }>;
+    };
+    /**
+     * Export metrics as JSON
+     */
+    exportJson(): string;
+    /**
+     * Reset all metrics
+     */
+    reset(): void;
+    /**
+     * Create empty scorer metrics
+     */
+    private _createEmptyScorerMetrics;
+    /**
+     * Create empty pipeline metrics
+     */
+    private _createEmptyPipelineMetrics;
+    /**
+     * Update scorer metrics with new result
+     */
+    private _updateScorerMetrics;
+    /**
+     * Update pipeline metrics with new result
+     */
+    private _updatePipelineMetrics;
+    /**
+     * Update score distribution
+     */
+    private _updateScoreDistribution;
+}
+/**
+ * Create a metrics collector
+ */
+export declare function createMetricsCollector(): MetricsCollector;
+/**
+ * Global metrics collector instance
+ */
+export declare const globalMetricsCollector: MetricsCollector;

package/dist/lib/evaluation/reporting/metricsCollector.js

@@ -0,0 +1,286 @@
+/**
+ * @file Metrics Collector
+ * Collect and aggregate evaluation metrics
+ */
+/**
+ * Metrics collector for evaluation data
+ */
+export class MetricsCollector {
+    _pipelineMetrics = new Map();
+    _scorerMetrics = new Map();
+    _collectionStartTime = Date.now();
+    _lastUpdateTime = Date.now();
+    _totalEvaluations = 0;
+    _scoreDistribution = {
+        excellent: 0,
+        good: 0,
+        fair: 0,
+        poor: 0,
+        failing: 0,
+    };
+    /**
+     * Record a scorer execution
+     */
+    recordScorer(scorerId, scorerName, result) {
+        let metrics = this._scorerMetrics.get(scorerId);
+        if (!metrics) {
+            metrics = this._createEmptyScorerMetrics(scorerId, scorerName);
+            this._scorerMetrics.set(scorerId, metrics);
+        }
+        this._updateScorerMetrics(metrics, result);
+        this._lastUpdateTime = Date.now();
+    }
+    /**
+     * Record a pipeline execution
+     */
+    recordPipeline(result) {
+        const pipelineName = result.pipelineConfig.name ?? "unnamed";
+        let metrics = this._pipelineMetrics.get(pipelineName);
+        if (!metrics) {
+            metrics = this._createEmptyPipelineMetrics(pipelineName);
+            this._pipelineMetrics.set(pipelineName, metrics);
+        }
+        this._updatePipelineMetrics(metrics, result);
+        this._updateScoreDistribution(result.overallScore);
+        this._totalEvaluations++;
+        this._lastUpdateTime = Date.now();
+    }
+    /**
+     * Get aggregated metrics
+     */
+    getMetrics() {
+        let totalScore = 0;
+        let totalDuration = 0;
+        let totalPassed = 0;
+        for (const metrics of this._pipelineMetrics.values()) {
+            totalScore += metrics.totalScore;
+            totalDuration += metrics.totalDuration;
+            totalPassed += metrics.passedCount;
+        }
+        return {
+            totalEvaluations: this._totalEvaluations,
+            overallPassRate: this._totalEvaluations > 0 ? totalPassed / this._totalEvaluations : 0,
+            averageScore: this._totalEvaluations > 0 ? totalScore / this._totalEvaluations : 0,
+            averageDuration: this._totalEvaluations > 0 ? totalDuration / this._totalEvaluations : 0,
+            scoreDistribution: { ...this._scoreDistribution },
+            pipelineMetrics: new Map(this._pipelineMetrics),
+            scorerMetrics: new Map(this._scorerMetrics),
+            collectionStartTime: this._collectionStartTime,
+            lastUpdateTime: this._lastUpdateTime,
+        };
+    }
+    /**
+     * Get metrics for a specific scorer
+     */
+    getScorerMetrics(scorerId) {
+        return this._scorerMetrics.get(scorerId);
+    }
+    /**
+     * Get metrics for a specific pipeline
+     */
+    getPipelineMetrics(pipelineName) {
+        return this._pipelineMetrics.get(pipelineName);
+    }
+    /**
+     * Get summary statistics
+     */
+    getSummary() {
+        const scorersList = Array.from(this._scorerMetrics.values())
+            .filter((m) => m.totalExecutions > 0)
+            .sort((a, b) => b.passRate - a.passRate);
+        const metrics = this.getMetrics();
+        return {
+            totalEvaluations: metrics.totalEvaluations,
+            passRate: metrics.overallPassRate,
+            averageScore: metrics.averageScore,
+            topScorers: scorersList.slice(0, 5).map((m) => ({
+                id: m.scorerId,
+                passRate: m.passRate,
+            })),
+            bottomScorers: scorersList
+                .slice(-5)
+                .reverse()
+                .map((m) => ({
+                id: m.scorerId,
+                passRate: m.passRate,
+            })),
+        };
+    }
+    /**
+     * Export metrics as JSON
+     */
+    exportJson() {
+        const metrics = this.getMetrics();
+        return JSON.stringify({
+            totalEvaluations: metrics.totalEvaluations,
+            overallPassRate: metrics.overallPassRate,
+            averageScore: metrics.averageScore,
+            averageDuration: metrics.averageDuration,
+            scoreDistribution: metrics.scoreDistribution,
+            collectionStartTime: metrics.collectionStartTime,
+            lastUpdateTime: metrics.lastUpdateTime,
+            pipelines: Array.from(metrics.pipelineMetrics.entries()).map(([name, pm]) => ({
+                name,
+                totalExecutions: pm.totalExecutions,
+                passRate: pm.passRate,
+                averageScore: pm.averageScore,
+                averageDuration: pm.averageDuration,
+            })),
+            scorers: Array.from(metrics.scorerMetrics.entries()).map(([id, sm]) => ({
+                id,
+                name: sm.scorerName,
+                totalExecutions: sm.totalExecutions,
+                passRate: sm.passRate,
+                averageScore: sm.averageScore,
+                averageDuration: sm.averageDuration,
+                minScore: Number.isFinite(sm.minScore) ? sm.minScore : null,
+                maxScore: Number.isFinite(sm.maxScore) ? sm.maxScore : null,
+            })),
+        }, null, 2);
+    }
+    /**
+     * Reset all metrics
+     */
+    reset() {
+        this._pipelineMetrics.clear();
+        this._scorerMetrics.clear();
+        this._collectionStartTime = Date.now();
+        this._lastUpdateTime = Date.now();
+        this._totalEvaluations = 0;
+        this._scoreDistribution = {
+            excellent: 0,
+            good: 0,
+            fair: 0,
+            poor: 0,
+            failing: 0,
+        };
+    }
+    /**
+     * Create empty scorer metrics
+     */
+    _createEmptyScorerMetrics(scorerId, scorerName) {
+        return {
+            scorerId,
+            scorerName,
+            totalExecutions: 0,
+            successfulExecutions: 0,
+            failedExecutions: 0,
+            passedCount: 0,
+            failedCount: 0,
+            totalScore: 0,
+            minScore: Infinity,
+            maxScore: -Infinity,
+            totalDuration: 0,
+            averageDuration: 0,
+            averageScore: 0,
+            passRate: 0,
+            lastExecutionTime: 0,
+        };
+    }
+    /**
+     * Create empty pipeline metrics
+     */
+    _createEmptyPipelineMetrics(pipelineName) {
+        return {
+            pipelineName,
+            totalExecutions: 0,
+            passedCount: 0,
+            failedCount: 0,
+            totalScore: 0,
+            minScore: Infinity,
+            maxScore: -Infinity,
+            totalDuration: 0,
+            averageDuration: 0,
+            averageScore: 0,
+            passRate: 0,
+            lastExecutionTime: 0,
+            scorerMetrics: new Map(),
+        };
+    }
+    /**
+     * Update scorer metrics with new result
+     */
+    _updateScorerMetrics(metrics, result) {
+        metrics.totalExecutions++;
+        if (result.error) {
+            metrics.failedExecutions++;
+        }
+        else {
+            metrics.successfulExecutions++;
+        }
+        if (result.passed) {
+            metrics.passedCount++;
+        }
+        else {
+            metrics.failedCount++;
+        }
+        metrics.totalScore += result.score;
+        metrics.minScore = Math.min(metrics.minScore, result.score);
+        metrics.maxScore = Math.max(metrics.maxScore, result.score);
+        metrics.totalDuration += result.computeTime;
+        metrics.averageScore = metrics.totalScore / metrics.totalExecutions;
+        metrics.averageDuration = metrics.totalDuration / metrics.totalExecutions;
+        metrics.passRate = metrics.passedCount / metrics.totalExecutions;
+        metrics.lastExecutionTime = Date.now();
+    }
+    /**
+     * Update pipeline metrics with new result
+     */
+    _updatePipelineMetrics(metrics, result) {
+        metrics.totalExecutions++;
+        if (result.passed) {
+            metrics.passedCount++;
+        }
+        else {
+            metrics.failedCount++;
+        }
+        metrics.totalScore += result.overallScore;
+        metrics.minScore = Math.min(metrics.minScore, result.overallScore);
+        metrics.maxScore = Math.max(metrics.maxScore, result.overallScore);
+        metrics.totalDuration += result.totalComputeTime;
+        metrics.averageScore = metrics.totalScore / metrics.totalExecutions;
+        metrics.averageDuration = metrics.totalDuration / metrics.totalExecutions;
+        metrics.passRate = metrics.passedCount / metrics.totalExecutions;
+        metrics.lastExecutionTime = Date.now();
+        // Update individual scorer metrics within pipeline
+        for (const scoreResult of result.scores) {
+            let scorerMetrics = metrics.scorerMetrics.get(scoreResult.scorerId);
+            if (!scorerMetrics) {
+                scorerMetrics = this._createEmptyScorerMetrics(scoreResult.scorerId, scoreResult.scorerName);
+                metrics.scorerMetrics.set(scoreResult.scorerId, scorerMetrics);
+            }
+            this._updateScorerMetrics(scorerMetrics, scoreResult);
+        }
+    }
+    /**
+     * Update score distribution
+     */
+    _updateScoreDistribution(score) {
+        if (score >= 9) {
+            this._scoreDistribution.excellent++;
+        }
+        else if (score >= 7) {
+            this._scoreDistribution.good++;
+        }
+        else if (score >= 5) {
+            this._scoreDistribution.fair++;
+        }
+        else if (score >= 3) {
+            this._scoreDistribution.poor++;
+        }
+        else {
+            this._scoreDistribution.failing++;
+        }
+    }
+}
+/**
+ * Create a metrics collector
+ */
+export function createMetricsCollector() {
+    return new MetricsCollector();
+}
+/**
+ * Global metrics collector instance
+ */
+export const globalMetricsCollector = new MetricsCollector();
+//# sourceMappingURL=metricsCollector.js.map

package/dist/lib/evaluation/reporting/reportGenerator.d.ts

@@ -0,0 +1,90 @@
+/**
+ * @file Report Generator
+ * Generate evaluation reports in various formats
+ */
+import type { JsonObject } from "../../types/common.js";
+import type { AggregatedScores, ReportConfig, ReportFormat } from "../../types/scorerTypes.js";
+import type { PipelineResult } from "../pipeline/evaluationPipeline.js";
+/**
+ * Report data structure
+ */
+export type ReportData = {
+    /** Report title */
+    title: string;
+    /** Timestamp */
+    timestamp: number;
+    /** Evaluation result */
+    result: PipelineResult | AggregatedScores;
+    /** Optional custom sections */
+    customSections?: Array<{
+        title: string;
+        content: string | JsonObject;
+    }>;
+};
+/**
+ * Generated report
+ */
+export type GeneratedReport = {
+    /** Report format */
+    format: ReportFormat;
+    /** Report content */
+    content: string;
+    /** Metadata */
+    metadata: {
+        generatedAt: number;
+        format: ReportFormat;
+        config: ReportConfig;
+    };
+};
+/**
+ * Report generator class
+ */
+export declare class ReportGenerator {
+    private _config;
+    constructor(config?: Partial<ReportConfig>);
+    /**
+     * Generate a report
+     */
+    generate(data: ReportData): GeneratedReport;
+    /**
+     * Generate text report
+     */
+    private _generateTextReport;
+    /**
+     * Generate JSON report
+     */
+    private _generateJsonReport;
+    /**
+     * Generate Markdown report
+     */
+    private _generateMarkdownReport;
+    /**
+     * Generate HTML report
+     */
+    private _generateHtmlReport;
+    /**
+     * Escape HTML special characters
+     */
+    private _escapeHtml;
+    /**
+     * Update configuration
+     */
+    configure(config: Partial<ReportConfig>): void;
+}
+/**
+ * Create a report generator
+ */
+export declare function createReportGenerator(config?: Partial<ReportConfig>): ReportGenerator;
+/**
+ * Quick report generation functions
+ */
+export declare const Reports: {
+    /** Generate text report */
+    text: (data: ReportData) => GeneratedReport;
+    /** Generate JSON report */
+    json: (data: ReportData) => GeneratedReport;
+    /** Generate Markdown report */
+    markdown: (data: ReportData) => GeneratedReport;
+    /** Generate HTML report */
+    html: (data: ReportData) => GeneratedReport;
+};