@juspay/neurolink 9.36.0 → 9.37.0

package/dist/evaluation/reporting/reportGenerator.js

@@ -0,0 +1,374 @@
+/**
+ * @file Report Generator
+ * Generate evaluation reports in various formats
+ */
+/**
+ * Default report configuration
+ */
+const DEFAULT_REPORT_CONFIG = {
+    format: "text",
+    includeReasoning: true,
+    includeMetadata: true,
+    includeTiming: true,
+};
+/**
+ * Report generator class
+ */
+export class ReportGenerator {
+    _config;
+    constructor(config) {
+        this._config = { ...DEFAULT_REPORT_CONFIG, ...config };
+    }
+    /**
+     * Generate a report
+     */
+    generate(data) {
+        let content;
+        switch (this._config.format) {
+            case "json":
+                content = this._generateJsonReport(data);
+                break;
+            case "markdown":
+                content = this._generateMarkdownReport(data);
+                break;
+            case "html":
+                content = this._generateHtmlReport(data);
+                break;
+            case "text":
+            default:
+                content = this._generateTextReport(data);
+                break;
+        }
+        return {
+            format: this._config.format,
+            content,
+            metadata: {
+                generatedAt: Date.now(),
+                format: this._config.format,
+                config: this._config,
+            },
+        };
+    }
+    /**
+     * Generate text report
+     */
+    _generateTextReport(data) {
+        const lines = [];
+        const result = data.result;
+        lines.push("=".repeat(60));
+        lines.push(data.title);
+        lines.push("=".repeat(60));
+        lines.push(`Generated: ${new Date(data.timestamp).toISOString()}`);
+        lines.push("");
+        // Overall result
+        lines.push("OVERALL RESULT");
+        lines.push("-".repeat(40));
+        lines.push(`Score: ${result.overallScore.toFixed(2)}/10`);
+        lines.push(`Status: ${result.passed ? "PASSED" : "FAILED"}`);
+        lines.push(`Aggregation: ${result.aggregationMethod}`);
+        if (this._config.includeTiming && result.totalComputeTime) {
+            lines.push(`Duration: ${result.totalComputeTime}ms`);
+        }
+        lines.push("");
+        // Individual scores
+        lines.push("INDIVIDUAL SCORES");
+        lines.push("-".repeat(40));
+        for (const score of result.scores) {
+            const status = score.passed ? "[PASS]" : "[FAIL]";
+            lines.push(`${status} ${score.scorerName}: ${score.score.toFixed(2)}/10`);
+            if (this._config.includeReasoning) {
+                lines.push(`       Reasoning: ${score.reasoning}`);
+            }
+            if (this._config.includeTiming) {
+                lines.push(`       Duration: ${score.computeTime}ms`);
+            }
+            lines.push("");
+        }
+        // Errors if any
+        const pipelineResult = result;
+        if (pipelineResult.errors && pipelineResult.errors.length > 0) {
+            lines.push("ERRORS");
+            lines.push("-".repeat(40));
+            for (const error of pipelineResult.errors) {
+                lines.push(`[${error.scorerId}] ${error.error}`);
+            }
+            lines.push("");
+        }
+        // Custom sections
+        if (data.customSections) {
+            for (const section of data.customSections) {
+                lines.push(section.title.toUpperCase());
+                lines.push("-".repeat(40));
+                lines.push(typeof section.content === "string"
+                    ? section.content
+                    : JSON.stringify(section.content, null, 2));
+                lines.push("");
+            }
+        }
+        lines.push("=".repeat(60));
+        return lines.join("\n");
+    }
+    /**
+     * Generate JSON report
+     */
+    _generateJsonReport(data) {
+        const report = {
+            title: data.title,
+            timestamp: data.timestamp,
+            generatedAt: new Date(data.timestamp).toISOString(),
+            overall: {
+                score: data.result.overallScore,
+                passed: data.result.passed,
+                aggregationMethod: data.result.aggregationMethod,
+            },
+            scores: data.result.scores.map((score) => {
+                const scoreObj = {
+                    scorerId: score.scorerId,
+                    scorerName: score.scorerName,
+                    score: score.score,
+                    normalizedScore: score.normalizedScore,
+                    passed: score.passed,
+                    threshold: score.threshold,
+                };
+                if (this._config.includeReasoning) {
+                    scoreObj.reasoning = score.reasoning;
+                }
+                if (this._config.includeTiming) {
+                    scoreObj.computeTime = score.computeTime;
+                }
+                if (this._config.includeMetadata && score.metadata) {
+                    scoreObj.metadata = score.metadata;
+                }
+                return scoreObj;
+            }),
+        };
+        if (this._config.includeTiming) {
+            report.totalComputeTime = data.result.totalComputeTime;
+        }
+        const pipelineResult = data.result;
+        if (pipelineResult.errors && pipelineResult.errors.length > 0) {
+            report.errors = pipelineResult.errors;
+        }
+        if (data.customSections) {
+            report.customSections = data.customSections;
+        }
+        return JSON.stringify(report, null, 2);
+    }
+    /**
+     * Generate Markdown report
+     */
+    _generateMarkdownReport(data) {
+        const lines = [];
+        const result = data.result;
+        lines.push(`# ${data.title}`);
+        lines.push("");
+        lines.push(`*Generated: ${new Date(data.timestamp).toISOString()}*`);
+        lines.push("");
+        // Overall result
+        lines.push("## Overall Result");
+        lines.push("");
+        lines.push(`| Metric | Value |`);
+        lines.push(`| ------ | ----- |`);
+        lines.push(`| Score | ${result.overallScore.toFixed(2)}/10 |`);
+        lines.push(`| Status | ${result.passed ? "**PASSED**" : "**FAILED**"} |`);
+        lines.push(`| Aggregation | ${result.aggregationMethod} |`);
+        if (this._config.includeTiming && result.totalComputeTime) {
+            lines.push(`| Duration | ${result.totalComputeTime}ms |`);
+        }
+        lines.push("");
+        // Individual scores
+        lines.push("## Individual Scores");
+        lines.push("");
+        lines.push(`| Scorer | Score | Status | ${this._config.includeTiming ? "Duration |" : ""}`);
+        lines.push(`| ------ | ----- | ------ | ${this._config.includeTiming ? "-------- |" : ""}`);
+        for (const score of result.scores) {
+            const status = score.passed ? "Pass" : "Fail";
+            let row = `| ${score.scorerName} | ${score.score.toFixed(2)} | ${status} |`;
+            if (this._config.includeTiming) {
+                row += ` ${score.computeTime}ms |`;
+            }
+            lines.push(row);
+        }
+        lines.push("");
+        // Reasoning
+        if (this._config.includeReasoning) {
+            lines.push("### Reasoning");
+            lines.push("");
+            for (const score of result.scores) {
+                lines.push(`**${score.scorerName}**: ${score.reasoning}`);
+                lines.push("");
+            }
+        }
+        // Errors
+        const pipelineResult = result;
+        if (pipelineResult.errors && pipelineResult.errors.length > 0) {
+            lines.push("## Errors");
+            lines.push("");
+            for (const error of pipelineResult.errors) {
+                lines.push(`- **${error.scorerId}**: ${error.error}`);
+            }
+            lines.push("");
+        }
+        // Custom sections
+        if (data.customSections) {
+            for (const section of data.customSections) {
+                lines.push(`## ${section.title}`);
+                lines.push("");
+                lines.push(typeof section.content === "string"
+                    ? section.content
+                    : "```json\n" + JSON.stringify(section.content, null, 2) + "\n```");
+                lines.push("");
+            }
+        }
+        return lines.join("\n");
+    }
+    /**
+     * Generate HTML report
+     */
+    _generateHtmlReport(data) {
+        const result = data.result;
+        const statusClass = result.passed ? "passed" : "failed";
+        let html = `
+<!DOCTYPE html>
+<html>
+<head>
+  <title>${this._escapeHtml(data.title)}</title>
+  <style>
+    body { font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; max-width: 800px; margin: 0 auto; padding: 20px; }
+    h1 { color: #333; border-bottom: 2px solid #eee; padding-bottom: 10px; }
+    h2 { color: #555; margin-top: 30px; }
+    .meta { color: #888; font-size: 0.9em; }
+    .overall { background: #f5f5f5; padding: 20px; border-radius: 8px; margin: 20px 0; }
+    .overall.passed { border-left: 4px solid #4caf50; }
+    .overall.failed { border-left: 4px solid #f44336; }
+    .score-value { font-size: 2em; font-weight: bold; }
+    .passed .score-value { color: #4caf50; }
+    .failed .score-value { color: #f44336; }
+    table { width: 100%; border-collapse: collapse; margin: 20px 0; }
+    th, td { padding: 12px; text-align: left; border-bottom: 1px solid #eee; }
+    th { background: #f9f9f9; font-weight: 600; }
+    .status-pass { color: #4caf50; }
+    .status-fail { color: #f44336; }
+    .reasoning { color: #666; font-size: 0.9em; font-style: italic; }
+    .error { background: #ffebee; padding: 10px; border-radius: 4px; margin: 5px 0; }
+    pre { background: #f5f5f5; padding: 15px; border-radius: 4px; overflow-x: auto; }
+  </style>
+</head>
+<body>
+  <h1>${this._escapeHtml(data.title)}</h1>
+  <p class="meta">Generated: ${new Date(data.timestamp).toISOString()}</p>
+
+  <div class="overall ${statusClass}">
+    <div class="score-value">${result.overallScore.toFixed(2)}/10</div>
+    <div>Status: <strong>${result.passed ? "PASSED" : "FAILED"}</strong></div>
+    <div>Aggregation: ${result.aggregationMethod}</div>
+    ${this._config.includeTiming ? `<div>Duration: ${result.totalComputeTime}ms</div>` : ""}
+  </div>
+
+  <h2>Individual Scores</h2>
+  <table>
+    <thead>
+      <tr>
+        <th>Scorer</th>
+        <th>Score</th>
+        <th>Status</th>
+        ${this._config.includeTiming ? "<th>Duration</th>" : ""}
+      </tr>
+    </thead>
+    <tbody>
+`;
+        for (const score of result.scores) {
+            const statusClass = score.passed ? "status-pass" : "status-fail";
+            html += `
+      <tr>
+        <td>${this._escapeHtml(score.scorerName)}</td>
+        <td>${score.score.toFixed(2)}</td>
+        <td class="${statusClass}">${score.passed ? "Pass" : "Fail"}</td>
+        ${this._config.includeTiming ? `<td>${score.computeTime}ms</td>` : ""}
+      </tr>
+`;
+            if (this._config.includeReasoning) {
+                html += `
+      <tr>
+        <td colspan="${this._config.includeTiming ? 4 : 3}" class="reasoning">
+          ${this._escapeHtml(score.reasoning)}
+        </td>
+      </tr>
+`;
+            }
+        }
+        html += `
+    </tbody>
+  </table>
+`;
+        // Errors
+        const pipelineResult = result;
+        if (pipelineResult.errors && pipelineResult.errors.length > 0) {
+            html += `
+  <h2>Errors</h2>
+`;
+            for (const error of pipelineResult.errors) {
+                html += `
+  <div class="error">
+    <strong>${this._escapeHtml(error.scorerId)}</strong>: ${this._escapeHtml(error.error)}
+  </div>
+`;
+            }
+        }
+        // Custom sections
+        if (data.customSections) {
+            for (const section of data.customSections) {
+                html += `
+  <h2>${this._escapeHtml(section.title)}</h2>
+`;
+                if (typeof section.content === "string") {
+                    html += `<p>${this._escapeHtml(section.content)}</p>`;
+                }
+                else {
+                    html += `<pre>${this._escapeHtml(JSON.stringify(section.content, null, 2))}</pre>`;
+                }
+            }
+        }
+        html += `
+</body>
+</html>
+`;
+        return html;
+    }
+    /**
+     * Escape HTML special characters
+     */
+    _escapeHtml(text) {
+        return text
+            .replace(/&/g, "&amp;")
+            .replace(/</g, "&lt;")
+            .replace(/>/g, "&gt;")
+            .replace(/"/g, "&quot;")
+            .replace(/'/g, "&#039;");
+    }
+    /**
+     * Update configuration
+     */
+    configure(config) {
+        this._config = { ...this._config, ...config };
+    }
+}
+/**
+ * Create a report generator
+ */
+export function createReportGenerator(config) {
+    return new ReportGenerator(config);
+}
+/**
+ * Quick report generation functions
+ */
+export const Reports = {
+    /** Generate text report */
+    text: (data) => new ReportGenerator({ format: "text" }).generate(data),
+    /** Generate JSON report */
+    json: (data) => new ReportGenerator({ format: "json" }).generate(data),
+    /** Generate Markdown report */
+    markdown: (data) => new ReportGenerator({ format: "markdown" }).generate(data),
+    /** Generate HTML report */
+    html: (data) => new ReportGenerator({ format: "html" }).generate(data),
+};

package/dist/evaluation/scorers/baseScorer.d.ts

@@ -0,0 +1,83 @@
+/**
+ * @file Abstract base scorer class providing common functionality
+ * All scorers extend this class for consistent behavior
+ */
+import type { JsonObject } from "../../types/common.js";
+import type { ScoreResult, Scorer, ScorerConfig, ScorerInput, ScorerMetadata, ScoreScale } from "../../types/scorerTypes.js";
+/**
+ * Default score scale (0-10)
+ */
+export declare const DEFAULT_SCORE_SCALE: ScoreScale;
+/**
+ * Default scorer configuration
+ */
+export declare const DEFAULT_SCORER_CONFIG: ScorerConfig;
+/**
+ * Abstract base class for all scorers
+ * Provides common functionality and enforces interface compliance
+ */
+export declare abstract class BaseScorer implements Scorer {
+    protected _config: ScorerConfig;
+    protected _metadata: ScorerMetadata;
+    constructor(metadata: ScorerMetadata, config?: ScorerConfig);
+    /**
+     * Get scorer metadata
+     */
+    get metadata(): ScorerMetadata;
+    /**
+     * Get current configuration
+     */
+    get config(): ScorerConfig;
+    /**
+     * Main scoring method - must be implemented by subclasses
+     */
+    abstract score(input: ScorerInput): Promise<ScoreResult>;
+    /**
+     * Validate input has required fields
+     */
+    validateInput(input: ScorerInput): {
+        valid: boolean;
+        errors: string[];
+    };
+    /**
+     * Update configuration
+     */
+    configure(config: Partial<ScorerConfig>): void;
+    /**
+     * Normalize a score to 0-1 scale
+     */
+    protected normalizeScore(score: number, scale?: ScoreScale): number;
+    /**
+     * Convert normalized score back to scale
+     */
+    protected denormalizeScore(normalizedScore: number, scale?: ScoreScale): number;
+    /**
+     * Check if score passes threshold
+     */
+    protected checkThreshold(normalizedScore: number): boolean;
+    /**
+     * Create a standardized score result
+     */
+    protected createScoreResult(score: number, reasoning: string, options?: {
+        scale?: ScoreScale;
+        confidence?: number;
+        metadata?: JsonObject;
+        error?: string;
+    }): ScoreResult;
+    /**
+     * Create an error score result
+     */
+    protected createErrorResult(error: Error | string): ScoreResult;
+    /**
+     * Execute scoring with timing and error handling
+     */
+    protected executeWithTiming(scoringFn: () => Promise<Omit<ScoreResult, "computeTime">>): Promise<ScoreResult>;
+    /**
+     * Execute scoring with timeout
+     */
+    protected executeWithTimeout<T>(fn: () => Promise<T>, timeoutMs: number, operationName: string): Promise<T>;
+    /**
+     * Execute with retry logic
+     */
+    protected executeWithRetry<T>(operation: () => Promise<T>, retries?: number): Promise<T>;
+}

package/dist/evaluation/scorers/baseScorer.js

@@ -0,0 +1,232 @@
+/**
+ * @file Abstract base scorer class providing common functionality
+ * All scorers extend this class for consistent behavior
+ */
+import { withTimeout, ErrorFactory } from "../../utils/errorHandling.js";
+import { logger } from "../../utils/logger.js";
+/**
+ * Default score scale (0-10)
+ */
+export const DEFAULT_SCORE_SCALE = {
+    min: 0,
+    max: 10,
+    precision: 2,
+};
+/**
+ * Default scorer configuration
+ */
+export const DEFAULT_SCORER_CONFIG = {
+    enabled: true,
+    threshold: 0.7,
+    weight: 1.0,
+    timeout: 30000,
+    retries: 2,
+};
+/**
+ * Abstract base class for all scorers
+ * Provides common functionality and enforces interface compliance
+ */
+export class BaseScorer {
+    _config;
+    _metadata;
+    constructor(metadata, config) {
+        this._metadata = metadata;
+        this._config = {
+            ...DEFAULT_SCORER_CONFIG,
+            ...metadata.defaultConfig,
+            ...config,
+        };
+    }
+    /**
+     * Get scorer metadata
+     */
+    get metadata() {
+        return this._metadata;
+    }
+    /**
+     * Get current configuration
+     */
+    get config() {
+        return this._config;
+    }
+    /**
+     * Validate input has required fields
+     */
+    validateInput(input) {
+        const errors = [];
+        for (const field of this._metadata.requiredInputs) {
+            const value = input[field];
+            if (value === undefined || value === null) {
+                errors.push(`Missing required input: ${field}`);
+            }
+        }
+        // Check for empty strings in required text fields
+        if (this._metadata.requiredInputs.includes("query") &&
+            typeof input.query === "string" &&
+            !input.query.trim()) {
+            errors.push("Query cannot be empty");
+        }
+        if (this._metadata.requiredInputs.includes("response") &&
+            typeof input.response === "string" &&
+            !input.response.trim()) {
+            errors.push("Response cannot be empty");
+        }
+        // Check context array is not empty if required
+        if (this._metadata.requiredInputs.includes("context") &&
+            input.context !== undefined) {
+            if (!Array.isArray(input.context) || input.context.length === 0) {
+                errors.push("Context must be a non-empty array");
+            }
+        }
+        return {
+            valid: errors.length === 0,
+            errors,
+        };
+    }
+    /**
+     * Update configuration
+     */
+    configure(config) {
+        this._config = {
+            ...this._config,
+            ...config,
+        };
+        logger.debug(`Scorer ${this._metadata.id} reconfigured`, {
+            config: this._config,
+        });
+    }
+    /**
+     * Normalize a score to 0-1 scale
+     */
+    normalizeScore(score, scale = DEFAULT_SCORE_SCALE) {
+        // Validate inputs are finite
+        if (!Number.isFinite(score) ||
+            !Number.isFinite(scale.min) ||
+            !Number.isFinite(scale.max)) {
+            return 0;
+        }
+        // Guard against zero denominator
+        const denominator = scale.max - scale.min;
+        if (denominator === 0) {
+            return 0;
+        }
+        // Clamp score to scale bounds
+        const clampedScore = Math.max(scale.min, Math.min(scale.max, score));
+        const normalized = (clampedScore - scale.min) / denominator;
+        return Math.max(0, Math.min(1, normalized));
+    }
+    /**
+     * Convert normalized score back to scale
+     */
+    denormalizeScore(normalizedScore, scale = DEFAULT_SCORE_SCALE) {
+        const clamped = Math.max(0, Math.min(1, normalizedScore));
+        return scale.min + clamped * (scale.max - scale.min);
+    }
+    /**
+     * Check if score passes threshold
+     */
+    checkThreshold(normalizedScore) {
+        const threshold = this._config.threshold ?? 0.7;
+        return normalizedScore >= threshold;
+    }
+    /**
+     * Create a standardized score result
+     */
+    createScoreResult(score, reasoning, options = {}) {
+        const scale = options.scale ?? DEFAULT_SCORE_SCALE;
+        const safeScore = Number.isFinite(score) ? score : scale.min;
+        const clampedScore = Math.max(scale.min, Math.min(scale.max, safeScore));
+        const normalizedScore = this.normalizeScore(clampedScore, scale);
+        // Ensure no NaN leaks into published scores
+        const finalScore = Number.isFinite(clampedScore) ? clampedScore : 0;
+        const finalNormalized = Number.isFinite(normalizedScore)
+            ? normalizedScore
+            : 0;
+        return {
+            scorerId: this._metadata.id,
+            scorerName: this._metadata.name,
+            score: Number(finalScore.toFixed(scale.precision)),
+            normalizedScore: Number(finalNormalized.toFixed(4)),
+            scale,
+            reasoning,
+            passed: this.checkThreshold(finalNormalized),
+            threshold: this._config.threshold ?? 0.7,
+            confidence: options.confidence === undefined
+                ? undefined
+                : Number.isFinite(options.confidence)
+                    ? Math.max(0, Math.min(1, options.confidence))
+                    : 0,
+            metadata: options.metadata,
+            computeTime: 0, // Set by caller via executeWithTiming
+            error: options.error,
+        };
+    }
+    /**
+     * Create an error score result
+     */
+    createErrorResult(error) {
+        const errorMessage = error instanceof Error ? error.message : error;
+        return {
+            scorerId: this._metadata.id,
+            scorerName: this._metadata.name,
+            score: 0,
+            normalizedScore: 0,
+            scale: DEFAULT_SCORE_SCALE,
+            reasoning: `Scoring failed: ${errorMessage}`,
+            passed: false,
+            threshold: this._config.threshold ?? 0.7,
+            computeTime: 0,
+            error: errorMessage,
+        };
+    }
+    /**
+     * Execute scoring with timing and error handling
+     */
+    async executeWithTiming(scoringFn) {
+        const startTime = Date.now();
+        try {
+            const result = await scoringFn();
+            return {
+                ...result,
+                computeTime: Date.now() - startTime,
+            };
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            logger.error(`Scorer ${this._metadata.id} failed`, {
+                error: errorMessage,
+            });
+            return {
+                ...this.createErrorResult(errorMessage),
+                computeTime: Date.now() - startTime,
+            };
+        }
+    }
+    /**
+     * Execute scoring with timeout
+     */
+    async executeWithTimeout(fn, timeoutMs, operationName) {
+        return withTimeout(fn(), timeoutMs, ErrorFactory.evaluationTimeout(operationName, timeoutMs));
+    }
+    /**
+     * Execute with retry logic
+     */
+    async executeWithRetry(operation, retries) {
+        const maxRetries = retries ?? this._config.retries ?? 2;
+        let lastError;
+        for (let attempt = 0; attempt <= maxRetries; attempt++) {
+            try {
+                return await operation();
+            }
+            catch (error) {
+                lastError = error instanceof Error ? error : new Error(String(error));
+                if (attempt < maxRetries) {
+                    logger.debug(`Scorer ${this._metadata.id} retry ${attempt + 1}/${maxRetries}`, { error: lastError.message });
+                    // Exponential backoff
+                    await new Promise((resolve) => setTimeout(resolve, 2 ** attempt * 1000));
+                }
+            }
+        }
+        throw lastError ?? new Error("Operation failed after retries");
+    }
+}