@juspay/neurolink 9.36.1 → 9.37.0

package/dist/lib/evaluation/hooks/langfuseAdapter.d.ts

@@ -0,0 +1,99 @@
+/**
+ * @file Langfuse Adapter
+ * Integration with Langfuse for LLM observability
+ */
+/**
+ * Langfuse client interface (minimal for type safety)
+ */
+export type LangfuseClient = {
+    score: (params: {
+        name: string;
+        value: number;
+        traceId?: string;
+        observationId?: string;
+        comment?: string;
+        metadata?: Record<string, unknown>;
+    }) => Promise<unknown>;
+    trace?: (params: {
+        name: string;
+        metadata?: Record<string, unknown>;
+        tags?: string[];
+    }) => {
+        id: string;
+    };
+    shutdown?: () => Promise<void>;
+};
+/**
+ * Langfuse adapter configuration
+ */
+export type LangfuseAdapterConfig = {
+    /** Langfuse client instance */
+    client: LangfuseClient;
+    /** Prefix for score names */
+    scorePrefix?: string;
+    /** Include detailed metadata */
+    includeMetadata?: boolean;
+    /** Tags to add to all scores */
+    tags?: string[];
+    /** Whether to send pipeline-level scores */
+    sendPipelineScores?: boolean;
+    /** Whether to send individual scorer scores */
+    sendScorerScores?: boolean;
+};
+/**
+ * Langfuse adapter for evaluation observability
+ */
+export declare class LangfuseAdapter {
+    private _config;
+    private _unsubscribers;
+    private _traceIdMap;
+    constructor(config: LangfuseAdapterConfig);
+    /**
+     * Start listening to evaluation events
+     */
+    start(): void;
+    /**
+     * Stop listening to events
+     */
+    stop(): void;
+    /**
+     * Send scorer score to Langfuse
+     */
+    private _sendScorerScore;
+    /**
+     * Send pipeline scores to Langfuse
+     */
+    private _sendPipelineScores;
+    /**
+     * Manually send a score to Langfuse
+     */
+    sendScore(name: string, value: number, options?: {
+        traceId?: string;
+        comment?: string;
+        metadata?: Record<string, unknown>;
+    }): Promise<void>;
+    /**
+     * Shutdown the adapter and flush any pending data
+     */
+    shutdown(): Promise<void>;
+}
+/**
+ * Create a Langfuse adapter
+ */
+export declare function createLangfuseAdapter(config: LangfuseAdapterConfig): LangfuseAdapter;
+/**
+ * Create and start a Langfuse adapter
+ */
+export declare function startLangfuseAdapter(config: LangfuseAdapterConfig): LangfuseAdapter;
+/**
+ * Helper: Create a mock Langfuse client for testing
+ */
+export declare function createMockLangfuseClient(): LangfuseClient & {
+    scores: Array<{
+        name: string;
+        value: number;
+        traceId?: string;
+        comment?: string;
+        metadata?: Record<string, unknown>;
+    }>;
+};

package/dist/lib/evaluation/hooks/langfuseAdapter.js

@@ -0,0 +1,173 @@
+/**
+ * @file Langfuse Adapter
+ * Integration with Langfuse for LLM observability
+ */
+import { logger } from "../../utils/logger.js";
+import { observabilityHooks } from "./observabilityHooks.js";
+/**
+ * Langfuse adapter for evaluation observability
+ */
+export class LangfuseAdapter {
+    _config;
+    _unsubscribers = [];
+    _traceIdMap = new Map();
+    constructor(config) {
+        this._config = {
+            scorePrefix: "eval",
+            includeMetadata: true,
+            tags: [],
+            sendPipelineScores: true,
+            sendScorerScores: true,
+            ...config,
+        };
+    }
+    /**
+     * Start listening to evaluation events
+     */
+    start() {
+        // Prevent duplicate subscriptions
+        if (this._unsubscribers.length > 0) {
+            return;
+        }
+        // Listen for scorer completions
+        if (this._config.sendScorerScores) {
+            const scorerUnsub = observabilityHooks.on("scorer:end", (event) => {
+                this._sendScorerScore(event.result, event.traceContext?.traceId);
+            });
+            this._unsubscribers.push(scorerUnsub);
+        }
+        // Listen for pipeline completions
+        if (this._config.sendPipelineScores) {
+            const pipelineUnsub = observabilityHooks.on("pipeline:end", (event) => {
+                this._sendPipelineScores(event.result, event.traceContext?.traceId);
+            });
+            this._unsubscribers.push(pipelineUnsub);
+        }
+        logger.debug("Langfuse adapter started");
+    }
+    /**
+     * Stop listening to events
+     */
+    stop() {
+        for (const unsub of this._unsubscribers) {
+            unsub();
+        }
+        this._unsubscribers = [];
+        this._traceIdMap.clear();
+        logger.debug("Langfuse adapter stopped");
+    }
+    /**
+     * Send scorer score to Langfuse
+     */
+    async _sendScorerScore(result, traceId) {
+        try {
+            const scoreName = `${this._config.scorePrefix}.${result.scorerId}`;
+            const normalizedValue = result.normalizedScore; // Already 0-1
+            await this._config.client.score({
+                name: scoreName,
+                value: normalizedValue,
+                traceId,
+                comment: result.reasoning,
+                metadata: this._config.includeMetadata
+                    ? {
+                        passed: result.passed,
+                        threshold: result.threshold,
+                        computeTime: result.computeTime,
+                        confidence: result.confidence,
+                        ...(result.metadata ?? {}),
+                    }
+                    : undefined,
+            });
+            logger.debug(`Sent score to Langfuse: ${scoreName}=${normalizedValue}`);
+        }
+        catch (error) {
+            logger.error("Failed to send score to Langfuse", {
+                scorerId: result.scorerId,
+                error: error instanceof Error ? error.message : String(error),
+            });
+        }
+    }
+    /**
+     * Send pipeline scores to Langfuse
+     */
+    async _sendPipelineScores(result, externalTraceId) {
+        const traceId = externalTraceId ?? result.correlationId;
+        const pipelineName = result.pipelineConfig.name ?? "unnamed";
+        try {
+            // Send overall pipeline score
+            await this._config.client.score({
+                name: `${this._config.scorePrefix}.pipeline.${pipelineName}.overall`,
+                value: result.overallScore / 10, // Normalize to 0-1
+                traceId,
+                comment: `Pipeline evaluation: ${result.passed ? "PASSED" : "FAILED"}`,
+                metadata: this._config.includeMetadata
+                    ? {
+                        passed: result.passed,
+                        aggregationMethod: result.aggregationMethod,
+                        scorerCount: result.scores.length,
+                        totalComputeTime: result.totalComputeTime,
+                        errorCount: result.errors.length,
+                    }
+                    : undefined,
+            });
+            logger.debug(`Sent pipeline score to Langfuse: ${pipelineName}=${result.overallScore / 10}`);
+        }
+        catch (error) {
+            logger.error("Failed to send pipeline score to Langfuse", {
+                pipelineName,
+                error: error instanceof Error ? error.message : String(error),
+            });
+        }
+    }
+    /**
+     * Manually send a score to Langfuse
+     */
+    async sendScore(name, value, options) {
+        const scoreName = `${this._config.scorePrefix}.${name}`;
+        await this._config.client.score({
+            name: scoreName,
+            value,
+            traceId: options?.traceId,
+            comment: options?.comment,
+            metadata: options?.metadata,
+        });
+    }
+    /**
+     * Shutdown the adapter and flush any pending data
+     */
+    async shutdown() {
+        this.stop();
+        if (this._config.client.shutdown) {
+            await this._config.client.shutdown();
+        }
+    }
+}
+/**
+ * Create a Langfuse adapter
+ */
+export function createLangfuseAdapter(config) {
+    return new LangfuseAdapter(config);
+}
+/**
+ * Create and start a Langfuse adapter
+ */
+export function startLangfuseAdapter(config) {
+    const adapter = new LangfuseAdapter(config);
+    adapter.start();
+    return adapter;
+}
+/**
+ * Helper: Create a mock Langfuse client for testing
+ */
+export function createMockLangfuseClient() {
+    const scores = [];
+    return {
+        scores,
+        score: async (params) => {
+            scores.push(params);
+            return { id: `score-${scores.length}` };
+        },
+        shutdown: async () => { },
+    };
+}
+//# sourceMappingURL=langfuseAdapter.js.map

package/dist/lib/evaluation/hooks/observabilityHooks.d.ts

@@ -0,0 +1,129 @@
+/**
+ * @file Observability Hooks
+ * OpenTelemetry integration for evaluation tracing
+ */
+import type { EvaluationTraceContext, ScoreResult } from "../../types/scorerTypes.js";
+import type { PipelineResult } from "../pipeline/evaluationPipeline.js";
+/**
+ * Event handler type
+ */
+export type EventHandler<T> = (event: T) => void | Promise<void>;
+/**
+ * Evaluation events
+ */
+export type EvaluationEvents = {
+    "scorer:start": {
+        scorerId: string;
+        scorerName: string;
+        timestamp: number;
+        traceContext?: EvaluationTraceContext;
+    };
+    "scorer:end": {
+        scorerId: string;
+        scorerName: string;
+        result: ScoreResult;
+        timestamp: number;
+        duration: number;
+        traceContext?: EvaluationTraceContext;
+    };
+    "scorer:error": {
+        scorerId: string;
+        scorerName: string;
+        error: string;
+        timestamp: number;
+        traceContext?: EvaluationTraceContext;
+    };
+    "pipeline:start": {
+        pipelineName: string;
+        scorerCount: number;
+        timestamp: number;
+        correlationId: string;
+        traceContext?: EvaluationTraceContext;
+    };
+    "pipeline:end": {
+        pipelineName: string;
+        result: PipelineResult;
+        timestamp: number;
+        duration: number;
+        traceContext?: EvaluationTraceContext;
+    };
+    "pipeline:error": {
+        pipelineName: string;
+        error: string;
+        timestamp: number;
+        traceContext?: EvaluationTraceContext;
+    };
+};
+/**
+ * Observability hooks manager
+ */
+export declare class ObservabilityHooks {
+    private _handlers;
+    private _traceContext?;
+    private _enabled;
+    /**
+     * Enable/disable observability
+     */
+    set enabled(value: boolean);
+    get enabled(): boolean;
+    /**
+     * Set trace context for all events
+     */
+    setTraceContext(context: EvaluationTraceContext): void;
+    /**
+     * Clear trace context
+     */
+    clearTraceContext(): void;
+    /**
+     * Get current trace context
+     */
+    getTraceContext(): EvaluationTraceContext | undefined;
+    /**
+     * Register an event handler
+     */
+    on<K extends keyof EvaluationEvents>(event: K, handler: EventHandler<EvaluationEvents[K]>): () => void;
+    /**
+     * Remove an event handler
+     */
+    off<K extends keyof EvaluationEvents>(event: K, handler: EventHandler<EvaluationEvents[K]>): void;
+    /**
+     * Emit an event
+     */
+    emit<K extends keyof EvaluationEvents>(event: K, data: Omit<EvaluationEvents[K], "traceContext">): Promise<void>;
+    /**
+     * Clear all handlers
+     */
+    clear(): void;
+    /**
+     * Get handler count for an event
+     */
+    listenerCount(event: keyof EvaluationEvents): number;
+}
+/**
+ * Global observability hooks instance
+ */
+export declare const observabilityHooks: ObservabilityHooks;
+/**
+ * Helper: Create a console logger hook
+ */
+export declare function createConsoleLoggerHook(): void;
+/**
+ * Helper: Create a metrics collector hook
+ * Accepts the actual MetricsCollector interface from reporting/metricsCollector
+ */
+export declare function createMetricsCollectorHook(collector: {
+    recordScorer: (scorerId: string, scorerName: string, result: ScoreResult) => void;
+    recordPipeline: (result: PipelineResult) => void;
+}): void;
+/**
+ * OpenTelemetry span attributes
+ */
+export type SpanAttributes = Record<string, string | number | boolean>;
+/**
+ * Create span attributes from scorer result
+ */
+export declare function scorerToSpanAttributes(result: ScoreResult): SpanAttributes;
+/**
+ * Create span attributes from pipeline result
+ */
+export declare function pipelineToSpanAttributes(result: PipelineResult): SpanAttributes;

package/dist/lib/evaluation/hooks/observabilityHooks.js

@@ -0,0 +1,182 @@
+/**
+ * @file Observability Hooks
+ * OpenTelemetry integration for evaluation tracing
+ */
+import { logger } from "../../utils/logger.js";
+/**
+ * Observability hooks manager
+ */
+export class ObservabilityHooks {
+    _handlers = new Map();
+    _traceContext;
+    _enabled = true;
+    /**
+     * Enable/disable observability
+     */
+    set enabled(value) {
+        this._enabled = value;
+    }
+    get enabled() {
+        return this._enabled;
+    }
+    /**
+     * Set trace context for all events
+     */
+    setTraceContext(context) {
+        this._traceContext = context;
+    }
+    /**
+     * Clear trace context
+     */
+    clearTraceContext() {
+        this._traceContext = undefined;
+    }
+    /**
+     * Get current trace context
+     */
+    getTraceContext() {
+        return this._traceContext;
+    }
+    /**
+     * Register an event handler
+     */
+    on(event, handler) {
+        if (!this._handlers.has(event)) {
+            this._handlers.set(event, new Set());
+        }
+        this._handlers.get(event).add(handler);
+        // Return unsubscribe function
+        return () => {
+            this._handlers.get(event)?.delete(handler);
+        };
+    }
+    /**
+     * Remove an event handler
+     */
+    off(event, handler) {
+        this._handlers.get(event)?.delete(handler);
+    }
+    /**
+     * Emit an event
+     */
+    async emit(event, data) {
+        if (!this._enabled) {
+            return;
+        }
+        const handlers = this._handlers.get(event);
+        if (!handlers || handlers.size === 0) {
+            return;
+        }
+        const eventData = {
+            ...data,
+            traceContext: this._traceContext,
+        };
+        const promises = [];
+        for (const handler of handlers) {
+            try {
+                const result = handler(eventData);
+                if (result instanceof Promise) {
+                    promises.push(result.catch((err) => {
+                        logger.error(`Event handler error for ${event}`, { error: err });
+                    }));
+                }
+            }
+            catch (error) {
+                logger.error(`Event handler error for ${event}`, { error });
+            }
+        }
+        // Wait for async handlers
+        await Promise.all(promises);
+    }
+    /**
+     * Clear all handlers
+     */
+    clear() {
+        this._handlers.clear();
+    }
+    /**
+     * Get handler count for an event
+     */
+    listenerCount(event) {
+        return this._handlers.get(event)?.size ?? 0;
+    }
+}
+/**
+ * Global observability hooks instance
+ */
+export const observabilityHooks = new ObservabilityHooks();
+/**
+ * Helper: Create a console logger hook
+ */
+export function createConsoleLoggerHook() {
+    observabilityHooks.on("scorer:start", (event) => {
+        logger.info(`[SCORER] ${event.scorerName} started at ${new Date(event.timestamp).toISOString()}`);
+    });
+    observabilityHooks.on("scorer:end", (event) => {
+        logger.info(`[SCORER] ${event.scorerName} completed: score=${event.result.score.toFixed(1)}, ` +
+            `passed=${event.result.passed}, duration=${event.duration}ms`);
+    });
+    observabilityHooks.on("scorer:error", (event) => {
+        logger.error(`[SCORER] ${event.scorerName} error: ${event.error}`);
+    });
+    observabilityHooks.on("pipeline:start", (event) => {
+        logger.info(`[PIPELINE] ${event.pipelineName} started with ${event.scorerCount} scorers ` +
+            `(correlationId: ${event.correlationId})`);
+    });
+    observabilityHooks.on("pipeline:end", (event) => {
+        logger.info(`[PIPELINE] ${event.pipelineName} completed: overall=${event.result.overallScore.toFixed(1)}, ` +
+            `passed=${event.result.passed}, duration=${event.duration}ms`);
+    });
+    observabilityHooks.on("pipeline:error", (event) => {
+        logger.error(`[PIPELINE] ${event.pipelineName} error: ${event.error}`);
+    });
+}
+/**
+ * Helper: Create a metrics collector hook
+ * Accepts the actual MetricsCollector interface from reporting/metricsCollector
+ */
+export function createMetricsCollectorHook(collector) {
+    observabilityHooks.on("scorer:end", (event) => {
+        collector.recordScorer(event.scorerId, event.scorerName, event.result);
+    });
+    observabilityHooks.on("pipeline:end", (event) => {
+        collector.recordPipeline(event.result);
+    });
+}
+/**
+ * Create span attributes from scorer result
+ */
+export function scorerToSpanAttributes(result) {
+    return {
+        "scorer.id": result.scorerId,
+        "scorer.name": result.scorerName,
+        "scorer.score": result.score,
+        "scorer.normalizedScore": result.normalizedScore,
+        "scorer.passed": result.passed,
+        "scorer.threshold": result.threshold,
+        "scorer.computeTime": result.computeTime,
+        ...(result.confidence !== undefined && {
+            "scorer.confidence": result.confidence,
+        }),
+        ...(result.error && { "scorer.error": result.error }),
+    };
+}
+/**
+ * Create span attributes from pipeline result
+ */
+export function pipelineToSpanAttributes(result) {
+    return {
+        "pipeline.name": result.pipelineConfig.name ?? "unnamed",
+        "pipeline.overallScore": result.overallScore,
+        "pipeline.passed": result.passed,
+        "pipeline.aggregationMethod": result.aggregationMethod,
+        "pipeline.scorerCount": result.scores.length,
+        "pipeline.totalComputeTime": result.totalComputeTime,
+        "pipeline.errorCount": result.errors.length,
+        "pipeline.skippedCount": result.skippedScorers.length,
+        ...(result.correlationId && {
+            "pipeline.correlationId": result.correlationId,
+        }),
+    };
+}
+//# sourceMappingURL=observabilityHooks.js.map

package/dist/lib/evaluation/index.d.ts

@@ -2,10 +2,19 @@
  * @file This file exports the main Evaluator class, which serves as the central entry point for the evaluation system.
  */
 import type { LanguageModelV3CallOptions } from "@ai-sdk/provider";
-import type { GenerateResult } from "../types/generateTypes.js";
+import type { EvaluationData } from "../types/evaluation.js";
 import type { EvaluationConfig } from "../types/evaluationTypes.js";
+import type { GenerateResult } from "../types/generateTypes.js";
 import type { AutoEvaluationConfig } from "../types/middlewareTypes.js";
-import type { EvaluationData } from "../types/evaluation.js";
+export * from "./errors/index.js";
+export * from "./hooks/index.js";
+export * from "./pipeline/index.js";
+export * from "./reporting/index.js";
+export * from "./scorers/index.js";
+export { BatchEvaluator, type BatchEvaluationConfig, type BatchEvaluationItem, type BatchEvaluationItemResult, type BatchEvaluationResult, } from "./BatchEvaluator.js";
+export { EvaluationAggregator, type ScoreStatistics, type ScoreDistribution, type TrendAnalysis, type DimensionAnalysis, type AlertSummary, type AggregationResult, } from "./EvaluationAggregator.js";
+export { EvaluatorFactory, getEvaluatorFactory, type EvaluatorPreset, } from "./EvaluatorFactory.js";
+export { EvaluatorRegistry, getEvaluatorRegistry, type EvaluationStrategyFunction, type EvaluationStrategyConfig, type EvaluationStrategyMetadata, } from "./EvaluatorRegistry.js";
 /**
  * A centralized class for performing response evaluations. It supports different
  * evaluation strategies, with RAGAS-style model-based evaluation as the default.

package/dist/lib/evaluation/index.js

@@ -4,6 +4,21 @@
 import { ContextBuilder } from "./contextBuilder.js";
 import { RAGASEvaluator } from "./ragasEvaluator.js";
 import { mapToEvaluationData } from "./scoring.js";
+// Re-export errors
+export * from "./errors/index.js";
+// Re-export hooks
+export * from "./hooks/index.js";
+// Re-export pipeline
+export * from "./pipeline/index.js";
+// Re-export reporting
+export * from "./reporting/index.js";
+// Re-export scorers
+export * from "./scorers/index.js";
+// Re-export Factory and Registry (Mastra-inspired patterns)
+export { BatchEvaluator, } from "./BatchEvaluator.js";
+export { EvaluationAggregator, } from "./EvaluationAggregator.js";
+export { EvaluatorFactory, getEvaluatorFactory, } from "./EvaluatorFactory.js";
+export { EvaluatorRegistry, getEvaluatorRegistry, } from "./EvaluatorRegistry.js";
 /**
  * A centralized class for performing response evaluations. It supports different
  * evaluation strategies, with RAGAS-style model-based evaluation as the default.