observability-toolkit 2.0.0 → 2.1.0

package/dist/lib/quality/quality-metrics.js

@@ -0,0 +1,1151 @@
+/**
+ * Quality Metrics Dashboard Helpers
+ *
+ * Pre-defined evaluation metrics with alerting thresholds for LLM quality monitoring.
+ * Designed for dashboard integration with observability platforms (SigNoz, Grafana, etc.).
+ *
+ * @see BACKLOG.md Quality Metrics Dashboard section
+ * @see https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-events/
+ */
+import { z } from 'zod';
+import { evaluationAggregationSchema, timeRangeSchema } from '../core/shared-schemas.js';
+// Internal imports from submodules (used by computeQualityMetric / computeDashboardSummary)
+import { computeTrend, computeConfidence } from './quality-trends.js';
+import { evaluateSLAs } from './quality-sla.js';
+import { computeNormalizedEntropy } from './quality-feature-engineering.js';
+import { RELEVANCE_WARNING, RELEVANCE_CRITICAL, TASK_COMPLETION_WARNING, TASK_COMPLETION_CRITICAL, TOOL_CORRECTNESS_WARNING, TOOL_CORRECTNESS_CRITICAL, TOOL_CORRECTNESS_RELIABILITY_FLOOR, HALLUCINATION_WARNING, HALLUCINATION_CRITICAL, EVAL_LATENCY_WARNING_S, EVAL_LATENCY_CRITICAL_S, FAITHFULNESS_WARNING, FAITHFULNESS_CRITICAL, COHERENCE_WARNING, EXPLANATION_QUALITY_WARNING, EXPLANATION_QUALITY_CRITICAL, HANDOFF_CORRECTNESS_WARNING, HANDOFF_CORRECTNESS_CRITICAL, DIVERGENCE_ENTROPY_THRESHOLD, DEFAULT_BIN_COUNT, MIN_DIVERGENCE_SAMPLES, QUALITY_PERCENTILE_P50, QUALITY_PERCENTILE_P95, QUALITY_PERCENTILE_P99, QUALITY_PERCENT_SCALE, UNIT_RANGE, LATENCY_RANGE, ALERT_MESSAGES, REMEDIATION_HINTS, } from './quality-constants.js';
+export { roleTypeSchema, qualityMetricNameSchema, cqiWeightsSchema, TREND_MIN_SAMPLE_SIZE, DEFAULT_BIN_COUNT, DEFAULT_ENTROPY_THRESHOLD } from './quality-constants.js';
+// ============================================================================
+// Constants (needed by schemas below)
+// ============================================================================
+/** Recognized SLA window formats */
+export const slaWindowFormatSchema = z.enum([
+    '7d rolling', '14d rolling', '30d rolling', '60d rolling', '90d rolling',
+    'calendar day', 'calendar week', 'calendar month', 'calendar quarter', 'calendar year',
+]);
+/** All possible SLA evaluation statuses */
+export const slaStatusSchema = z.enum(['compliant', 'non_compliant', 'no_data', 'metric_not_found']);
+// ============================================================================
+// Pre-defined Quality Metrics (from BACKLOG.md)
+// ============================================================================
+/**
+ * Pre-defined quality metrics with recommended thresholds.
+ * Based on industry best practices for LLM evaluation.
+ */
+export const QUALITY_METRICS = {
+    // Response quality metric
+    relevance: {
+        name: 'relevance',
+        displayName: 'Response Relevance',
+        description: 'Measures how relevant LLM responses are to the input query',
+        aggregations: ['avg', 'p50', 'p95', 'min', 'count'],
+        alerts: [
+            {
+                aggregation: 'p50',
+                value: RELEVANCE_WARNING,
+                direction: 'below',
+                severity: 'warning',
+                message: ALERT_MESSAGES.relevance.warning,
+            },
+            {
+                aggregation: 'p50',
+                value: RELEVANCE_CRITICAL,
+                direction: 'below',
+                severity: 'critical',
+                message: ALERT_MESSAGES.relevance.critical,
+            },
+            {
+                type: 'divergence',
+                entropyThreshold: DIVERGENCE_ENTROPY_THRESHOLD,
+                bins: DEFAULT_BIN_COUNT,
+                severity: 'warning',
+                message: ALERT_MESSAGES.relevance.divergence,
+            },
+        ],
+        range: UNIT_RANGE,
+        unit: 'score',
+        remediationHints: [...REMEDIATION_HINTS.relevance],
+    },
+    // Agent effectiveness metric
+    task_completion: {
+        name: 'task_completion',
+        displayName: 'Task Completion Rate',
+        description: 'Rate at which agents successfully complete assigned tasks',
+        aggregations: ['avg', 'p50', 'count'],
+        alerts: [
+            {
+                aggregation: 'avg',
+                value: TASK_COMPLETION_WARNING,
+                direction: 'below',
+                severity: 'warning',
+                message: ALERT_MESSAGES.task_completion.warning,
+            },
+            {
+                aggregation: 'avg',
+                value: TASK_COMPLETION_CRITICAL,
+                direction: 'below',
+                severity: 'critical',
+                message: ALERT_MESSAGES.task_completion.critical,
+            },
+            {
+                type: 'divergence',
+                entropyThreshold: DIVERGENCE_ENTROPY_THRESHOLD,
+                bins: DEFAULT_BIN_COUNT,
+                severity: 'warning',
+                message: ALERT_MESSAGES.task_completion.divergence,
+            },
+        ],
+        range: UNIT_RANGE,
+        unit: 'rate',
+        remediationHints: [...REMEDIATION_HINTS.task_completion],
+    },
+    // Tool selection accuracy
+    tool_correctness: {
+        name: 'tool_correctness',
+        displayName: 'Tool Selection Accuracy',
+        description: 'Accuracy of tool selection in agent workflows',
+        aggregations: ['avg', 'p50', 'count'],
+        alerts: [
+            {
+                aggregation: 'avg',
+                value: TOOL_CORRECTNESS_WARNING,
+                direction: 'below',
+                severity: 'warning',
+                message: ALERT_MESSAGES.tool_correctness.warning,
+            },
+            {
+                aggregation: 'avg',
+                value: TOOL_CORRECTNESS_CRITICAL,
+                direction: 'below',
+                severity: 'critical',
+                message: ALERT_MESSAGES.tool_correctness.critical,
+            },
+        ],
+        range: UNIT_RANGE,
+        unit: 'rate',
+        remediationHints: [...REMEDIATION_HINTS.tool_correctness],
+    },
+    // Factual accuracy metric
+    hallucination: {
+        name: 'hallucination',
+        displayName: 'Hallucination Rate',
+        description: 'Rate of factually incorrect or fabricated information in responses',
+        aggregations: ['avg', 'p95', 'max', 'count'],
+        alerts: [
+            {
+                aggregation: 'avg',
+                value: HALLUCINATION_WARNING,
+                direction: 'above',
+                severity: 'warning',
+                message: ALERT_MESSAGES.hallucination.warning,
+            },
+            {
+                aggregation: 'avg',
+                value: HALLUCINATION_CRITICAL,
+                direction: 'above',
+                severity: 'critical',
+                message: ALERT_MESSAGES.hallucination.critical,
+            },
+        ],
+        range: UNIT_RANGE,
+        unit: 'rate',
+        remediationHints: [...REMEDIATION_HINTS.hallucination],
+    },
+    // Evaluation latency metric
+    evaluation_latency: {
+        name: 'evaluation_latency',
+        displayName: 'Evaluation Latency',
+        description: 'Time taken to complete LLM-as-Judge evaluations',
+        aggregations: ['avg', 'p50', 'p95', 'p99', 'max', 'count'],
+        alerts: [
+            {
+                aggregation: 'p95',
+                value: EVAL_LATENCY_WARNING_S,
+                direction: 'above',
+                severity: 'warning',
+                message: ALERT_MESSAGES.evaluation_latency.warning,
+            },
+            {
+                aggregation: 'p95',
+                value: EVAL_LATENCY_CRITICAL_S,
+                direction: 'above',
+                severity: 'critical',
+                message: ALERT_MESSAGES.evaluation_latency.critical,
+            },
+        ],
+        range: LATENCY_RANGE,
+        unit: 'seconds',
+        remediationHints: [...REMEDIATION_HINTS.evaluation_latency],
+    },
+    // Faithfulness metric (QAG)
+    faithfulness: {
+        name: 'faithfulness',
+        displayName: 'Response Faithfulness',
+        description: 'Measures if responses are grounded in provided context',
+        aggregations: ['avg', 'p50', 'p95', 'count'],
+        alerts: [
+            {
+                aggregation: 'p50',
+                value: FAITHFULNESS_WARNING,
+                direction: 'below',
+                severity: 'warning',
+                message: ALERT_MESSAGES.faithfulness.warning,
+            },
+            {
+                aggregation: 'p50',
+                value: FAITHFULNESS_CRITICAL,
+                direction: 'below',
+                severity: 'critical',
+                message: ALERT_MESSAGES.faithfulness.critical,
+            },
+            {
+                type: 'divergence',
+                entropyThreshold: DIVERGENCE_ENTROPY_THRESHOLD,
+                bins: DEFAULT_BIN_COUNT,
+                severity: 'warning',
+                message: ALERT_MESSAGES.faithfulness.divergence,
+            },
+        ],
+        range: UNIT_RANGE,
+        unit: 'score',
+        remediationHints: [...REMEDIATION_HINTS.faithfulness],
+    },
+    // Coherence metric
+    coherence: {
+        name: 'coherence',
+        displayName: 'Response Coherence',
+        description: 'Logical consistency and clarity of responses',
+        aggregations: ['avg', 'p50', 'p95', 'count'],
+        alerts: [
+            {
+                aggregation: 'p50',
+                value: COHERENCE_WARNING,
+                direction: 'below',
+                severity: 'warning',
+                message: ALERT_MESSAGES.coherence.warning,
+            },
+            {
+                type: 'divergence',
+                entropyThreshold: DIVERGENCE_ENTROPY_THRESHOLD,
+                bins: DEFAULT_BIN_COUNT,
+                severity: 'warning',
+                message: ALERT_MESSAGES.coherence.divergence,
+            },
+        ],
+        range: UNIT_RANGE,
+        unit: 'score',
+        remediationHints: [...REMEDIATION_HINTS.coherence],
+    },
+    // Meta-evaluation of judge explanation quality
+    explanation_quality: {
+        name: 'explanation_quality',
+        displayName: 'Explanation Quality',
+        description: 'Meta-evaluation of judge explanation specificity, evidence citation, and actionability',
+        aggregations: ['avg', 'p50', 'p95', 'count'],
+        alerts: [
+            {
+                aggregation: 'p50',
+                value: EXPLANATION_QUALITY_WARNING,
+                direction: 'below',
+                severity: 'warning',
+                message: ALERT_MESSAGES.explanation_quality.warning,
+            },
+            {
+                aggregation: 'p50',
+                value: EXPLANATION_QUALITY_CRITICAL,
+                direction: 'below',
+                severity: 'critical',
+                message: ALERT_MESSAGES.explanation_quality.critical,
+            },
+        ],
+        range: UNIT_RANGE,
+        unit: 'score',
+        remediationHints: [...REMEDIATION_HINTS.explanation_quality],
+    },
+    // Multi-agent handoff accuracy
+    handoff_correctness: {
+        name: 'handoff_correctness',
+        displayName: 'Handoff Correctness',
+        description: 'Accuracy of agent-to-agent task handoffs in multi-agent workflows',
+        aggregations: ['avg', 'p50', 'count'],
+        alerts: [
+            {
+                aggregation: 'avg',
+                value: HANDOFF_CORRECTNESS_WARNING,
+                direction: 'below',
+                severity: 'warning',
+                message: ALERT_MESSAGES.handoff_correctness.warning,
+            },
+            {
+                aggregation: 'avg',
+                value: HANDOFF_CORRECTNESS_CRITICAL,
+                direction: 'below',
+                severity: 'critical',
+                message: ALERT_MESSAGES.handoff_correctness.critical,
+            },
+        ],
+        range: UNIT_RANGE,
+        unit: 'rate',
+        remediationHints: [...REMEDIATION_HINTS.handoff_correctness],
+    },
+};
+// ============================================================================
+// Zod Schemas
+// ============================================================================
+const finiteNumber = z.number().finite({
+    error: 'Must be a finite number (no NaN/Infinity). Check alert threshold values and metric range min/max.'
+});
+const DECIMAL_BASE = 10;
+const SCHEMA_LONG_TEXT_MULTIPLIER = 10;
+const SCHEMA_MESSAGE_MAX_CHARS = 500;
+const SCHEMA_DISPLAY_NAME_MAX_CHARS = 200;
+const SCHEMA_CONDITIONS_MIN = 2;
+const SCHEMA_CONDITIONS_MAX = 10;
+const SCHEMA_MIN_DIVERGENCE_BINS = 2;
+const SCHEMA_MAX_DIVERGENCE_BINS = 100;
+const SCHEMA_MAX_SUSTAINED_PERIODS = 12;
+const SCHEMA_MAX_REMEDIATION_HINTS = 10;
+const SCHEMA_SHORT_TEXT_MAX = QUALITY_PERCENT_SCALE;
+const SCHEMA_LONG_TEXT_MAX = QUALITY_PERCENT_SCALE * SCHEMA_LONG_TEXT_MULTIPLIER;
+export const alertSeveritySchema = z.enum(['info', 'warning', 'critical']);
+const thresholdDirectionSchema = z.enum(['above', 'below']);
+export const healthStatusSchema = z.enum(['healthy', 'warning', 'critical', 'no_data']);
+const valueAlertSchema = z.object({
+    aggregation: evaluationAggregationSchema,
+    value: finiteNumber,
+    direction: thresholdDirectionSchema,
+    severity: alertSeveritySchema,
+    message: z.string().max(SCHEMA_MESSAGE_MAX_CHARS),
+});
+const divergenceAlertSchema = z.object({
+    type: z.literal('divergence'),
+    entropyThreshold: finiteNumber.refine(v => v >= 0 && v <= 1, {
+        error: 'entropyThreshold must be between 0 and 1'
+    }),
+    bins: z.int().min(SCHEMA_MIN_DIVERGENCE_BINS).max(SCHEMA_MAX_DIVERGENCE_BINS),
+    severity: alertSeveritySchema,
+    message: z.string().max(SCHEMA_MESSAGE_MAX_CHARS),
+});
+// NOTE: divergenceAlertSchema must come first because valueAlertSchema has no type field.
+// If AlertThreshold gains a `type` field in the future, switch to z.discriminatedUnion('type', [...]).
+const alertThresholdUnionSchema = z.union([divergenceAlertSchema, valueAlertSchema]);
+export const divergenceDetailsSchema = z.object({
+    entropy: finiteNumber,
+    entropyThreshold: finiteNumber,
+    bins: z.int().positive(),
+    histogram: z.array(z.int().nonnegative()),
+});
+export const severityContextSchema = z.object({
+    environment: z.string().optional(),
+    escalateWarnings: z.boolean().optional(),
+    suppressInfo: z.boolean().optional(),
+    minSampleCount: z.int().nonnegative().optional(),
+});
+export const metricTrendSchema = z.object({
+    direction: z.enum(['improving', 'degrading', 'stable']),
+    delta: finiteNumber,
+    percentChange: finiteNumber.nullable(),
+    previousValue: finiteNumber,
+    currentValue: finiteNumber,
+    aggregation: evaluationAggregationSchema,
+    lowSampleWarning: z.boolean().optional(),
+});
+export const confidenceIndicatorSchema = z.object({
+    level: z.enum(['low', 'medium', 'high']),
+    sampleCount: z.int().nonnegative(),
+    scoreStdDev: finiteNumber.nullable(),
+    evaluatorCount: z.int().nonnegative(),
+    evaluatorAgreement: finiteNumber.nullable(),
+});
+export const worstExplanationSchema = z.object({
+    score: finiteNumber,
+    explanation: z.string().optional(),
+    traceId: z.string().optional(),
+    timestamp: z.string().optional(),
+});
+export const triggeredAlertSchema = z.object({
+    severity: alertSeveritySchema,
+    message: z.string(),
+    aggregation: evaluationAggregationSchema,
+    threshold: finiteNumber,
+    actualValue: finiteNumber,
+    direction: thresholdDirectionSchema,
+    affectedCount: z.int().nonnegative().optional(),
+    remediationHints: z.array(z.string().max(SCHEMA_MESSAGE_MAX_CHARS)).max(SCHEMA_MAX_REMEDIATION_HINTS).optional(),
+    relatedMetrics: z.array(z.string()).optional(),
+    isCompound: z.boolean().optional(),
+    divergenceDetails: divergenceDetailsSchema.optional(),
+});
+export const qualityMetricResultSchema = z.object({
+    name: z.string().min(1),
+    displayName: z.string().min(1),
+    values: z.record(evaluationAggregationSchema, finiteNumber.nullable()),
+    sampleCount: z.int().nonnegative(),
+    alerts: z.array(triggeredAlertSchema),
+    status: healthStatusSchema,
+    period: timeRangeSchema.optional(),
+    worstExplanation: worstExplanationSchema.optional(),
+    trend: metricTrendSchema.optional(),
+    confidence: confidenceIndicatorSchema.optional(),
+});
+export const qualityMetricConfigSchema = z.object({
+    name: z.string().min(1).max(SCHEMA_SHORT_TEXT_MAX),
+    displayName: z.string().min(1).max(SCHEMA_DISPLAY_NAME_MAX_CHARS),
+    description: z.string().max(SCHEMA_LONG_TEXT_MAX),
+    aggregations: z.array(evaluationAggregationSchema).min(1),
+    alerts: z.array(alertThresholdUnionSchema),
+    range: z.object({
+        min: finiteNumber,
+        max: finiteNumber,
+    }),
+    unit: z.enum(['score', 'rate', 'seconds', 'percentage']),
+    remediationHints: z.array(z.string().max(SCHEMA_MESSAGE_MAX_CHARS)).max(SCHEMA_MAX_REMEDIATION_HINTS).optional(),
+});
+export const metricConditionSchema = z.object({
+    metric: z.string().min(1).max(SCHEMA_SHORT_TEXT_MAX),
+    aggregation: evaluationAggregationSchema,
+    value: finiteNumber,
+    direction: thresholdDirectionSchema,
+    window: slaWindowFormatSchema.optional(),
+    sustainedPeriods: z.int().min(1).max(SCHEMA_MAX_SUSTAINED_PERIODS).optional(),
+});
+export const metricCorrelationRuleSchema = z.object({
+    name: z.string().min(1).max(SCHEMA_SHORT_TEXT_MAX),
+    displayName: z.string().min(1).max(SCHEMA_DISPLAY_NAME_MAX_CHARS),
+    conditions: z.array(metricConditionSchema).min(SCHEMA_CONDITIONS_MIN).max(SCHEMA_CONDITIONS_MAX),
+    severity: alertSeveritySchema,
+    explanation: z.string().min(1).max(SCHEMA_LONG_TEXT_MAX),
+});
+export const slaDefinitionSchema = z.object({
+    metric: z.string().min(1).max(SCHEMA_SHORT_TEXT_MAX),
+    aggregation: evaluationAggregationSchema,
+    target: finiteNumber,
+    direction: thresholdDirectionSchema,
+    window: slaWindowFormatSchema.optional(),
+});
+export const slaComplianceResultSchema = z.object({
+    sla: slaDefinitionSchema,
+    compliant: z.boolean(),
+    status: slaStatusSchema,
+    actualValue: finiteNumber.nullable(),
+    gap: finiteNumber.nullable(),
+    marginPercent: finiteNumber.nullable(),
+});
+/** TriggeredAlert with its source metric name — used in dashboard alerts and view schemas. */
+export const alertWithMetricSchema = triggeredAlertSchema.extend({ metricName: z.string() });
+/** Per-metric count breakdown shown in ExecutiveView summary. */
+export const dashboardSummaryStatsSchema = z.object({
+    totalMetrics: z.int().nonnegative(),
+    healthyMetrics: z.int().nonnegative(),
+    warningMetrics: z.int().nonnegative(),
+    criticalMetrics: z.int().nonnegative(),
+    noDataMetrics: z.int().nonnegative(),
+});
+export const qualityDashboardSummarySchema = z.object({
+    overallStatus: healthStatusSchema,
+    metrics: z.array(qualityMetricResultSchema),
+    alerts: z.array(alertWithMetricSchema),
+    summary: dashboardSummaryStatsSchema,
+    slaCompliance: z.array(slaComplianceResultSchema).optional(),
+    timestamp: z.string(),
+});
+// ============================================================================
+// Shared Constants
+// ============================================================================
+/** Severity ordering for alert sorting (critical first). */
+const SEVERITY_ORDER = { critical: 0, warning: 1, info: 2 };
+/** @internal Decimal precision for score values (e.g., 0.8500) */
+export const SCORE_PRECISION = 4;
+/** @internal Decimal precision for percentage values (e.g., 12.50%) */
+export const PERCENT_PRECISION = 2;
+/** @internal Round a number to the given decimal places */
+export function roundTo(value, decimals) {
+    const factor = DECIMAL_BASE ** decimals;
+    return Math.round(value * factor) / factor;
+}
+/** @internal Sort alerts by severity (critical first, then warning, then info). Returns new array. */
+export function sortAlertsBySeverity(alerts) {
+    return [...alerts].sort((a, b) => SEVERITY_ORDER[a.severity] - SEVERITY_ORDER[b.severity]);
+}
+// ============================================================================
+// Computation Functions
+// ============================================================================
+/**
+ * Calculate percentile from a sorted array using R-7 linear interpolation.
+ * @param sorted - Pre-sorted ascending array of numbers
+ * @param percentile - Percentile (0-100)
+ */
+function calculatePercentile(sorted, percentile) {
+    if (sorted.length === 0)
+        return NaN;
+    if (percentile < 0 || percentile > QUALITY_PERCENT_SCALE)
+        return NaN;
+    if (sorted.length === 1)
+        return sorted[0];
+    const rank = (percentile / QUALITY_PERCENT_SCALE) * (sorted.length - 1);
+    const lower = Math.floor(rank);
+    const upper = Math.ceil(rank);
+    const weight = rank - lower;
+    if (lower === upper)
+        return sorted[lower];
+    return sorted[lower] * (1 - weight) + sorted[upper] * weight;
+}
+/**
+ * Compute aggregations for a set of evaluation scores.
+ * @param scores - Array of score values
+ * @param aggregations - Aggregations to compute
+ */
+export function computeAggregations(scores, aggregations) {
+    const result = {
+        avg: null,
+        min: null,
+        max: null,
+        count: null,
+        p50: null,
+        p95: null,
+        p99: null,
+    };
+    // Filter out NaN/Infinity before processing
+    const validScores = scores.filter((v) => Number.isFinite(v));
+    if (validScores.length === 0) {
+        return result;
+    }
+    // Always compute count (of valid scores)
+    result.count = validScores.length;
+    // Lazy sort: only copy+sort when aggregations need ordered data
+    const needsSort = aggregations.some(a => a === 'min' || a === 'max' || a === 'p50' || a === 'p95' || a === 'p99');
+    const sorted = needsSort ? [...validScores].sort((a, b) => a - b) : [];
+    for (const agg of aggregations) {
+        switch (agg) {
+            case 'avg':
+                result.avg = validScores.reduce((sum, v) => sum + v, 0) / validScores.length;
+                break;
+            case 'min':
+                result.min = sorted.length > 0 ? sorted[0] : null;
+                break;
+            case 'max':
+                result.max = sorted.length > 0 ? sorted[sorted.length - 1] : null;
+                break;
+            case 'count':
+                // Already computed
+                break;
+            case 'p50':
+                result.p50 = calculatePercentile(sorted, QUALITY_PERCENTILE_P50);
+                break;
+            case 'p95':
+                result.p95 = calculatePercentile(sorted, QUALITY_PERCENTILE_P95);
+                break;
+            case 'p99':
+                result.p99 = calculatePercentile(sorted, QUALITY_PERCENTILE_P99);
+                break;
+        }
+    }
+    // Round to consistent score precision
+    for (const key of Object.keys(result)) {
+        const value = result[key];
+        if (value !== null && key !== 'count') {
+            result[key] = roundTo(value, SCORE_PRECISION);
+        }
+    }
+    return result;
+}
+/**
+ * Check a divergence alert threshold against raw scores.
+ * Returns a triggered alert if the score distribution entropy is below the threshold,
+ * or null if entropy is acceptable or there are insufficient samples.
+ *
+ * @param scores - Raw score values
+ * @param threshold - Divergence alert threshold configuration
+ */
+export function checkDivergenceAlert(scores, threshold) {
+    // Filter to finite scores in [0, 1] — public API must be self-defensive
+    const validScores = scores.filter(s => Number.isFinite(s) && s >= 0 && s <= 1);
+    if (validScores.length < MIN_DIVERGENCE_SAMPLES)
+        return null;
+    const histogram = new Array(threshold.bins).fill(0);
+    for (const s of validScores) {
+        const bin = Math.min(Math.floor(s * threshold.bins), threshold.bins - 1);
+        histogram[bin]++;
+    }
+    const entropy = computeNormalizedEntropy(histogram);
+    if (entropy >= threshold.entropyThreshold)
+        return null;
+    return {
+        severity: threshold.severity,
+        message: threshold.message.replace('{entropy}', entropy.toFixed(SCORE_PRECISION)),
+        aggregation: 'count',
+        threshold: threshold.entropyThreshold,
+        actualValue: entropy,
+        direction: 'below',
+        affectedCount: validScores.length,
+        divergenceDetails: {
+            entropy,
+            entropyThreshold: threshold.entropyThreshold,
+            bins: threshold.bins,
+            histogram,
+        },
+    };
+}
+/**
+ * Check alert thresholds and return triggered alerts.
+ * Supports both value-based (AlertThreshold) and divergence-based (DivergenceAlertThreshold) alerts.
+ *
+ * @param values - Computed aggregation values
+ * @param thresholds - Alert thresholds to check (value and/or divergence)
+ * @param options - Optional enrichment data; `scores` required for divergence alerts
+ */
+export function checkAlertThresholds(values, thresholds, options) {
+    const triggered = [];
+    for (const threshold of thresholds) {
+        // Divergence alert — requires raw scores
+        if ('type' in threshold && threshold.type === 'divergence') {
+            if (options?.scores) {
+                const alert = checkDivergenceAlert(options.scores, threshold);
+                if (alert)
+                    triggered.push(alert);
+            }
+            continue;
+        }
+        // Value alert (existing behavior) — narrowed from union after divergence guard
+        const vt = threshold;
+        const value = values[vt.aggregation];
+        if (value === null)
+            continue;
+        // Strict inequality (> / <) for alerts — SLA uses >= for boundary compliance.
+        // This is intentional: alerts fire when value crosses threshold, SLA checks if target is met.
+        const isTriggered = vt.direction === 'above'
+            ? value > vt.value
+            : value < vt.value;
+        if (isTriggered) {
+            const count = options?.sampleCount ?? values.count;
+            const affectedCount = typeof count === 'number' ? count : undefined;
+            triggered.push({
+                severity: vt.severity,
+                message: vt.message.replace('{value}', value.toFixed(SCORE_PRECISION)) +
+                    (affectedCount != null ? ` (n=${affectedCount} evaluations)` : ''),
+                aggregation: vt.aggregation,
+                threshold: vt.value,
+                actualValue: value,
+                direction: vt.direction,
+                affectedCount,
+                remediationHints: options?.remediationHints ? [...options.remediationHints] : undefined,
+            });
+        }
+    }
+    // Sort by severity (critical > warning > info)
+    return sortAlertsBySeverity(triggered);
+}
+/**
+ * Determine overall health status from triggered alerts.
+ */
+export function determineHealthStatus(alerts, hasData) {
+    if (!hasData)
+        return 'no_data';
+    if (alerts.some(a => a.severity === 'critical'))
+        return 'critical';
+    if (alerts.some(a => a.severity === 'warning'))
+        return 'warning';
+    return 'healthy';
+}
+/**
+ * Compute a single quality metric from evaluation results.
+ * @param evaluations - Evaluation results filtered by metric name
+ * @param config - Metric configuration
+ * @param period - Optional time period
+ * @param previousValues - Optional previous period aggregation values for trend computation
+ */
+export function computeQualityMetric(evaluations, config, period, previousValues) {
+    // Extract scored evaluations, filtering null/undefined and non-finite values (NaN, Infinity)
+    const scored = evaluations
+        .filter((e) => e.scoreValue !== undefined && e.scoreValue !== null);
+    const scores = scored.map(e => e.scoreValue).filter((v) => Number.isFinite(v));
+    // Compute aggregations
+    const values = computeAggregations(scores, config.aggregations);
+    // Check alerts (pass raw scores for divergence detection)
+    const alerts = checkAlertThresholds(values, config.alerts, {
+        sampleCount: scores.length,
+        remediationHints: config.remediationHints,
+        scores,
+    });
+    // Determine status
+    const status = determineHealthStatus(alerts, scores.length > 0);
+    // Find worst-performing evaluation for drill-down
+    let worstExplanation;
+    if (scored.length > 0) {
+        const worst = scored.reduce((min, e) => e.scoreValue < min.scoreValue ? e : min);
+        worstExplanation = {
+            score: worst.scoreValue,
+            explanation: worst.explanation,
+            traceId: worst.traceId,
+            timestamp: worst.timestamp,
+        };
+    }
+    // Compute trend if previous period data is available
+    const trend = previousValues ? computeTrend(values, previousValues, config) : undefined;
+    // Compute confidence indicator from evaluation data
+    const confidence = scored.length > 0 ? computeConfidence(evaluations, undefined, config.range) : undefined;
+    return {
+        name: config.name,
+        displayName: config.displayName,
+        values,
+        sampleCount: scores.length,
+        alerts,
+        status,
+        period,
+        worstExplanation,
+        trend,
+        confidence,
+    };
+}
+/**
+ * Compute dashboard summary for all quality metrics.
+ * @param evaluationsByMetric - Map of metric name to evaluation results
+ * @param customMetrics - Optional custom metric configurations to include
+ * @param period - Optional time period
+ * @param slas - Optional SLA definitions to evaluate
+ * @param previousPeriodValues - Optional previous period aggregation values per metric for trend computation
+ */
+export function computeDashboardSummary(evaluationsByMetric, customMetrics, period, slas, previousPeriodValues) {
+    const allMetrics = { ...QUALITY_METRICS, ...customMetrics };
+    const results = [];
+    const allAlerts = [];
+    for (const [name, config] of Object.entries(allMetrics)) {
+        const evaluations = evaluationsByMetric.get(name) || [];
+        const prevValues = previousPeriodValues?.get(name);
+        const result = computeQualityMetric(evaluations, config, period, prevValues);
+        results.push(result);
+        // Collect alerts with metric name
+        for (const alert of result.alerts) {
+            allAlerts.push({ ...alert, metricName: name });
+        }
+    }
+    // Evaluate correlation rules for compound alerts
+    const correlationRules = getCorrelationRules();
+    const compoundAlerts = evaluateCorrelationRules(results, correlationRules);
+    allAlerts.push(...compoundAlerts);
+    // Post-process: populate relatedMetrics on single-metric alerts
+    const alertingMetrics = [...new Set(allAlerts.map(a => a.metricName))];
+    for (const alert of allAlerts) {
+        if (!alert.isCompound) {
+            alert.relatedMetrics = alertingMetrics.filter(m => m !== alert.metricName);
+        }
+    }
+    // Compute summary counts (single pass instead of 4 filter passes)
+    const summary = { totalMetrics: results.length, healthyMetrics: 0, warningMetrics: 0, criticalMetrics: 0, noDataMetrics: 0 };
+    for (const r of results) {
+        if (r.status === 'healthy')
+            summary.healthyMetrics++;
+        else if (r.status === 'warning')
+            summary.warningMetrics++;
+        else if (r.status === 'critical')
+            summary.criticalMetrics++;
+        else if (r.status === 'no_data')
+            summary.noDataMetrics++;
+        else {
+            const _exhaustive = r.status;
+            throw new Error(`Unhandled status: ${_exhaustive}`);
+        }
+    }
+    // Determine overall status (compound alerts can upgrade severity)
+    let overallStatus = 'healthy';
+    const hasCriticalCompound = compoundAlerts.some(a => a.severity === 'critical');
+    const hasWarningCompound = compoundAlerts.some(a => a.severity === 'warning');
+    if (summary.criticalMetrics > 0 || hasCriticalCompound) {
+        overallStatus = 'critical';
+    }
+    else if (summary.warningMetrics > 0 || hasWarningCompound) {
+        overallStatus = 'warning';
+    }
+    else if (summary.noDataMetrics === summary.totalMetrics) {
+        overallStatus = 'no_data';
+    }
+    // Evaluate SLAs if definitions provided
+    const slaEval = slas?.length ? evaluateSLAs(slas, results) : undefined;
+    const slaCompliance = slaEval?.results;
+    return {
+        overallStatus,
+        metrics: results,
+        alerts: allAlerts,
+        summary,
+        slaCompliance,
+        timestamp: new Date().toISOString(),
+    };
+}
+// ============================================================================
+// Metric Registration
+// ============================================================================
+/**
+ * Registry for custom quality metrics.
+ */
+const customMetricRegistry = new Map();
+/**
+ * Register a custom quality metric.
+ * @param config - Metric configuration
+ * @throws If metric with same name already exists
+ */
+export function registerQualityMetric(config) {
+    // Validate config
+    qualityMetricConfigSchema.parse(config);
+    if ((config.name in QUALITY_METRICS) || customMetricRegistry.has(config.name)) {
+        throw new Error(`Quality metric '${config.name}' already exists`);
+    }
+    customMetricRegistry.set(config.name, config);
+}
+/**
+ * Check if a quality metric exists (built-in or custom).
+ * @param name - Metric name
+ * @returns true if metric exists
+ */
+export function hasQualityMetric(name) {
+    return name in QUALITY_METRICS || customMetricRegistry.has(name);
+}
+/**
+ * Unregister a custom quality metric.
+ * @param name - Metric name
+ * @returns true if metric was removed
+ */
+export function unregisterQualityMetric(name) {
+    return customMetricRegistry.delete(name);
+}
+/**
+ * Get all registered quality metrics (built-in + custom).
+ * @returns Combined record of built-in and custom metric configs
+ */
+export function getAllQualityMetrics() {
+    return {
+        ...QUALITY_METRICS,
+        ...Object.fromEntries(customMetricRegistry),
+    };
+}
+/**
+ * Get a specific quality metric configuration.
+ * @param name - Metric name
+ * @returns Metric config if found, undefined otherwise
+ */
+export function getQualityMetric(name) {
+    return (name in QUALITY_METRICS ? QUALITY_METRICS[name] : undefined) || customMetricRegistry.get(name);
+}
+/**
+ * Clear all custom metrics (for testing).
+ * @internal
+ */
+export function clearCustomMetrics() {
+    customMetricRegistry.clear();
+}
+// ============================================================================
+// Correlation Rule Registry
+// ============================================================================
+const correlationRuleRegistry = new Map();
+/**
+ * Default correlation rules matching the 3 patterns from the UX review.
+ */
+const _defaultRules = [
+    {
+        name: 'content_quality_crisis',
+        displayName: 'Content Quality Crisis',
+        conditions: [
+            { metric: 'relevance', aggregation: 'p50', value: RELEVANCE_WARNING, direction: 'below' },
+            { metric: 'hallucination', aggregation: 'avg', value: HALLUCINATION_WARNING, direction: 'above' },
+        ],
+        severity: 'critical',
+        explanation: 'Low relevance combined with high hallucination indicates responses are both off-topic and factually unreliable. This combination warrants immediate investigation of prompt quality and grounding context.',
+    },
+    {
+        name: 'agent_reliability_failure',
+        displayName: 'Agent Reliability Failure',
+        conditions: [
+            { metric: 'task_completion', aggregation: 'avg', value: TASK_COMPLETION_WARNING, direction: 'below' },
+            { metric: 'tool_correctness', aggregation: 'avg', value: TOOL_CORRECTNESS_RELIABILITY_FLOOR, direction: 'below' },
+        ],
+        severity: 'critical',
+        explanation: 'Low task completion with poor tool selection indicates the agent is both choosing wrong tools and failing to complete tasks. Check tool descriptions and agent planning prompts.',
+    },
+    {
+        name: 'response_degradation',
+        displayName: 'Response Degradation',
+        conditions: [
+            { metric: 'faithfulness', aggregation: 'p50', value: FAITHFULNESS_WARNING, direction: 'below' },
+            { metric: 'coherence', aggregation: 'p50', value: COHERENCE_WARNING, direction: 'below' },
+        ],
+        severity: 'warning',
+        explanation: 'Below-threshold faithfulness and coherence together suggest responses are both ungrounded and poorly structured. Review context retrieval pipeline and response formatting.',
+    },
+];
+/** Immutable default correlation rules. */
+export const DEFAULT_CORRELATION_RULES = Object.freeze(_defaultRules);
+/**
+ * Register a correlation rule.
+ * @throws If rule with same name exists or validation fails
+ */
+export function registerCorrelationRule(rule) {
+    metricCorrelationRuleSchema.parse(rule);
+    if (correlationRuleRegistry.has(rule.name) ||
+        DEFAULT_CORRELATION_RULES.some(r => r.name === rule.name)) {
+        throw new Error(`Correlation rule '${rule.name}' already exists`);
+    }
+    correlationRuleRegistry.set(rule.name, rule);
+}
+/**
+ * Unregister a custom correlation rule.
+ */
+export function unregisterCorrelationRule(name) {
+    return correlationRuleRegistry.delete(name);
+}
+/**
+ * Get all correlation rules (defaults + custom).
+ */
+export function getCorrelationRules() {
+    return [...DEFAULT_CORRELATION_RULES, ...correlationRuleRegistry.values()];
+}
+/**
+ * Clear all custom correlation rules (for testing).
+ * @internal
+ */
+export function clearCorrelationRules() {
+    correlationRuleRegistry.clear();
+}
+/**
+ * Check whether a single condition is met against a set of metric results.
+ * @internal Extracted to avoid duplication between standard and temporal evaluation paths.
+ */
+function checkConditionValue(cond, resultsByName) {
+    const metricResult = resultsByName.get(cond.metric);
+    if (!metricResult)
+        return false;
+    const value = metricResult.values[cond.aggregation];
+    if (value == null || !Number.isFinite(value))
+        return false;
+    return cond.direction === 'above' ? value > cond.value : value < cond.value;
+}
+/**
+ * Format a single correlation condition for display in a compound alert message.
+ * Resolves the value from the temporal source (window or sustained) when applicable.
+ * @internal
+ */
+function formatConditionSummaryEntry(cond, resultsByName, previousByWindow) {
+    let source = resultsByName;
+    let suffix = '';
+    if (cond.sustainedPeriods !== undefined) {
+        suffix = `[sustained ${cond.sustainedPeriods}p]`;
+    }
+    else if (cond.window) {
+        const windowSource = previousByWindow.get(cond.window);
+        if (windowSource) {
+            source = windowSource;
+            suffix = `[${cond.window}]`;
+        }
+    }
+    const value = source.get(cond.metric)?.values[cond.aggregation];
+    return `${cond.metric}.${cond.aggregation}${suffix}=${value?.toFixed(SCORE_PRECISION)}`;
+}
+/**
+ * Evaluate correlation rules against computed metric results.
+ * Returns compound alerts for rules where all conditions match.
+ *
+ * When `previousResults` is provided, conditions with temporal fields are evaluated:
+ * - `window`: condition checks the metric value from the previous-period result whose
+ *   window label matches the condition's window (falls back to current results).
+ * - `sustainedPeriods`: condition must hold in N consecutive previous result sets
+ *   (most recent first). If fewer than N periods are available, the condition is not met.
+ *
+ * @param results - Current period metric results
+ * @param rules - Correlation rules to evaluate
+ * @param previousResults - Optional ordered array of previous-period results (most recent first).
+ *   Each entry is tagged with a `window` label indicating its time range.
+ */
+export function evaluateCorrelationRules(results, rules, previousResults) {
+    const compoundAlerts = [];
+    const resultsByName = new Map(results.map(r => [r.name, r]));
+    // Pre-index previous results by window label
+    const previousByWindow = new Map((previousResults ?? []).map(p => [p.window, new Map(p.results.map(r => [r.name, r]))]));
+    for (const rule of rules) {
+        const allMatch = rule.conditions.every(cond => {
+            // --- Sustained periods check (Zod enforces min(1); > 0 guard handles direct construction) ---
+            if (cond.sustainedPeriods !== undefined) {
+                if (!previousResults || previousResults.length < cond.sustainedPeriods)
+                    return false;
+                // Current period must match
+                if (!checkConditionValue(cond, resultsByName))
+                    return false;
+                // Check N most-recent previous periods
+                let sustained = 0;
+                for (const prev of previousResults) {
+                    const prevByName = previousByWindow.get(prev.window);
+                    if (!prevByName || !checkConditionValue(cond, prevByName))
+                        break;
+                    sustained++;
+                    if (sustained >= cond.sustainedPeriods)
+                        break;
+                }
+                return sustained >= cond.sustainedPeriods;
+            }
+            // --- Window check ---
+            if (cond.window) {
+                const windowResults = previousByWindow.get(cond.window);
+                if (windowResults) {
+                    return checkConditionValue(cond, windowResults);
+                }
+                // Fall back to current results if window not found
+            }
+            // --- Standard (non-temporal) check ---
+            return checkConditionValue(cond, resultsByName);
+        });
+        if (allMatch) {
+            const conditionSummary = rule.conditions
+                .map(c => formatConditionSummaryEntry(c, resultsByName, previousByWindow))
+                .join(', ');
+            // Compound alerts use the first condition's aggregation/direction for interface compliance.
+            // Actual condition details are in the message and relatedMetrics fields.
+            const firstCond = rule.conditions[0];
+            const firstValue = resultsByName.get(firstCond.metric)?.values[firstCond.aggregation] ?? 0;
+            compoundAlerts.push({
+                severity: rule.severity,
+                message: `${rule.displayName}: ${conditionSummary}`,
+                aggregation: firstCond.aggregation,
+                threshold: firstCond.value,
+                actualValue: firstValue,
+                direction: firstCond.direction,
+                isCompound: true,
+                remediationHints: [rule.explanation],
+                relatedMetrics: rule.conditions.map(c => c.metric),
+                metricName: rule.name,
+            });
+        }
+    }
+    return compoundAlerts;
+}
+// ============================================================================
+// Submodule Barrel Re-exports
+// ============================================================================
+export { evaluateHandoffs, computeTurnLevelResults, computeMultiAgentEvaluation } from './quality-multi-agent.js';
+export { computeRoleView, computeMetricDetail, toEvaluationDetail, scoreDistributionEntrySchema, evaluationSnapshotSchema, metricDetailFieldsSchema, evaluationDetailResultSchema, roleViewOptionsSchema } from './quality-views.js';
+export { applySeverityContext } from './quality-alerts.js';
+export { evaluateSLA, evaluateSLAs } from './quality-sla.js';
+export { computeConfidence, computeTrend } from './quality-trends.js';
+export { computePipelineView, computeCoverageHeatmap } from './quality-visualization.js';
+export { scoreColorBand, adaptiveScoreColorBand, empiricalCDF, inferScoreDirection, labelToOrdinal, ordinalToCategory, computeCQI, computeCQISensitivity, DEFAULT_CQI_WEIGHTS, computeMetricDynamics, computeCoverageWeightedConfidence, computeNormalizedEntropy, computeDegradationSignal, computeEWMA, computeMAD, detectEWMADrift, VARIANCE_INCREASE_THRESHOLD, EWMA_LAMBDA, computePearsonR, computeSpearmanR, computeCorrelationMatrix, pearsonPValue, benjaminiHochberg, getRoleFeatureConfig, roleFeatureConfigSchema, METRIC_SCALE_STRATEGY, } from './quality-feature-engineering.js';
+// ============================================================================
+// Utility Functions
+// ============================================================================
+/**
+ * Format metric value for display.
+ * @param value - Numeric value
+ * @param unit - Unit type
+ * @returns Formatted string (e.g., "85.0%", "1.23s", "0.8500")
+ */
+export function formatMetricValue(value, unit) {
+    if (value === null)
+        return 'N/A';
+    switch (unit) {
+        case 'percentage':
+        case 'rate':
+            return `${(value * QUALITY_PERCENT_SCALE).toFixed(1)}%`;
+        case 'seconds':
+            return `${value.toFixed(PERCENT_PRECISION)}s`;
+        case 'score':
+        default:
+            return value.toFixed(SCORE_PRECISION);
+    }
+}
+/**
+ * Create a custom metric configuration builder.
+ * @param name - Metric name
+ * @returns Fluent builder for constructing QualityMetricConfig
+ */
+export function createMetricConfig(name) {
+    return new MetricConfigBuilder(name);
+}
+/**
+ * Fluent builder for QualityMetricConfig.
+ */
+export class MetricConfigBuilder {
+    config;
+    constructor(name) {
+        this.config = {
+            name,
+            aggregations: ['avg', 'count'],
+            alerts: [],
+            range: UNIT_RANGE,
+            unit: 'score',
+        };
+    }
+    displayName(name) {
+        this.config.displayName = name;
+        return this;
+    }
+    description(desc) {
+        this.config.description = desc;
+        return this;
+    }
+    aggregations(...aggs) {
+        this.config.aggregations = aggs;
+        return this;
+    }
+    range(min, max) {
+        this.config.range = { min, max };
+        return this;
+    }
+    unit(unit) {
+        this.config.unit = unit;
+        return this;
+    }
+    alertBelow(aggregation, value, severity, message) {
+        (this.config.alerts ??= []).push({
+            aggregation,
+            value,
+            direction: 'below',
+            severity,
+            message: message || `${this.config.name} ${aggregation} ({value}) below ${value} threshold`,
+        });
+        return this;
+    }
+    remediationHints(...hints) {
+        this.config.remediationHints = hints;
+        return this;
+    }
+    alertAbove(aggregation, value, severity, message) {
+        (this.config.alerts ??= []).push({
+            aggregation,
+            value,
+            direction: 'above',
+            severity,
+            message: message || `${this.config.name} ${aggregation} ({value}) above ${value} threshold`,
+        });
+        return this;
+    }
+    divergenceAlert(entropyThreshold, severity, message, bins = DEFAULT_BIN_COUNT) {
+        (this.config.alerts ??= []).push({
+            type: 'divergence',
+            entropyThreshold,
+            bins,
+            severity,
+            message: message || `${this.config.name} score distribution is bimodal (entropy {entropy})`,
+        });
+        return this;
+    }
+    sla(aggregation, target, direction = 'above', window) {
+        if (!this.config.name) {
+            throw new Error('MetricConfigBuilder: name is required before defining an SLA');
+        }
+        return {
+            metric: this.config.name,
+            aggregation,
+            target,
+            direction,
+            window,
+        };
+    }
+    build() {
+        if (!this.config.name)
+            throw new Error('MetricConfigBuilder: name is required');
+        if (!this.config.aggregations || this.config.aggregations.length === 0) {
+            throw new Error('MetricConfigBuilder: at least one aggregation is required');
+        }
+        if (!this.config.range)
+            throw new Error('MetricConfigBuilder: range is required');
+        this.config.displayName ??= this.config.name;
+        this.config.description ??= `Quality metric: ${this.config.name}`;
+        return this.config;
+    }
+}
+//# sourceMappingURL=quality-metrics.js.map