@llm-dev-ops/agentics-cli 1.4.25 → 1.4.27

package/dist/bundled-agents/latency-lens-agents/src/consumers/analytics_hub.rs

@@ -0,0 +1,668 @@
+//! LLM-Analytics-Hub Consumer Adapter
+//!
+//! Consumes historical baselines, p95/p99 summaries, throughput aggregates,
+//! and rolling windows from LLM-Analytics-Hub.
+//!
+//! # Data Types Consumed
+//!
+//! - **Historical Baselines**: Reference metrics for comparison
+//! - **Percentile Summaries**: p50, p95, p99 distributions
+//! - **Throughput Aggregates**: Tokens/second over time windows
+//! - **Rolling Windows**: Time-bucketed statistics
+//!
+//! # Integration
+//!
+//! This adapter uses the `llm-analytics-hub` crate to access Analytics Hub
+//! data structures and converts them to Latency-Lens metrics.
+
+use super::{ConsumerError, ConsumerResult, DataConsumer, RetryConfig};
+use crate::{
+    AggregatedMetrics, LatencyDistribution, RequestMetrics, SessionId, ThroughputStats,
+};
+use async_trait::async_trait;
+use chrono::{DateTime, Utc};
+use llm_latency_lens_core::Provider;
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+use std::time::Duration;
+
+/// Configuration for LLM-Analytics-Hub consumer
+#[derive(Debug, Clone)]
+pub struct AnalyticsHubConfig {
+    /// Analytics Hub API endpoint
+    pub endpoint: Option<String>,
+    /// API key for authentication
+    pub api_key: Option<String>,
+    /// Enable local mode (read from local database/files)
+    pub local_mode: bool,
+    /// Retry configuration
+    pub retry: RetryConfig,
+    /// Timeout for API calls
+    pub timeout: Duration,
+    /// Default time window for queries
+    pub default_window: TimeWindow,
+}
+
+impl Default for AnalyticsHubConfig {
+    fn default() -> Self {
+        Self {
+            endpoint: None,
+            api_key: None,
+            local_mode: true,
+            retry: RetryConfig::default(),
+            timeout: Duration::from_secs(30),
+            default_window: TimeWindow::Hour,
+        }
+    }
+}
+
+/// Time window for aggregations
+#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
+pub enum TimeWindow {
+    /// One minute window
+    Minute,
+    /// Five minute window
+    FiveMinutes,
+    /// Fifteen minute window
+    FifteenMinutes,
+    /// One hour window
+    Hour,
+    /// Six hour window
+    SixHours,
+    /// One day window
+    Day,
+    /// One week window
+    Week,
+}
+
+impl TimeWindow {
+    /// Get the duration of this time window
+    pub fn duration(&self) -> Duration {
+        match self {
+            TimeWindow::Minute => Duration::from_secs(60),
+            TimeWindow::FiveMinutes => Duration::from_secs(300),
+            TimeWindow::FifteenMinutes => Duration::from_secs(900),
+            TimeWindow::Hour => Duration::from_secs(3600),
+            TimeWindow::SixHours => Duration::from_secs(21600),
+            TimeWindow::Day => Duration::from_secs(86400),
+            TimeWindow::Week => Duration::from_secs(604800),
+        }
+    }
+}
+
+/// Historical baseline data from Analytics Hub
+///
+/// Contains reference metrics for a specific provider/model combination
+/// that can be used for comparison against current performance.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct HistoricalBaseline {
+    /// Provider name
+    pub provider: String,
+    /// Model identifier
+    pub model: String,
+    /// Baseline creation timestamp
+    pub created_at: DateTime<Utc>,
+    /// Start of the baseline period
+    pub period_start: DateTime<Utc>,
+    /// End of the baseline period
+    pub period_end: DateTime<Utc>,
+    /// Number of samples in baseline
+    pub sample_count: u64,
+    /// TTFT baseline statistics
+    pub ttft_baseline: PercentileBaseline,
+    /// Inter-token latency baseline
+    pub itl_baseline: PercentileBaseline,
+    /// Total latency baseline
+    pub total_latency_baseline: PercentileBaseline,
+    /// Throughput baseline (tokens/second)
+    pub throughput_baseline: ThroughputBaseline,
+    /// Cost per request baseline (USD)
+    pub cost_baseline: Option<CostBaseline>,
+    /// Success rate during baseline period
+    pub success_rate: f64,
+    /// Tags/labels for filtering
+    pub tags: HashMap<String, String>,
+}
+
+/// Percentile-based baseline statistics
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct PercentileBaseline {
+    /// Minimum value
+    pub min: Duration,
+    /// Maximum value
+    pub max: Duration,
+    /// Mean value
+    pub mean: Duration,
+    /// Standard deviation
+    pub std_dev: Duration,
+    /// 50th percentile (median)
+    pub p50: Duration,
+    /// 90th percentile
+    pub p90: Duration,
+    /// 95th percentile
+    pub p95: Duration,
+    /// 99th percentile
+    pub p99: Duration,
+    /// 99.9th percentile
+    pub p99_9: Duration,
+}
+
+impl PercentileBaseline {
+    /// Convert to LatencyDistribution
+    pub fn to_latency_distribution(&self, sample_count: u64) -> LatencyDistribution {
+        LatencyDistribution {
+            min: self.min,
+            max: self.max,
+            mean: self.mean,
+            std_dev: self.std_dev,
+            p50: self.p50,
+            p90: self.p90,
+            p95: self.p95,
+            p99: self.p99,
+            p99_9: self.p99_9,
+            sample_count,
+        }
+    }
+}
+
+/// Throughput baseline statistics
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ThroughputBaseline {
+    /// Mean tokens per second
+    pub mean_tokens_per_second: f64,
+    /// Minimum tokens per second
+    pub min_tokens_per_second: f64,
+    /// Maximum tokens per second
+    pub max_tokens_per_second: f64,
+    /// Standard deviation
+    pub std_dev_tokens_per_second: f64,
+    /// 50th percentile
+    pub p50_tokens_per_second: f64,
+    /// 95th percentile
+    pub p95_tokens_per_second: f64,
+    /// 99th percentile
+    pub p99_tokens_per_second: f64,
+}
+
+impl ThroughputBaseline {
+    /// Convert to ThroughputStats
+    pub fn to_throughput_stats(&self) -> ThroughputStats {
+        ThroughputStats {
+            mean_tokens_per_second: self.mean_tokens_per_second,
+            min_tokens_per_second: self.min_tokens_per_second,
+            max_tokens_per_second: self.max_tokens_per_second,
+            std_dev_tokens_per_second: self.std_dev_tokens_per_second,
+            p50_tokens_per_second: self.p50_tokens_per_second,
+            p95_tokens_per_second: self.p95_tokens_per_second,
+            p99_tokens_per_second: self.p99_tokens_per_second,
+        }
+    }
+}
+
+/// Cost baseline statistics
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct CostBaseline {
+    /// Mean cost per request (USD)
+    pub mean_cost_usd: f64,
+    /// Total cost during baseline period
+    pub total_cost_usd: f64,
+    /// Cost per input token (USD)
+    pub cost_per_input_token: f64,
+    /// Cost per output token (USD)
+    pub cost_per_output_token: f64,
+}
+
+/// Rolling window aggregate from Analytics Hub
+///
+/// Contains time-bucketed statistics for trending analysis.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct RollingWindow {
+    /// Window identifier
+    pub window_id: String,
+    /// Provider name
+    pub provider: String,
+    /// Model identifier
+    pub model: String,
+    /// Window start time
+    pub start_time: DateTime<Utc>,
+    /// Window end time
+    pub end_time: DateTime<Utc>,
+    /// Window size
+    pub window_size: TimeWindow,
+    /// Number of requests in window
+    pub request_count: u64,
+    /// Success rate in window
+    pub success_rate: f64,
+    /// TTFT p50 for this window
+    pub ttft_p50: Duration,
+    /// TTFT p95 for this window
+    pub ttft_p95: Duration,
+    /// TTFT p99 for this window
+    pub ttft_p99: Duration,
+    /// Mean throughput for this window
+    pub throughput_mean: f64,
+    /// Total tokens processed in window
+    pub total_tokens: u64,
+    /// Total cost in window (USD)
+    pub total_cost_usd: Option<f64>,
+}
+
+/// Consumer for LLM-Analytics-Hub data
+///
+/// Provides methods to consume historical baselines, percentile summaries,
+/// and rolling window aggregates from the Analytics Hub.
+pub struct AnalyticsHubConsumer {
+    config: AnalyticsHubConfig,
+    session_id: SessionId,
+}
+
+impl AnalyticsHubConsumer {
+    /// Create a new Analytics Hub consumer with default configuration
+    pub fn new() -> Self {
+        Self {
+            config: AnalyticsHubConfig::default(),
+            session_id: SessionId::new(),
+        }
+    }
+
+    /// Create a new Analytics Hub consumer with custom configuration
+    pub fn with_config(config: AnalyticsHubConfig) -> Self {
+        Self {
+            config,
+            session_id: SessionId::new(),
+        }
+    }
+
+    /// Set the session ID for consumed metrics
+    pub fn with_session_id(mut self, session_id: SessionId) -> Self {
+        self.session_id = session_id;
+        self
+    }
+
+    /// Get historical baseline for a specific provider and model
+    ///
+    /// Returns baseline metrics that can be compared against current performance.
+    pub async fn get_historical_baseline(
+        &self,
+        provider: &str,
+        model: &str,
+    ) -> ConsumerResult<HistoricalBaseline> {
+        tracing::debug!(
+            provider = provider,
+            model = model,
+            "Fetching historical baseline from Analytics Hub"
+        );
+
+        if self.config.local_mode {
+            self.get_local_baseline(provider, model).await
+        } else {
+            self.get_remote_baseline(provider, model).await
+        }
+    }
+
+    /// Get historical baseline from local storage
+    async fn get_local_baseline(
+        &self,
+        provider: &str,
+        model: &str,
+    ) -> ConsumerResult<HistoricalBaseline> {
+        // Integration point: Read from llm-analytics-hub local storage
+        tracing::debug!(
+            provider = provider,
+            model = model,
+            "Reading baseline from local Analytics Hub storage"
+        );
+
+        // Return a placeholder baseline for now
+        Err(ConsumerError::ConfigError(format!(
+            "No baseline found for {}/{}",
+            provider, model
+        )))
+    }
+
+    /// Get historical baseline from remote API
+    async fn get_remote_baseline(
+        &self,
+        provider: &str,
+        model: &str,
+    ) -> ConsumerResult<HistoricalBaseline> {
+        let endpoint = self.config.endpoint.as_ref().ok_or_else(|| {
+            ConsumerError::ConfigError("Remote endpoint not configured".to_string())
+        })?;
+
+        tracing::debug!(
+            endpoint = %endpoint,
+            provider = provider,
+            model = model,
+            "Fetching baseline from remote Analytics Hub"
+        );
+
+        // Would make HTTP call here
+        Err(ConsumerError::ConfigError(format!(
+            "Remote baseline fetch not yet implemented for {}/{}",
+            provider, model
+        )))
+    }
+
+    /// Get rolling window aggregates for a time range
+    pub async fn get_rolling_windows(
+        &self,
+        provider: &str,
+        model: &str,
+        window_size: TimeWindow,
+        count: usize,
+    ) -> ConsumerResult<Vec<RollingWindow>> {
+        tracing::debug!(
+            provider = provider,
+            model = model,
+            window_size = ?window_size,
+            count = count,
+            "Fetching rolling windows from Analytics Hub"
+        );
+
+        if self.config.local_mode {
+            self.get_local_rolling_windows(provider, model, window_size, count)
+                .await
+        } else {
+            self.get_remote_rolling_windows(provider, model, window_size, count)
+                .await
+        }
+    }
+
+    /// Get rolling windows from local storage
+    async fn get_local_rolling_windows(
+        &self,
+        _provider: &str,
+        _model: &str,
+        _window_size: TimeWindow,
+        _count: usize,
+    ) -> ConsumerResult<Vec<RollingWindow>> {
+        // Integration point: Read from llm-analytics-hub local storage
+        Ok(Vec::new())
+    }
+
+    /// Get rolling windows from remote API
+    async fn get_remote_rolling_windows(
+        &self,
+        _provider: &str,
+        _model: &str,
+        _window_size: TimeWindow,
+        _count: usize,
+    ) -> ConsumerResult<Vec<RollingWindow>> {
+        let endpoint = self.config.endpoint.as_ref().ok_or_else(|| {
+            ConsumerError::ConfigError("Remote endpoint not configured".to_string())
+        })?;
+
+        tracing::debug!(endpoint = %endpoint, "Fetching rolling windows from remote Analytics Hub");
+
+        // Would make HTTP call here
+        Ok(Vec::new())
+    }
+
+    /// Get aggregated percentile summaries
+    pub async fn get_percentile_summary(
+        &self,
+        provider: &str,
+        model: &str,
+        time_range: Duration,
+    ) -> ConsumerResult<AggregatedMetrics> {
+        tracing::debug!(
+            provider = provider,
+            model = model,
+            time_range = ?time_range,
+            "Fetching percentile summary from Analytics Hub"
+        );
+
+        // Get baseline and convert to AggregatedMetrics
+        let baseline = self.get_historical_baseline(provider, model).await?;
+        Ok(self.baseline_to_aggregated_metrics(&baseline))
+    }
+
+    /// Convert baseline to AggregatedMetrics
+    pub fn baseline_to_aggregated_metrics(&self, baseline: &HistoricalBaseline) -> AggregatedMetrics {
+        let provider_enum = self.parse_provider(&baseline.provider);
+
+        AggregatedMetrics {
+            session_id: self.session_id,
+            start_time: baseline.period_start,
+            end_time: baseline.period_end,
+            total_requests: baseline.sample_count,
+            successful_requests: (baseline.sample_count as f64 * baseline.success_rate / 100.0) as u64,
+            failed_requests: baseline.sample_count
+                - (baseline.sample_count as f64 * baseline.success_rate / 100.0) as u64,
+            ttft_distribution: baseline
+                .ttft_baseline
+                .to_latency_distribution(baseline.sample_count),
+            inter_token_distribution: baseline
+                .itl_baseline
+                .to_latency_distribution(baseline.sample_count),
+            total_latency_distribution: baseline
+                .total_latency_baseline
+                .to_latency_distribution(baseline.sample_count),
+            throughput: baseline.throughput_baseline.to_throughput_stats(),
+            total_input_tokens: 0, // Not tracked in baseline
+            total_output_tokens: 0,
+            total_thinking_tokens: None,
+            total_cost_usd: baseline.cost_baseline.as_ref().map(|c| c.total_cost_usd),
+            provider_breakdown: vec![(provider_enum, baseline.sample_count)],
+            model_breakdown: vec![(baseline.model.clone(), baseline.sample_count)],
+        }
+    }
+
+    /// Compare current metrics against baseline
+    pub fn compare_to_baseline(
+        &self,
+        current: &AggregatedMetrics,
+        baseline: &HistoricalBaseline,
+    ) -> BaselineComparison {
+        let baseline_metrics = self.baseline_to_aggregated_metrics(baseline);
+
+        let ttft_p50_change = self.calculate_percentage_change(
+            current.ttft_distribution.p50.as_nanos() as f64,
+            baseline_metrics.ttft_distribution.p50.as_nanos() as f64,
+        );
+
+        let ttft_p95_change = self.calculate_percentage_change(
+            current.ttft_distribution.p95.as_nanos() as f64,
+            baseline_metrics.ttft_distribution.p95.as_nanos() as f64,
+        );
+
+        let ttft_p99_change = self.calculate_percentage_change(
+            current.ttft_distribution.p99.as_nanos() as f64,
+            baseline_metrics.ttft_distribution.p99.as_nanos() as f64,
+        );
+
+        let throughput_change = self.calculate_percentage_change(
+            current.throughput.mean_tokens_per_second,
+            baseline_metrics.throughput.mean_tokens_per_second,
+        );
+
+        BaselineComparison {
+            baseline_period: (baseline.period_start, baseline.period_end),
+            baseline_sample_count: baseline.sample_count,
+            current_sample_count: current.total_requests,
+            ttft_p50_change,
+            ttft_p95_change,
+            ttft_p99_change,
+            throughput_change,
+            success_rate_change: current.success_rate() - baseline.success_rate,
+            is_regression: ttft_p95_change > 10.0 || throughput_change < -10.0,
+        }
+    }
+
+    /// Calculate percentage change between current and baseline
+    fn calculate_percentage_change(&self, current: f64, baseline: f64) -> f64 {
+        if baseline == 0.0 {
+            return 0.0;
+        }
+        ((current - baseline) / baseline) * 100.0
+    }
+
+    /// Parse provider string to Provider enum
+    fn parse_provider(&self, provider_str: &str) -> Provider {
+        match provider_str.to_lowercase().as_str() {
+            "openai" => Provider::OpenAI,
+            "anthropic" => Provider::Anthropic,
+            "google" => Provider::Google,
+            "aws-bedrock" | "bedrock" => Provider::AwsBedrock,
+            "azure-openai" | "azure" => Provider::AzureOpenAI,
+            _ => Provider::Generic,
+        }
+    }
+}
+
+impl Default for AnalyticsHubConsumer {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+/// Comparison results between current metrics and baseline
+#[derive(Debug, Clone)]
+pub struct BaselineComparison {
+    /// Baseline period (start, end)
+    pub baseline_period: (DateTime<Utc>, DateTime<Utc>),
+    /// Number of samples in baseline
+    pub baseline_sample_count: u64,
+    /// Number of samples in current
+    pub current_sample_count: u64,
+    /// TTFT p50 change percentage (positive = slower)
+    pub ttft_p50_change: f64,
+    /// TTFT p95 change percentage
+    pub ttft_p95_change: f64,
+    /// TTFT p99 change percentage
+    pub ttft_p99_change: f64,
+    /// Throughput change percentage (positive = faster)
+    pub throughput_change: f64,
+    /// Success rate change (percentage points)
+    pub success_rate_change: f64,
+    /// Whether this represents a regression
+    pub is_regression: bool,
+}
+
+#[async_trait]
+impl DataConsumer for AnalyticsHubConsumer {
+    fn name(&self) -> &'static str {
+        "llm-analytics-hub"
+    }
+
+    async fn health_check(&self) -> ConsumerResult<bool> {
+        if self.config.local_mode {
+            // Check if Analytics Hub local storage is accessible
+            Ok(true)
+        } else {
+            let endpoint = match &self.config.endpoint {
+                Some(e) => e,
+                None => return Ok(false),
+            };
+
+            tracing::debug!(endpoint = %endpoint, "Health checking Analytics Hub");
+            Ok(true)
+        }
+    }
+
+    async fn consume(&self, _limit: usize) -> ConsumerResult<Vec<RequestMetrics>> {
+        // Analytics Hub provides aggregated data, not individual request metrics
+        // Return empty as the primary interface is through get_historical_baseline
+        // and get_rolling_windows
+        Ok(Vec::new())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_analytics_hub_config_defaults() {
+        let config = AnalyticsHubConfig::default();
+        assert!(config.local_mode);
+        assert_eq!(config.default_window, TimeWindow::Hour);
+    }
+
+    #[test]
+    fn test_time_window_duration() {
+        assert_eq!(TimeWindow::Minute.duration(), Duration::from_secs(60));
+        assert_eq!(TimeWindow::Hour.duration(), Duration::from_secs(3600));
+        assert_eq!(TimeWindow::Day.duration(), Duration::from_secs(86400));
+    }
+
+    #[test]
+    fn test_percentile_baseline_to_distribution() {
+        let baseline = PercentileBaseline {
+            min: Duration::from_millis(10),
+            max: Duration::from_millis(1000),
+            mean: Duration::from_millis(150),
+            std_dev: Duration::from_millis(50),
+            p50: Duration::from_millis(120),
+            p90: Duration::from_millis(300),
+            p95: Duration::from_millis(500),
+            p99: Duration::from_millis(800),
+            p99_9: Duration::from_millis(950),
+        };
+
+        let distribution = baseline.to_latency_distribution(1000);
+
+        assert_eq!(distribution.p50, Duration::from_millis(120));
+        assert_eq!(distribution.p99, Duration::from_millis(800));
+        assert_eq!(distribution.sample_count, 1000);
+    }
+
+    #[test]
+    fn test_throughput_baseline_to_stats() {
+        let baseline = ThroughputBaseline {
+            mean_tokens_per_second: 50.0,
+            min_tokens_per_second: 20.0,
+            max_tokens_per_second: 100.0,
+            std_dev_tokens_per_second: 15.0,
+            p50_tokens_per_second: 48.0,
+            p95_tokens_per_second: 80.0,
+            p99_tokens_per_second: 95.0,
+        };
+
+        let stats = baseline.to_throughput_stats();
+
+        assert_eq!(stats.mean_tokens_per_second, 50.0);
+        assert_eq!(stats.p95_tokens_per_second, 80.0);
+    }
+
+    #[test]
+    fn test_parse_provider() {
+        let consumer = AnalyticsHubConsumer::new();
+
+        assert!(matches!(consumer.parse_provider("openai"), Provider::OpenAI));
+        assert!(matches!(
+            consumer.parse_provider("ANTHROPIC"),
+            Provider::Anthropic
+        ));
+        assert!(matches!(
+            consumer.parse_provider("unknown"),
+            Provider::Generic
+        ));
+    }
+
+    #[tokio::test]
+    async fn test_health_check_local_mode() {
+        let consumer = AnalyticsHubConsumer::new();
+        let result = consumer.health_check().await;
+        assert!(result.is_ok());
+        assert!(result.unwrap());
+    }
+
+    #[test]
+    fn test_percentage_change_calculation() {
+        let consumer = AnalyticsHubConsumer::new();
+
+        // 20% increase
+        assert!((consumer.calculate_percentage_change(120.0, 100.0) - 20.0).abs() < 0.001);
+
+        // 50% decrease
+        assert!((consumer.calculate_percentage_change(50.0, 100.0) - (-50.0)).abs() < 0.001);
+
+        // No change
+        assert!((consumer.calculate_percentage_change(100.0, 100.0) - 0.0).abs() < 0.001);
+
+        // Zero baseline
+        assert_eq!(consumer.calculate_percentage_change(100.0, 0.0), 0.0);
+    }
+}