@llm-dev-ops/agentics-cli 1.4.29 → 1.4.32

package/dist/bundled-agents/analytics-hub-agents/src/analytics/aggregation_engine.rs

@@ -0,0 +1,342 @@
+//! Time-Series Aggregation Engine
+//!
+//! High-performance aggregation of events into statistical measures across
+//! multiple time windows (1m, 5m, 15m, 1h, 6h, 1d, 1w, 1M).
+
+use crate::database::Database;
+use crate::models::metrics::{StatisticalMeasures, TimeWindow};
+use crate::schemas::events::AnalyticsEvent;
+use anyhow::{Context, Result};
+use chrono::{DateTime, Duration, Utc};
+use dashmap::DashMap;
+use std::collections::HashMap;
+use std::sync::Arc;
+use tokio::sync::RwLock;
+use tracing::{debug, info, instrument};
+
+/// Aggregation engine for time-series data
+pub struct AggregationEngine {
+    database: Arc<Database>,
+    // Metric name + window -> Aggregated data
+    aggregates: Arc<DashMap<AggregateKey, WindowedAggregates>>,
+    // Active time buckets
+    active_buckets: Arc<RwLock<HashMap<TimeWindow, DateTime<Utc>>>>,
+}
+
+#[derive(Debug, Clone, Hash, Eq, PartialEq)]
+struct AggregateKey {
+    metric_name: String,
+    window: TimeWindow,
+    tags_hash: u64,
+}
+
+impl AggregationEngine {
+    /// Create a new aggregation engine
+    pub fn new(database: Arc<Database>) -> Self {
+        Self {
+            database,
+            aggregates: Arc::new(DashMap::new()),
+            active_buckets: Arc::new(RwLock::new(HashMap::new())),
+        }
+    }
+
+    /// Process an event and update aggregations
+    #[instrument(skip(self, event))]
+    pub async fn process_event(&self, event: &AnalyticsEvent) -> Result<()> {
+        // Extract numeric metrics from the event
+        let metrics = self.extract_metrics(event)?;
+
+        for (metric_name, value) in metrics {
+            // Aggregate across all time windows
+            for window in Self::all_windows() {
+                self.update_aggregation(
+                    &metric_name,
+                    value,
+                    window,
+                    event.common.timestamp,
+                    &event.common.tags,
+                )
+                .await?;
+            }
+        }
+
+        Ok(())
+    }
+
+    /// Extract numeric metrics from an event
+    fn extract_metrics(&self, event: &AnalyticsEvent) -> Result<Vec<(String, f64)>> {
+        let mut metrics = Vec::new();
+
+        // This is a simplified extraction - in reality, you'd parse the payload
+        // based on event type and extract relevant metrics
+        if let Ok(value) = serde_json::to_value(&event.payload) {
+            if let Some(obj) = value.as_object() {
+                for (key, val) in obj {
+                    if let Some(num) = val.as_f64() {
+                        metrics.push((key.clone(), num));
+                    }
+                }
+            }
+        }
+
+        Ok(metrics)
+    }
+
+    /// Update aggregation for a specific metric and time window
+    async fn update_aggregation(
+        &self,
+        metric_name: &str,
+        value: f64,
+        window: TimeWindow,
+        timestamp: DateTime<Utc>,
+        tags: &HashMap<String, String>,
+    ) -> Result<()> {
+        let window_start = self.align_to_window(timestamp, window);
+        let tags_hash = self.hash_tags(tags);
+
+        let key = AggregateKey {
+            metric_name: metric_name.to_string(),
+            window,
+            tags_hash,
+        };
+
+        // Get or create windowed aggregates
+        let mut agg = self.aggregates
+            .entry(key.clone())
+            .or_insert_with(|| WindowedAggregates::new(window_start));
+
+        // Add value to current window
+        agg.add_value(value);
+
+        // Check if we need to flush this window
+        if timestamp >= agg.window_end() {
+            // Compute statistics and store
+            let measures = agg.compute_statistics();
+
+            // Store to database
+            let tags_json = serde_json::to_value(tags)?;
+            self.database
+                .store_aggregated_metric(
+                    metric_name,
+                    window,
+                    agg.window_start,
+                    &tags_json,
+                    &measures,
+                )
+                .await?;
+
+            debug!(
+                "Flushed {} window for {}: avg={:.2}, count={}",
+                window.as_str(),
+                metric_name,
+                measures.avg,
+                measures.count
+            );
+
+            // Create new window
+            *agg = WindowedAggregates::new(window_start);
+        }
+
+        Ok(())
+    }
+
+    /// Align timestamp to window boundary
+    fn align_to_window(&self, timestamp: DateTime<Utc>, window: TimeWindow) -> DateTime<Utc> {
+        let seconds = window.to_seconds() as i64;
+        let ts = timestamp.timestamp();
+        let aligned = (ts / seconds) * seconds;
+        DateTime::from_timestamp(aligned, 0).unwrap_or(timestamp)
+    }
+
+    /// Hash tags for deduplication
+    fn hash_tags(&self, tags: &HashMap<String, String>) -> u64 {
+        use std::collections::hash_map::DefaultHasher;
+        use std::hash::{Hash, Hasher};
+
+        let mut hasher = DefaultHasher::new();
+        let mut sorted: Vec<_> = tags.iter().collect();
+        sorted.sort_by_key(|(k, _)| *k);
+        sorted.hash(&mut hasher);
+        hasher.finish()
+    }
+
+    /// All supported time windows
+    fn all_windows() -> &'static [TimeWindow] {
+        &[
+            TimeWindow::OneMinute,
+            TimeWindow::FiveMinutes,
+            TimeWindow::FifteenMinutes,
+            TimeWindow::OneHour,
+            TimeWindow::SixHours,
+            TimeWindow::OneDay,
+            TimeWindow::OneWeek,
+            TimeWindow::OneMonth,
+        ]
+    }
+
+    /// Get aggregated statistics for a metric
+    #[instrument(skip(self))]
+    pub async fn get_aggregated_stats(
+        &self,
+        metric_name: &str,
+        window: TimeWindow,
+        start: DateTime<Utc>,
+        end: DateTime<Utc>,
+    ) -> Result<Vec<StatisticalMeasures>> {
+        let rows = self.database
+            .query_aggregated_metrics(metric_name, window, start, end)
+            .await?;
+
+        let stats: Vec<StatisticalMeasures> = rows
+            .into_iter()
+            .map(|row| StatisticalMeasures {
+                avg: row.avg,
+                min: row.min,
+                max: row.max,
+                p50: row.p50,
+                p95: row.p95,
+                p99: row.p99,
+                stddev: row.stddev,
+                count: row.count as u64,
+                sum: row.sum,
+            })
+            .collect();
+
+        Ok(stats)
+    }
+
+    /// Force flush all pending aggregates
+    pub async fn flush_all(&self) -> Result<usize> {
+        let mut flushed = 0;
+
+        for entry in self.aggregates.iter() {
+            let key = entry.key();
+            let agg = entry.value();
+
+            if agg.values.is_empty() {
+                continue;
+            }
+
+            let measures = agg.compute_statistics();
+            let tags_json = serde_json::json!({});
+
+            self.database
+                .store_aggregated_metric(
+                    &key.metric_name,
+                    key.window,
+                    agg.window_start,
+                    &tags_json,
+                    &measures,
+                )
+                .await?;
+
+            flushed += 1;
+        }
+
+        info!("Flushed {} pending aggregates", flushed);
+        Ok(flushed)
+    }
+}
+
+/// Windowed aggregates for a metric
+struct WindowedAggregates {
+    window_start: DateTime<Utc>,
+    window_duration: Duration,
+    values: Vec<f64>,
+}
+
+impl WindowedAggregates {
+    fn new(window_start: DateTime<Utc>) -> Self {
+        Self {
+            window_start,
+            window_duration: Duration::minutes(1),
+            values: Vec::new(),
+        }
+    }
+
+    fn add_value(&mut self, value: f64) {
+        self.values.push(value);
+    }
+
+    fn window_end(&self) -> DateTime<Utc> {
+        self.window_start + self.window_duration
+    }
+
+    fn compute_statistics(&self) -> StatisticalMeasures {
+        if self.values.is_empty() {
+            return StatisticalMeasures::default();
+        }
+
+        let mut sorted = self.values.clone();
+        sorted.sort_by(|a, b| a.partial_cmp(b).unwrap());
+
+        let count = sorted.len() as u64;
+        let sum: f64 = sorted.iter().sum();
+        let avg = sum / count as f64;
+
+        let min = sorted[0];
+        let max = sorted[sorted.len() - 1];
+
+        // Percentiles
+        let p50 = Self::percentile(&sorted, 50.0);
+        let p95 = Self::percentile(&sorted, 95.0);
+        let p99 = Self::percentile(&sorted, 99.0);
+
+        // Standard deviation
+        let variance: f64 = sorted
+            .iter()
+            .map(|v| {
+                let diff = v - avg;
+                diff * diff
+            })
+            .sum::<f64>() / count as f64;
+        let stddev = Some(variance.sqrt());
+
+        StatisticalMeasures {
+            avg,
+            min,
+            max,
+            p50,
+            p95,
+            p99,
+            stddev,
+            count,
+            sum,
+        }
+    }
+
+    fn percentile(sorted_values: &[f64], percentile: f64) -> f64 {
+        if sorted_values.is_empty() {
+            return 0.0;
+        }
+
+        let index = (percentile / 100.0 * (sorted_values.len() - 1) as f64) as usize;
+        sorted_values[index.min(sorted_values.len() - 1)]
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_percentile_calculation() {
+        let values = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0];
+        assert_eq!(WindowedAggregates::percentile(&values, 50.0), 5.0);
+        assert_eq!(WindowedAggregates::percentile(&values, 95.0), 10.0);
+    }
+
+    #[test]
+    fn test_statistical_measures() {
+        let mut agg = WindowedAggregates::new(Utc::now());
+        for i in 1..=10 {
+            agg.add_value(i as f64);
+        }
+
+        let stats = agg.compute_statistics();
+        assert_eq!(stats.count, 10);
+        assert_eq!(stats.min, 1.0);
+        assert_eq!(stats.max, 10.0);
+        assert_eq!(stats.avg, 5.5);
+    }
+}

package/dist/bundled-agents/analytics-hub-agents/src/analytics/anomaly.rs

@@ -0,0 +1,268 @@
+//! Anomaly Detection Module
+//!
+//! Statistical and machine learning-based anomaly detection.
+
+use anyhow::Result;
+use chrono::{DateTime, Utc};
+use dashmap::DashMap;
+use std::collections::VecDeque;
+use std::sync::Arc;
+use tracing::debug;
+
+use super::AnalyticsConfig;
+
+/// Anomaly detector
+pub struct AnomalyDetector {
+    config: Arc<AnalyticsConfig>,
+    // Metric name -> Historical data
+    baselines: Arc<DashMap<String, MetricBaseline>>,
+    // Detected anomalies
+    anomalies: Arc<DashMap<String, Vec<Anomaly>>>,
+}
+
+impl AnomalyDetector {
+    /// Create a new anomaly detector
+    pub async fn new(config: Arc<AnalyticsConfig>) -> Result<Self> {
+        Ok(Self {
+            config,
+            baselines: Arc::new(DashMap::new()),
+            anomalies: Arc::new(DashMap::new()),
+        })
+    }
+
+    /// Add a data point and check for anomalies
+    pub fn check_anomaly(
+        &self,
+        metric_name: &str,
+        value: f64,
+        timestamp: DateTime<Utc>,
+    ) -> Result<Option<Anomaly>> {
+        // Get or create baseline
+        let mut baseline = self
+            .baselines
+            .entry(metric_name.to_string())
+            .or_insert_with(|| MetricBaseline::new(100));
+
+        // Add value to baseline
+        baseline.add_value(value, timestamp);
+
+        // Check if we have enough data
+        if baseline.values.len() < 10 {
+            return Ok(None);
+        }
+
+        // Calculate statistics
+        let mean = baseline.calculate_mean();
+        let stddev = baseline.calculate_stddev(mean);
+
+        // Z-score method for anomaly detection
+        let z_score = (value - mean).abs() / stddev;
+        let threshold = self.get_threshold_for_sensitivity();
+
+        if z_score > threshold {
+            let anomaly = Anomaly {
+                metric_name: metric_name.to_string(),
+                timestamp,
+                value,
+                expected_value: mean,
+                deviation: z_score,
+                anomaly_type: self.classify_anomaly(value, mean, &baseline),
+                severity: self.calculate_severity(z_score),
+            };
+
+            debug!(
+                "Anomaly detected in {}: value={}, expected={}, z-score={}",
+                metric_name, value, mean, z_score
+            );
+
+            // Store anomaly
+            self.anomalies
+                .entry(metric_name.to_string())
+                .or_insert_with(Vec::new)
+                .push(anomaly.clone());
+
+            return Ok(Some(anomaly));
+        }
+
+        Ok(None)
+    }
+
+    /// Get threshold based on sensitivity configuration
+    fn get_threshold_for_sensitivity(&self) -> f64 {
+        // Convert sensitivity (0.0-1.0) to z-score threshold
+        // Higher sensitivity = lower threshold
+        let sensitivity = self.config.anomaly_sensitivity;
+        3.0 - (sensitivity * 2.0) // Range: 1.0 to 3.0
+    }
+
+    /// Classify type of anomaly
+    fn classify_anomaly(&self, value: f64, mean: f64, baseline: &MetricBaseline) -> AnomalyType {
+        if value > mean {
+            // Check if it's a spike (sudden increase)
+            if let Some(&last_value) = baseline.values.back() {
+                if value > last_value * 1.5 {
+                    return AnomalyType::Spike;
+                }
+            }
+            AnomalyType::HighValue
+        } else {
+            // Check if it's a drop (sudden decrease)
+            if let Some(&last_value) = baseline.values.back() {
+                if value < last_value * 0.5 {
+                    return AnomalyType::Drop;
+                }
+            }
+            AnomalyType::LowValue
+        }
+    }
+
+    /// Calculate anomaly severity
+    fn calculate_severity(&self, z_score: f64) -> AnomalySeverity {
+        match z_score {
+            z if z > 5.0 => AnomalySeverity::Critical,
+            z if z > 4.0 => AnomalySeverity::High,
+            z if z > 3.0 => AnomalySeverity::Medium,
+            _ => AnomalySeverity::Low,
+        }
+    }
+
+    /// Get recent anomalies for a metric
+    pub fn get_anomalies(&self, metric_name: &str, limit: usize) -> Vec<Anomaly> {
+        self.anomalies
+            .get(metric_name)
+            .map(|anomalies| {
+                anomalies
+                    .iter()
+                    .rev()
+                    .take(limit)
+                    .cloned()
+                    .collect()
+            })
+            .unwrap_or_default()
+    }
+
+    /// Get all anomalies across all metrics
+    pub fn get_all_anomalies(&self, limit: usize) -> Vec<Anomaly> {
+        let mut all_anomalies = Vec::new();
+
+        for entry in self.anomalies.iter() {
+            all_anomalies.extend(entry.value().iter().cloned());
+        }
+
+        // Sort by timestamp descending
+        all_anomalies.sort_by(|a, b| b.timestamp.cmp(&a.timestamp));
+        all_anomalies.truncate(limit);
+
+        all_anomalies
+    }
+
+    /// Reset baseline for a metric
+    pub fn reset_baseline(&self, metric_name: &str) {
+        self.baselines.remove(metric_name);
+    }
+
+    /// Get detector statistics
+    pub fn get_stats(&self) -> DetectorStats {
+        let total_anomalies = self
+            .anomalies
+            .iter()
+            .map(|entry| entry.value().len())
+            .sum();
+
+        DetectorStats {
+            total_metrics: self.baselines.len(),
+            total_anomalies,
+            active_baselines: self.baselines.len(),
+        }
+    }
+}
+
+/// Metric baseline for anomaly detection
+struct MetricBaseline {
+    values: VecDeque<f64>,
+    timestamps: VecDeque<DateTime<Utc>>,
+    max_size: usize,
+}
+
+impl MetricBaseline {
+    fn new(max_size: usize) -> Self {
+        Self {
+            values: VecDeque::with_capacity(max_size),
+            timestamps: VecDeque::with_capacity(max_size),
+            max_size,
+        }
+    }
+
+    fn add_value(&mut self, value: f64, timestamp: DateTime<Utc>) {
+        if self.values.len() >= self.max_size {
+            self.values.pop_front();
+            self.timestamps.pop_front();
+        }
+        self.values.push_back(value);
+        self.timestamps.push_back(timestamp);
+    }
+
+    fn calculate_mean(&self) -> f64 {
+        if self.values.is_empty() {
+            return 0.0;
+        }
+        self.values.iter().sum::<f64>() / self.values.len() as f64
+    }
+
+    fn calculate_stddev(&self, mean: f64) -> f64 {
+        if self.values.len() < 2 {
+            return 1.0; // Avoid division by zero
+        }
+
+        let variance = self
+            .values
+            .iter()
+            .map(|v| {
+                let diff = v - mean;
+                diff * diff
+            })
+            .sum::<f64>()
+            / (self.values.len() - 1) as f64;
+
+        variance.sqrt().max(0.0001) // Avoid zero stddev
+    }
+}
+
+/// Detected anomaly
+#[derive(Debug, Clone)]
+pub struct Anomaly {
+    pub metric_name: String,
+    pub timestamp: DateTime<Utc>,
+    pub value: f64,
+    pub expected_value: f64,
+    pub deviation: f64,
+    pub anomaly_type: AnomalyType,
+    pub severity: AnomalySeverity,
+}
+
+/// Type of anomaly
+#[derive(Debug, Clone, PartialEq)]
+pub enum AnomalyType {
+    Spike,
+    Drop,
+    HighValue,
+    LowValue,
+    Pattern,
+}
+
+/// Anomaly severity
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
+pub enum AnomalySeverity {
+    Low,
+    Medium,
+    High,
+    Critical,
+}
+
+/// Detector statistics
+#[derive(Debug, Clone)]
+pub struct DetectorStats {
+    pub total_metrics: usize,
+    pub total_anomalies: usize,
+    pub active_baselines: usize,
+}