@llm-dev-ops/agentics-cli 1.4.27 → 1.4.29

package/dist/bundled-agents/latency-lens-agents/src/benchmarks/markdown.rs

@@ -1,334 +0,0 @@
-//! Markdown summary generation for benchmark results
-//!
-//! This module provides utilities for generating human-readable markdown
-//! summaries of benchmark results.
-
-use chrono::{DateTime, Utc};
-use std::fmt::Write as FmtWrite;
-
-use super::result::BenchmarkResult;
-
-/// Generate a markdown summary from benchmark results
-pub fn generate_summary(results: &[BenchmarkResult]) -> String {
-    let mut md = String::new();
-
-    // Header
-    writeln!(md, "# LLM Latency Lens - Benchmark Summary").unwrap();
-    writeln!(md).unwrap();
-    writeln!(
-        md,
-        "Generated: {}",
-        Utc::now().format("%Y-%m-%d %H:%M:%S UTC")
-    )
-    .unwrap();
-    writeln!(md).unwrap();
-
-    if results.is_empty() {
-        writeln!(md, "No benchmark results available.").unwrap();
-        return md;
-    }
-
-    // Overview section
-    writeln!(md, "## Overview").unwrap();
-    writeln!(md).unwrap();
-    writeln!(md, "| Target | Status | Success Rate | TTFT (mean) | Throughput |").unwrap();
-    writeln!(md, "|--------|--------|--------------|-------------|------------|").unwrap();
-
-    for result in results {
-        let status = if result.is_success() { "Pass" } else { "Fail" };
-        let success_rate = result
-            .success_rate()
-            .map(|r| format!("{:.1}%", r))
-            .unwrap_or_else(|| "N/A".to_string());
-
-        let ttft_mean = result
-            .get_metric("ttft_distribution.mean")
-            .and_then(|v| v.as_u64())
-            .map(|ns| format!("{:.2}ms", ns as f64 / 1_000_000.0))
-            .unwrap_or_else(|| "N/A".to_string());
-
-        let throughput = result
-            .get_metric("throughput.mean_tokens_per_second")
-            .and_then(|v| v.as_f64())
-            .map(|t| format!("{:.1} tok/s", t))
-            .unwrap_or_else(|| "N/A".to_string());
-
-        writeln!(
-            md,
-            "| {} | {} | {} | {} | {} |",
-            result.target_id(),
-            status,
-            success_rate,
-            ttft_mean,
-            throughput
-        )
-        .unwrap();
-    }
-
-    writeln!(md).unwrap();
-
-    // Detailed results section
-    writeln!(md, "## Detailed Results").unwrap();
-    writeln!(md).unwrap();
-
-    for result in results {
-        write_result_details(&mut md, result);
-    }
-
-    // Footer
-    writeln!(md, "---").unwrap();
-    writeln!(md).unwrap();
-    writeln!(
-        md,
-        "*Generated by [LLM Latency Lens](https://github.com/llm-devops/llm-latency-lens)*"
-    )
-    .unwrap();
-
-    md
-}
-
-/// Write detailed results for a single benchmark
-fn write_result_details(md: &mut String, result: &BenchmarkResult) {
-    writeln!(md, "### {}", result.target_id()).unwrap();
-    writeln!(md).unwrap();
-    writeln!(
-        md,
-        "**Timestamp:** {}",
-        result.timestamp().format("%Y-%m-%d %H:%M:%S UTC")
-    )
-    .unwrap();
-    writeln!(md).unwrap();
-
-    // Request summary
-    if let (Some(total), Some(successful), Some(failed)) = (
-        result.get_metric("total_requests").and_then(|v| v.as_u64()),
-        result
-            .get_metric("successful_requests")
-            .and_then(|v| v.as_u64()),
-        result
-            .get_metric("failed_requests")
-            .and_then(|v| v.as_u64()),
-    ) {
-        writeln!(md, "#### Request Summary").unwrap();
-        writeln!(md).unwrap();
-        writeln!(md, "| Metric | Value |").unwrap();
-        writeln!(md, "|--------|-------|").unwrap();
-        writeln!(md, "| Total Requests | {} |", total).unwrap();
-        writeln!(md, "| Successful | {} |", successful).unwrap();
-        writeln!(md, "| Failed | {} |", failed).unwrap();
-        if total > 0 {
-            writeln!(
-                md,
-                "| Success Rate | {:.1}% |",
-                (successful as f64 / total as f64) * 100.0
-            )
-            .unwrap();
-        }
-        writeln!(md).unwrap();
-    }
-
-    // TTFT distribution
-    write_latency_distribution(md, result, "ttft_distribution", "Time to First Token (TTFT)");
-
-    // Total latency distribution
-    write_latency_distribution(
-        md,
-        result,
-        "total_latency_distribution",
-        "Total Latency",
-    );
-
-    // Inter-token latency distribution
-    write_latency_distribution(
-        md,
-        result,
-        "inter_token_distribution",
-        "Inter-Token Latency",
-    );
-
-    // Throughput
-    if let Some(throughput) = result.get_metric("throughput") {
-        writeln!(md, "#### Throughput (tokens/sec)").unwrap();
-        writeln!(md).unwrap();
-        writeln!(md, "| Metric | Value |").unwrap();
-        writeln!(md, "|--------|-------|").unwrap();
-
-        if let Some(mean) = throughput.get("mean_tokens_per_second").and_then(|v| v.as_f64()) {
-            writeln!(md, "| Mean | {:.2} |", mean).unwrap();
-        }
-        if let Some(min) = throughput.get("min_tokens_per_second").and_then(|v| v.as_f64()) {
-            writeln!(md, "| Min | {:.2} |", min).unwrap();
-        }
-        if let Some(max) = throughput.get("max_tokens_per_second").and_then(|v| v.as_f64()) {
-            writeln!(md, "| Max | {:.2} |", max).unwrap();
-        }
-        if let Some(p50) = throughput.get("p50_tokens_per_second").and_then(|v| v.as_f64()) {
-            writeln!(md, "| P50 | {:.2} |", p50).unwrap();
-        }
-        if let Some(p95) = throughput.get("p95_tokens_per_second").and_then(|v| v.as_f64()) {
-            writeln!(md, "| P95 | {:.2} |", p95).unwrap();
-        }
-        writeln!(md).unwrap();
-    }
-
-    // Cost summary
-    if let Some(cost) = result.get_metric("total_cost_usd").and_then(|v| v.as_f64()) {
-        writeln!(md, "#### Cost").unwrap();
-        writeln!(md).unwrap();
-        writeln!(md, "- **Total Cost:** ${:.6}", cost).unwrap();
-        if let (Some(total), Some(_)) = (
-            result.get_metric("total_requests").and_then(|v| v.as_u64()),
-            result
-                .get_metric("successful_requests")
-                .and_then(|v| v.as_u64()),
-        ) {
-            if total > 0 {
-                writeln!(md, "- **Avg Cost/Request:** ${:.6}", cost / total as f64).unwrap();
-            }
-        }
-        writeln!(md).unwrap();
-    }
-
-    writeln!(md).unwrap();
-}
-
-/// Write a latency distribution table
-fn write_latency_distribution(
-    md: &mut String,
-    result: &BenchmarkResult,
-    key: &str,
-    title: &str,
-) {
-    if let Some(dist) = result.get_metric(key) {
-        writeln!(md, "#### {}", title).unwrap();
-        writeln!(md).unwrap();
-        writeln!(md, "| Percentile | Latency |").unwrap();
-        writeln!(md, "|------------|---------|").unwrap();
-
-        let format_latency = |ns: u64| -> String {
-            let ms = ns as f64 / 1_000_000.0;
-            if ms < 1.0 {
-                format!("{:.2}us", ns as f64 / 1000.0)
-            } else if ms < 1000.0 {
-                format!("{:.2}ms", ms)
-            } else {
-                format!("{:.2}s", ms / 1000.0)
-            }
-        };
-
-        if let Some(min) = dist.get("min").and_then(|v| v.as_u64()) {
-            writeln!(md, "| Min | {} |", format_latency(min)).unwrap();
-        }
-        if let Some(mean) = dist.get("mean").and_then(|v| v.as_u64()) {
-            writeln!(md, "| Mean | {} |", format_latency(mean)).unwrap();
-        }
-        if let Some(p50) = dist.get("p50").and_then(|v| v.as_u64()) {
-            writeln!(md, "| P50 (Median) | {} |", format_latency(p50)).unwrap();
-        }
-        if let Some(p90) = dist.get("p90").and_then(|v| v.as_u64()) {
-            writeln!(md, "| P90 | {} |", format_latency(p90)).unwrap();
-        }
-        if let Some(p95) = dist.get("p95").and_then(|v| v.as_u64()) {
-            writeln!(md, "| P95 | {} |", format_latency(p95)).unwrap();
-        }
-        if let Some(p99) = dist.get("p99").and_then(|v| v.as_u64()) {
-            writeln!(md, "| P99 | {} |", format_latency(p99)).unwrap();
-        }
-        if let Some(max) = dist.get("max").and_then(|v| v.as_u64()) {
-            writeln!(md, "| Max | {} |", format_latency(max)).unwrap();
-        }
-
-        writeln!(md).unwrap();
-    }
-}
-
-/// Generate a simple one-line summary for a benchmark result
-pub fn one_line_summary(result: &BenchmarkResult) -> String {
-    let status = if result.is_success() {
-        "PASS"
-    } else {
-        "FAIL"
-    };
-
-    let ttft = result
-        .get_metric("ttft_distribution.mean")
-        .and_then(|v| v.as_u64())
-        .map(|ns| format!("TTFT:{:.1}ms", ns as f64 / 1_000_000.0))
-        .unwrap_or_default();
-
-    let throughput = result
-        .get_metric("throughput.mean_tokens_per_second")
-        .and_then(|v| v.as_f64())
-        .map(|t| format!("Throughput:{:.1}tok/s", t))
-        .unwrap_or_default();
-
-    format!(
-        "[{}] {} - {} {} @ {}",
-        status,
-        result.target_id(),
-        ttft,
-        throughput,
-        result.timestamp().format("%H:%M:%S")
-    )
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use serde_json::json;
-
-    fn sample_result() -> BenchmarkResult {
-        BenchmarkResult::new(
-            "openai:gpt-4o",
-            json!({
-                "total_requests": 10,
-                "successful_requests": 9,
-                "failed_requests": 1,
-                "ttft_distribution": {
-                    "min": 100000000,
-                    "mean": 150000000,
-                    "p50": 140000000,
-                    "p90": 180000000,
-                    "p95": 200000000,
-                    "p99": 250000000,
-                    "max": 300000000
-                },
-                "throughput": {
-                    "mean_tokens_per_second": 50.5,
-                    "min_tokens_per_second": 40.0,
-                    "max_tokens_per_second": 65.0,
-                    "p50_tokens_per_second": 52.0,
-                    "p95_tokens_per_second": 60.0
-                },
-                "total_cost_usd": 0.05
-            }),
-        )
-    }
-
-    #[test]
-    fn test_generate_summary() {
-        let results = vec![sample_result()];
-        let summary = generate_summary(&results);
-
-        assert!(summary.contains("# LLM Latency Lens - Benchmark Summary"));
-        assert!(summary.contains("openai:gpt-4o"));
-        assert!(summary.contains("90.0%")); // success rate
-    }
-
-    #[test]
-    fn test_one_line_summary() {
-        let result = sample_result();
-        let line = one_line_summary(&result);
-
-        assert!(line.contains("[PASS]"));
-        assert!(line.contains("openai:gpt-4o"));
-        assert!(line.contains("TTFT:"));
-        assert!(line.contains("Throughput:"));
-    }
-
-    #[test]
-    fn test_empty_results() {
-        let summary = generate_summary(&[]);
-        assert!(summary.contains("No benchmark results available"));
-    }
-}

package/dist/bundled-agents/latency-lens-agents/src/benchmarks/mod.rs

@@ -1,245 +0,0 @@
-//! Canonical benchmarks module for LLM Latency Lens
-//!
-//! This module provides the standardized benchmark interface required
-//! across all 25 benchmark-target repositories, including:
-//! - `run_all_benchmarks()` entrypoint returning `Vec<BenchmarkResult>`
-//! - Standardized `BenchmarkResult` struct
-//! - Markdown summary generation
-//! - File I/O utilities for benchmark output
-
-pub mod io;
-pub mod markdown;
-pub mod result;
-
-pub use io::{
-    cleanup_old_results, ensure_output_dirs, output_dir, raw_output_dir, read_all_results,
-    read_result, read_summary, summary_path, write_result, write_results, write_summary,
-    DEFAULT_OUTPUT_DIR, RAW_OUTPUT_DIR, SUMMARY_FILENAME,
-};
-pub use markdown::{generate_summary, one_line_summary};
-pub use result::BenchmarkResult;
-
-use anyhow::{Context, Result};
-use std::sync::Arc;
-use tracing::info;
-
-use super::adapters::{all_targets, BenchTarget};
-use super::config::Config;
-
-/// Configuration for running all benchmarks
-#[derive(Debug, Clone)]
-pub struct BenchmarkConfig {
-    /// Number of requests per target
-    pub requests: u32,
-    /// Concurrency level
-    pub concurrency: u32,
-    /// Rate limit (requests per second, 0 = unlimited)
-    pub rate_limit: u32,
-    /// Warmup requests before actual benchmark
-    pub warmup: u32,
-    /// Maximum tokens to generate per request
-    pub max_tokens: u32,
-    /// Temperature for generation
-    pub temperature: f32,
-    /// Request timeout in seconds
-    pub timeout_secs: u64,
-    /// Show progress bars
-    pub show_progress: bool,
-    /// Default prompt to use for benchmarks
-    pub default_prompt: String,
-}
-
-impl Default for BenchmarkConfig {
-    fn default() -> Self {
-        Self {
-            requests: 10,
-            concurrency: 1,
-            rate_limit: 0,
-            warmup: 0,
-            max_tokens: 1024,
-            temperature: 0.7,
-            timeout_secs: 120,
-            show_progress: true,
-            default_prompt: "Explain the concept of machine learning in simple terms.".to_string(),
-        }
-    }
-}
-
-/// Run all benchmarks and return standardized results
-///
-/// This is the canonical entrypoint required for compatibility with
-/// the benchmark-target interface across all 25 repositories.
-///
-/// # Arguments
-///
-/// * `config` - Application configuration with provider credentials
-/// * `bench_config` - Benchmark-specific configuration
-///
-/// # Returns
-///
-/// A vector of `BenchmarkResult` containing metrics for each target
-pub async fn run_all_benchmarks(
-    config: Config,
-    bench_config: BenchmarkConfig,
-) -> Result<Vec<BenchmarkResult>> {
-    info!("Starting run_all_benchmarks with {} targets", all_targets(&config).len());
-
-    // Ensure output directories exist
-    ensure_output_dirs()?;
-
-    let shutdown_signal = Arc::new(tokio::sync::Notify::new());
-    let mut results = Vec::new();
-
-    // Get all configured benchmark targets
-    let targets = all_targets(&config);
-
-    if targets.is_empty() {
-        tracing::warn!("No benchmark targets configured");
-        return Ok(results);
-    }
-
-    info!("Running benchmarks for {} targets", targets.len());
-
-    for target in targets {
-        let target_id = target.id();
-        info!("Benchmarking target: {}", target_id);
-
-        match target
-            .run(
-                &config,
-                &bench_config,
-                Arc::clone(&shutdown_signal),
-            )
-            .await
-        {
-            Ok(result) => {
-                // Write result to file
-                if let Err(e) = write_result(&result) {
-                    tracing::warn!("Failed to write result for {}: {}", target_id, e);
-                }
-
-                info!(
-                    "Completed benchmark for {}: {}",
-                    target_id,
-                    one_line_summary(&result)
-                );
-                results.push(result);
-            }
-            Err(e) => {
-                tracing::error!("Benchmark failed for {}: {}", target_id, e);
-
-                // Create a failure result
-                let failure_result = BenchmarkResult::new(
-                    target_id.clone(),
-                    serde_json::json!({
-                        "error": e.to_string(),
-                        "total_requests": 0,
-                        "successful_requests": 0,
-                        "failed_requests": 0
-                    }),
-                );
-
-                if let Err(we) = write_result(&failure_result) {
-                    tracing::warn!("Failed to write failure result for {}: {}", target_id, we);
-                }
-
-                results.push(failure_result);
-            }
-        }
-    }
-
-    // Generate and write summary
-    let summary = generate_summary(&results);
-    write_summary(&summary).context("Failed to write summary")?;
-
-    info!(
-        "Completed all benchmarks: {}/{} successful",
-        results.iter().filter(|r| r.is_success()).count(),
-        results.len()
-    );
-
-    Ok(results)
-}
-
-/// Run benchmarks for specific targets only
-pub async fn run_benchmarks_for_targets(
-    config: Config,
-    bench_config: BenchmarkConfig,
-    target_ids: &[String],
-) -> Result<Vec<BenchmarkResult>> {
-    info!("Running benchmarks for {} specific targets", target_ids.len());
-
-    ensure_output_dirs()?;
-
-    let shutdown_signal = Arc::new(tokio::sync::Notify::new());
-    let mut results = Vec::new();
-
-    let all = all_targets(&config);
-
-    for target in all {
-        if target_ids.contains(&target.id()) {
-            let target_id = target.id();
-            info!("Benchmarking target: {}", target_id);
-
-            match target
-                .run(&config, &bench_config, Arc::clone(&shutdown_signal))
-                .await
-            {
-                Ok(result) => {
-                    if let Err(e) = write_result(&result) {
-                        tracing::warn!("Failed to write result for {}: {}", target_id, e);
-                    }
-                    results.push(result);
-                }
-                Err(e) => {
-                    tracing::error!("Benchmark failed for {}: {}", target_id, e);
-                    let failure_result = BenchmarkResult::new(
-                        target_id.clone(),
-                        serde_json::json!({
-                            "error": e.to_string(),
-                            "total_requests": 0,
-                            "successful_requests": 0,
-                            "failed_requests": 0
-                        }),
-                    );
-                    results.push(failure_result);
-                }
-            }
-        }
-    }
-
-    // Generate and write summary
-    let summary = generate_summary(&results);
-    write_summary(&summary)?;
-
-    Ok(results)
-}
-
-/// Get a summary of all available benchmark results
-pub fn get_results_summary() -> Result<String> {
-    let results = read_all_results()?;
-    Ok(generate_summary(&results))
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_benchmark_config_default() {
-        let config = BenchmarkConfig::default();
-
-        assert_eq!(config.requests, 10);
-        assert_eq!(config.concurrency, 1);
-        assert_eq!(config.rate_limit, 0);
-        assert_eq!(config.warmup, 0);
-        assert_eq!(config.max_tokens, 1024);
-        assert!(config.show_progress);
-    }
-
-    #[test]
-    fn test_ensure_output_dirs() {
-        // This test would need a temp directory setup
-        // Skipping actual filesystem operations in unit tests
-    }
-}