@musashishao/agent-kit 1.6.0 → 1.6.2

package/.agent/skills/opentelemetry-expert/scripts/trace_analyzer.py

@@ -0,0 +1,351 @@
+#!/usr/bin/env python3
+"""
+Skill: opentelemetry-expert
+Script: trace_analyzer.py
+Purpose: Analyze trace spans for performance issues in AI applications
+Usage: python trace_analyzer.py <trace_file.json> [--output json|summary]
+       python trace_analyzer.py --stdin < trace.json
+Output: Performance analysis and recommendations
+
+This script analyzes:
+1. Span duration anomalies
+2. Token usage patterns
+3. Error rates
+4. Agent loop efficiency
+"""
+import os
+import sys
+import json
+import argparse
+from typing import Dict, List, Any, Optional
+from datetime import datetime
+from statistics import mean, median, stdev
+
+# Fix console encoding
+try:
+    sys.stdout.reconfigure(encoding='utf-8', errors='replace')
+    sys.stderr.reconfigure(encoding='utf-8', errors='replace')
+except AttributeError:
+    pass
+
+# ============================================================================
+#  ANALYSIS FUNCTIONS
+# ============================================================================
+
+def parse_trace_data(data: Any) -> List[Dict]:
+    """Parse trace data from various formats."""
+    spans = []
+    
+    if isinstance(data, list):
+        for item in data:
+            if isinstance(item, dict):
+                if "spans" in item:
+                    spans.extend(item["spans"])
+                elif "traceId" in item or "trace_id" in item:
+                    spans.append(item)
+    elif isinstance(data, dict):
+        if "spans" in data:
+            spans = data["spans"]
+        elif "data" in data and isinstance(data["data"], list):
+            for trace in data["data"]:
+                if "spans" in trace:
+                    spans.extend(trace["spans"])
+    
+    return spans
+
+
+def extract_span_info(span: Dict) -> Dict:
+    """Extract relevant information from a span."""
+    # Handle different attribute formats
+    attributes = span.get("attributes", {})
+    if isinstance(attributes, list):
+        attributes = {a.get("key"): a.get("value", {}).get("stringValue") or a.get("value", {}).get("intValue") for a in attributes}
+    
+    # Calculate duration
+    start_time = span.get("startTimeUnixNano", 0) or span.get("start_time", 0)
+    end_time = span.get("endTimeUnixNano", 0) or span.get("end_time", 0)
+    duration_ms = (end_time - start_time) / 1_000_000 if start_time and end_time else 0
+    
+    return {
+        "name": span.get("name", "unknown"),
+        "trace_id": span.get("traceId") or span.get("trace_id", ""),
+        "span_id": span.get("spanId") or span.get("span_id", ""),
+        "parent_id": span.get("parentSpanId") or span.get("parent_id", ""),
+        "duration_ms": duration_ms,
+        "status": span.get("status", {}).get("code", "OK"),
+        "attributes": attributes,
+        "is_llm": any(k.startswith("llm.") for k in attributes.keys()),
+        "is_agent": any(k.startswith("agent.") for k in attributes.keys()),
+        "model": attributes.get("llm.model") or attributes.get("llm.request.model", ""),
+        "tokens": int(attributes.get("llm.usage.total_tokens", 0) or 0),
+        "error": span.get("status", {}).get("code") == "ERROR",
+    }
+
+
+def analyze_llm_performance(spans: List[Dict]) -> Dict:
+    """Analyze LLM-related spans for performance issues."""
+    llm_spans = [s for s in spans if s["is_llm"]]
+    
+    if not llm_spans:
+        return {"status": "No LLM spans found"}
+    
+    # Group by model
+    by_model: Dict[str, List[Dict]] = {}
+    for span in llm_spans:
+        model = span["model"] or "unknown"
+        if model not in by_model:
+            by_model[model] = []
+        by_model[model].append(span)
+    
+    analysis = {
+        "total_calls": len(llm_spans),
+        "total_tokens": sum(s["tokens"] for s in llm_spans),
+        "total_duration_ms": sum(s["duration_ms"] for s in llm_spans),
+        "error_count": sum(1 for s in llm_spans if s["error"]),
+        "by_model": {}
+    }
+    
+    for model, model_spans in by_model.items():
+        durations = [s["duration_ms"] for s in model_spans if s["duration_ms"] > 0]
+        tokens = [s["tokens"] for s in model_spans if s["tokens"] > 0]
+        
+        analysis["by_model"][model] = {
+            "count": len(model_spans),
+            "avg_duration_ms": mean(durations) if durations else 0,
+            "p50_duration_ms": median(durations) if durations else 0,
+            "max_duration_ms": max(durations) if durations else 0,
+            "avg_tokens": mean(tokens) if tokens else 0,
+            "total_tokens": sum(tokens),
+            "errors": sum(1 for s in model_spans if s["error"]),
+        }
+    
+    # Find outliers (> 2 std deviations)
+    all_durations = [s["duration_ms"] for s in llm_spans if s["duration_ms"] > 0]
+    if len(all_durations) > 2:
+        avg_dur = mean(all_durations)
+        std_dur = stdev(all_durations)
+        threshold = avg_dur + 2 * std_dur
+        
+        analysis["outliers"] = [
+            {"name": s["name"], "duration_ms": s["duration_ms"], "tokens": s["tokens"]}
+            for s in llm_spans if s["duration_ms"] > threshold
+        ][:5]
+    
+    return analysis
+
+
+def analyze_agent_loops(spans: List[Dict]) -> Dict:
+    """Analyze agent loop patterns."""
+    agent_spans = [s for s in spans if s["is_agent"]]
+    
+    if not agent_spans:
+        return {"status": "No agent spans found"}
+    
+    # Find root agent runs
+    root_runs = [s for s in agent_spans if "agent.run" in s["name"] or s["name"].endswith(".run")]
+    
+    # Find step spans
+    step_spans = [s for s in agent_spans if "step" in s["name"].lower()]
+    tool_spans = [s for s in agent_spans if "tool" in s["name"].lower()]
+    
+    analysis = {
+        "total_runs": len(root_runs),
+        "total_steps": len(step_spans),
+        "total_tool_calls": len(tool_spans),
+        "avg_steps_per_run": len(step_spans) / len(root_runs) if root_runs else 0,
+        "tool_usage": {},
+    }
+    
+    # Analyze tool usage
+    for span in tool_spans:
+        tool_name = span["attributes"].get("agent.tool.name") or span["attributes"].get("agent.tool", "unknown")
+        if tool_name not in analysis["tool_usage"]:
+            analysis["tool_usage"][tool_name] = {"count": 0, "errors": 0, "total_duration_ms": 0}
+        
+        analysis["tool_usage"][tool_name]["count"] += 1
+        analysis["tool_usage"][tool_name]["total_duration_ms"] += span["duration_ms"]
+        if span["error"]:
+            analysis["tool_usage"][tool_name]["errors"] += 1
+    
+    # Calculate run durations
+    if root_runs:
+        run_durations = [s["duration_ms"] for s in root_runs if s["duration_ms"] > 0]
+        if run_durations:
+            analysis["avg_run_duration_ms"] = mean(run_durations)
+            analysis["max_run_duration_ms"] = max(run_durations)
+    
+    return analysis
+
+
+def find_bottlenecks(spans: List[Dict]) -> List[Dict]:
+    """Identify performance bottlenecks."""
+    bottlenecks = []
+    
+    # Find slowest spans (top 5)
+    sorted_by_duration = sorted(spans, key=lambda s: s["duration_ms"], reverse=True)[:5]
+    for span in sorted_by_duration:
+        if span["duration_ms"] > 1000:  # > 1 second
+            bottlenecks.append({
+                "type": "slow_span",
+                "name": span["name"],
+                "duration_ms": span["duration_ms"],
+                "recommendation": f"Optimize {span['name']} - taking {span['duration_ms']:.0f}ms"
+            })
+    
+    # Find high-token operations
+    high_token_spans = [s for s in spans if s["tokens"] > 5000]
+    for span in high_token_spans[:3]:
+        bottlenecks.append({
+            "type": "high_tokens",
+            "name": span["name"],
+            "tokens": span["tokens"],
+            "recommendation": f"Reduce token usage in {span['name']} ({span['tokens']} tokens)"
+        })
+    
+    # Find error clusters
+    error_spans = [s for s in spans if s["error"]]
+    if len(error_spans) > 3:
+        bottlenecks.append({
+            "type": "error_cluster",
+            "count": len(error_spans),
+            "recommendation": f"Investigate {len(error_spans)} errors - may indicate systematic issue"
+        })
+    
+    return bottlenecks
+
+
+def generate_recommendations(analysis: Dict) -> List[str]:
+    """Generate actionable recommendations based on analysis."""
+    recommendations = []
+    
+    llm = analysis.get("llm_analysis", {})
+    agent = analysis.get("agent_analysis", {})
+    bottlenecks = analysis.get("bottlenecks", [])
+    
+    # LLM recommendations
+    if llm.get("total_calls", 0) > 0:
+        error_rate = llm.get("error_count", 0) / llm["total_calls"]
+        if error_rate > 0.05:
+            recommendations.append(f"High LLM error rate ({error_rate:.1%}) - implement retry logic")
+        
+        for model, stats in llm.get("by_model", {}).items():
+            if stats.get("avg_duration_ms", 0) > 5000:
+                recommendations.append(f"Consider caching for {model} - avg latency {stats['avg_duration_ms']:.0f}ms")
+    
+    # Agent recommendations
+    if agent.get("avg_steps_per_run", 0) > 10:
+        recommendations.append(f"Agent averaging {agent['avg_steps_per_run']:.1f} steps - consider optimizing reasoning")
+    
+    for tool, stats in agent.get("tool_usage", {}).items():
+        if stats["errors"] > stats["count"] * 0.2:
+            recommendations.append(f"Tool '{tool}' has {stats['errors']}/{stats['count']} errors - fix reliability")
+    
+    # Bottleneck recommendations
+    for bn in bottlenecks[:3]:
+        if bn.get("recommendation"):
+            recommendations.append(bn["recommendation"])
+    
+    return recommendations
+
+
+# ============================================================================
+#  MAIN
+# ============================================================================
+
+def analyze_traces(trace_data: Any) -> Dict[str, Any]:
+    """Main analysis function."""
+    spans_raw = parse_trace_data(trace_data)
+    
+    if not spans_raw:
+        return {"error": "No spans found in trace data"}
+    
+    spans = [extract_span_info(s) for s in spans_raw]
+    
+    analysis = {
+        "timestamp": datetime.now().isoformat(),
+        "total_spans": len(spans),
+        "llm_analysis": analyze_llm_performance(spans),
+        "agent_analysis": analyze_agent_loops(spans),
+        "bottlenecks": find_bottlenecks(spans),
+    }
+    
+    analysis["recommendations"] = generate_recommendations(analysis)
+    
+    # Overall status
+    error_count = sum(1 for s in spans if s["error"])
+    if error_count > len(spans) * 0.1:
+        analysis["status"] = f"[!!] HIGH ERROR RATE: {error_count}/{len(spans)} spans failed"
+    elif analysis["bottlenecks"]:
+        analysis["status"] = f"[!] PERFORMANCE ISSUES: {len(analysis['bottlenecks'])} bottlenecks found"
+    else:
+        analysis["status"] = "[OK] Trace analysis complete"
+    
+    return analysis
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Analyze trace spans for AI application performance"
+    )
+    parser.add_argument("trace_file", nargs="?", help="Path to trace JSON file")
+    parser.add_argument("--stdin", action="store_true", help="Read trace data from stdin")
+    parser.add_argument("--output", choices=["json", "summary"], default="json",
+                        help="Output format")
+    
+    args = parser.parse_args()
+    
+    # Read trace data
+    if args.stdin:
+        trace_data = json.load(sys.stdin)
+    elif args.trace_file:
+        if not os.path.isfile(args.trace_file):
+            print(json.dumps({"error": f"File not found: {args.trace_file}"}))
+            sys.exit(1)
+        with open(args.trace_file) as f:
+            trace_data = json.load(f)
+    else:
+        print(json.dumps({"error": "Provide trace file or use --stdin"}))
+        sys.exit(1)
+    
+    results = analyze_traces(trace_data)
+    
+    if args.output == "summary":
+        print(f"\n{'='*60}")
+        print(f"Trace Analysis")
+        print(f"{'='*60}")
+        print(f"Status: {results['status']}")
+        print(f"Total Spans: {results['total_spans']}")
+        
+        llm = results.get("llm_analysis", {})
+        if llm.get("total_calls"):
+            print(f"\nLLM Performance:")
+            print(f"  Total Calls: {llm['total_calls']}")
+            print(f"  Total Tokens: {llm['total_tokens']}")
+            print(f"  Errors: {llm['error_count']}")
+            for model, stats in llm.get("by_model", {}).items():
+                print(f"  {model}: {stats['count']} calls, avg {stats['avg_duration_ms']:.0f}ms")
+        
+        agent = results.get("agent_analysis", {})
+        if agent.get("total_runs"):
+            print(f"\nAgent Performance:")
+            print(f"  Total Runs: {agent['total_runs']}")
+            print(f"  Avg Steps/Run: {agent['avg_steps_per_run']:.1f}")
+            print(f"  Tool Calls: {agent['total_tool_calls']}")
+        
+        if results.get("bottlenecks"):
+            print(f"\nBottlenecks:")
+            for bn in results["bottlenecks"][:5]:
+                print(f"  • {bn['type']}: {bn.get('name', bn.get('count', 'n/a'))}")
+        
+        if results.get("recommendations"):
+            print(f"\nRecommendations:")
+            for rec in results["recommendations"]:
+                print(f"  → {rec}")
+        
+        print(f"{'='*60}\n")
+    else:
+        print(json.dumps(results, indent=2))
+
+
+if __name__ == "__main__":
+    main()