@musashishao/agent-kit 1.6.1 → 1.7.0

package/.agent/skills/observability-patterns/scripts/otel_validator.py

@@ -0,0 +1,330 @@
+#!/usr/bin/env python3
+"""
+Skill: observability-patterns
+Script: otel_validator.py
+Purpose: Validate OpenTelemetry instrumentation in AI applications
+Usage: python otel_validator.py <project_path> [--output json|summary]
+Output: JSON with instrumentation findings
+
+This script checks for:
+1. OpenTelemetry SDK setup
+2. LLM call instrumentation
+3. Proper span attributes for AI
+4. Context propagation patterns
+"""
+import os
+import sys
+import re
+import json
+import argparse
+from pathlib import Path
+from typing import Dict, List, Any
+from datetime import datetime
+
+# Fix console encoding
+try:
+    sys.stdout.reconfigure(encoding='utf-8', errors='replace')
+    sys.stderr.reconfigure(encoding='utf-8', errors='replace')
+except AttributeError:
+    pass
+
+# ============================================================================
+#  CONFIGURATION
+# ============================================================================
+
+SKIP_DIRS = {'node_modules', '.git', 'dist', 'build', '__pycache__', '.venv', 'venv', '.next'}
+CODE_EXTENSIONS = {'.js', '.ts', '.jsx', '.tsx', '.py', '.go'}
+CONFIG_EXTENSIONS = {'.json', '.yaml', '.yml', '.toml'}
+
+# Patterns to check for good observability practices
+GOOD_PATTERNS = [
+    # OpenTelemetry setup
+    (r"@opentelemetry|opentelemetry-sdk|from opentelemetry", "otel_sdk", "OpenTelemetry SDK imported"),
+    (r"getTracer|trace\.get_tracer", "tracer_init", "Tracer initialized"),
+    (r"getMeter|metrics\.get_meter", "meter_init", "Meter initialized"),
+    
+    # Span creation
+    (r"startActiveSpan|start_as_current_span|start_span", "span_creation", "Spans being created"),
+    (r"span\.setAttributes?|span\.set_attribute", "span_attributes", "Span attributes set"),
+    (r"span\.recordException|record_exception", "error_recording", "Errors recorded in spans"),
+    
+    # AI-specific attributes
+    (r"llm\.|model|tokens|completion", "ai_attributes", "AI-related attributes found"),
+    (r"agent\.step|agent\.tool", "agent_tracing", "Agent tracing attributes"),
+    
+    # Metrics
+    (r"createCounter|create_counter|Counter\(", "counter_metric", "Counter metrics defined"),
+    (r"createHistogram|create_histogram|Histogram\(", "histogram_metric", "Histogram metrics defined"),
+    
+    # Context propagation
+    (r"propagation\.inject|inject_context", "context_inject", "Context injection found"),
+    (r"propagation\.extract|extract_context", "context_extract", "Context extraction found"),
+    
+    # Logging with trace context
+    (r"trace_id|span_id|traceId|spanId", "log_correlation", "Log-trace correlation"),
+]
+
+# Patterns that indicate missing observability
+MISSING_PATTERNS = [
+    # LLM calls without tracing
+    (r"openai\.|anthropic\.|llm\.|completion", "llm_call", "LLM call found"),
+    (r"fetch|axios|requests\.", "http_call", "HTTP call found"),
+]
+
+# Check if LLM/HTTP calls are within spans
+SPAN_CONTEXT_PATTERN = r"(startActiveSpan|start_as_current_span|with_span).*?(openai|anthropic|llm|fetch|axios|requests)"
+
+
+# ============================================================================
+#  SCANNING FUNCTIONS
+# ============================================================================
+
+def check_package_deps(project_path: Path) -> Dict[str, Any]:
+    """Check package.json or requirements.txt for OTel dependencies."""
+    result = {
+        "has_otel": False,
+        "packages": [],
+        "missing": []
+    }
+    
+    # Check package.json
+    pkg_json = project_path / "package.json"
+    if pkg_json.exists():
+        try:
+            with open(pkg_json) as f:
+                pkg = json.load(f)
+                all_deps = {**pkg.get("dependencies", {}), **pkg.get("devDependencies", {})}
+                
+                otel_packages = [k for k in all_deps if "opentelemetry" in k.lower()]
+                if otel_packages:
+                    result["has_otel"] = True
+                    result["packages"] = otel_packages
+                
+                # Check for recommended packages
+                recommended = ["@opentelemetry/sdk-node", "@opentelemetry/api"]
+                result["missing"] = [r for r in recommended if r not in all_deps]
+        except Exception:
+            pass
+    
+    # Check requirements.txt
+    req_txt = project_path / "requirements.txt"
+    if req_txt.exists():
+        try:
+            with open(req_txt) as f:
+                content = f.read().lower()
+                if "opentelemetry" in content:
+                    result["has_otel"] = True
+                    result["packages"].append("opentelemetry (Python)")
+        except Exception:
+            pass
+    
+    return result
+
+
+def scan_file_for_patterns(filepath: Path, patterns: List[tuple]) -> Dict[str, List[Dict]]:
+    """Scan file for specific patterns."""
+    findings = {}
+    
+    try:
+        with open(filepath, 'r', encoding='utf-8', errors='ignore') as f:
+            content = f.read()
+            lines = content.split('\n')
+            
+            for pattern, key, description in patterns:
+                matches = list(re.finditer(pattern, content, re.IGNORECASE | re.DOTALL))
+                if matches:
+                    if key not in findings:
+                        findings[key] = []
+                    
+                    for match in matches[:3]:  # Limit to 3 per pattern
+                        # Find line number
+                        line_num = content[:match.start()].count('\n') + 1
+                        findings[key].append({
+                            "line": line_num,
+                            "snippet": lines[line_num - 1].strip()[:80] if line_num <= len(lines) else "",
+                            "description": description
+                        })
+    except Exception:
+        pass
+    
+    return findings
+
+
+def analyze_instrumentation(project_path: str) -> Dict[str, Any]:
+    """Analyze project for observability instrumentation."""
+    project = Path(project_path)
+    
+    results = {
+        "project": project_path,
+        "timestamp": datetime.now().isoformat(),
+        "dependencies": check_package_deps(project),
+        "instrumentation": {
+            "good_practices": {},
+            "potential_gaps": [],
+            "files_analyzed": 0
+        },
+        "recommendations": [],
+        "score": 0
+    }
+    
+    all_good_findings = {}
+    llm_calls_without_traces = []
+    
+    for root, dirs, files in os.walk(project):
+        dirs[:] = [d for d in dirs if d not in SKIP_DIRS]
+        
+        for file in files:
+            ext = Path(file).suffix.lower()
+            if ext not in CODE_EXTENSIONS:
+                continue
+            
+            filepath = Path(root) / file
+            results["instrumentation"]["files_analyzed"] += 1
+            
+            # Check for good patterns
+            file_findings = scan_file_for_patterns(filepath, GOOD_PATTERNS)
+            for key, findings in file_findings.items():
+                if key not in all_good_findings:
+                    all_good_findings[key] = []
+                for f in findings:
+                    f["file"] = str(filepath.relative_to(project))
+                    all_good_findings[key].append(f)
+            
+            # Check for potential gaps (LLM calls without spans)
+            try:
+                with open(filepath, 'r', encoding='utf-8', errors='ignore') as f:
+                    content = f.read()
+                    lines = content.split('\n')
+                    
+                    # Find LLM/API calls
+                    for pattern, key, desc in MISSING_PATTERNS:
+                        for match in re.finditer(pattern, content, re.IGNORECASE):
+                            line_num = content[:match.start()].count('\n') + 1
+                            
+                            # Check if there's a span context nearby (within 10 lines)
+                            context_start = max(0, line_num - 10)
+                            context_end = min(len(lines), line_num + 5)
+                            context = '\n'.join(lines[context_start:context_end])
+                            
+                            has_span = bool(re.search(r"startActiveSpan|start_as_current_span|with_span|tracer\.", context, re.IGNORECASE))
+                            
+                            if not has_span and key == "llm_call":
+                                llm_calls_without_traces.append({
+                                    "file": str(filepath.relative_to(project)),
+                                    "line": line_num,
+                                    "snippet": lines[line_num - 1].strip()[:80]
+                                })
+            except Exception:
+                pass
+    
+    results["instrumentation"]["good_practices"] = all_good_findings
+    results["instrumentation"]["potential_gaps"] = llm_calls_without_traces[:10]  # Limit output
+    
+    # Calculate score
+    score = 0
+    max_score = 100
+    
+    if results["dependencies"]["has_otel"]:
+        score += 20
+    if all_good_findings.get("tracer_init"):
+        score += 15
+    if all_good_findings.get("span_creation"):
+        score += 15
+    if all_good_findings.get("span_attributes"):
+        score += 10
+    if all_good_findings.get("ai_attributes"):
+        score += 10
+    if all_good_findings.get("counter_metric") or all_good_findings.get("histogram_metric"):
+        score += 10
+    if all_good_findings.get("error_recording"):
+        score += 10
+    if all_good_findings.get("log_correlation"):
+        score += 5
+    if all_good_findings.get("context_inject") or all_good_findings.get("context_extract"):
+        score += 5
+    
+    # Deduct for gaps
+    if len(llm_calls_without_traces) > 0:
+        score -= min(20, len(llm_calls_without_traces) * 5)
+    
+    results["score"] = max(0, score)
+    
+    # Generate recommendations
+    if not results["dependencies"]["has_otel"]:
+        results["recommendations"].append("Install OpenTelemetry SDK: npm install @opentelemetry/sdk-node")
+    if not all_good_findings.get("tracer_init"):
+        results["recommendations"].append("Initialize a tracer: const tracer = trace.getTracer('service-name')")
+    if not all_good_findings.get("span_creation"):
+        results["recommendations"].append("Add spans to track operations: tracer.startActiveSpan('operation', ...)")
+    if not all_good_findings.get("ai_attributes"):
+        results["recommendations"].append("Add AI-specific attributes: llm.model, llm.tokens, agent.step")
+    if llm_calls_without_traces:
+        results["recommendations"].append(f"Wrap {len(llm_calls_without_traces)} LLM calls in spans for tracing")
+    if not all_good_findings.get("error_recording"):
+        results["recommendations"].append("Record exceptions in spans: span.recordException(error)")
+    
+    # Determine status
+    if score >= 80:
+        results["status"] = "[OK] Good observability instrumentation"
+    elif score >= 50:
+        results["status"] = "[?] Partial instrumentation - improvements recommended"
+    elif score >= 20:
+        results["status"] = "[!] Basic instrumentation - significant improvements needed"
+    else:
+        results["status"] = "[!!] Missing observability - add OpenTelemetry instrumentation"
+    
+    return results
+
+
+# ============================================================================
+#  MAIN
+# ============================================================================
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Validate OpenTelemetry instrumentation in AI applications"
+    )
+    parser.add_argument("project_path", nargs="?", default=".", help="Project directory to scan")
+    parser.add_argument("--output", choices=["json", "summary"], default="json",
+                        help="Output format")
+    
+    args = parser.parse_args()
+    
+    if not os.path.isdir(args.project_path):
+        print(json.dumps({"error": f"Directory not found: {args.project_path}"}))
+        sys.exit(1)
+    
+    results = analyze_instrumentation(args.project_path)
+    
+    if args.output == "summary":
+        print(f"\n{'='*60}")
+        print(f"Observability Validator: {results['project']}")
+        print(f"{'='*60}")
+        print(f"Status: {results['status']}")
+        print(f"Score: {results['score']}/100")
+        print(f"\nDependencies:")
+        print(f"  OpenTelemetry installed: {'Yes' if results['dependencies']['has_otel'] else 'No'}")
+        if results['dependencies']['packages']:
+            print(f"  Packages: {', '.join(results['dependencies']['packages'][:5])}")
+        
+        print(f"\nInstrumentation Found:")
+        for key, findings in results['instrumentation']['good_practices'].items():
+            print(f"  ✓ {key}: {len(findings)} occurrences")
+        
+        if results['instrumentation']['potential_gaps']:
+            print(f"\nPotential Gaps:")
+            for gap in results['instrumentation']['potential_gaps'][:5]:
+                print(f"  ⚠ {gap['file']}:{gap['line']} - LLM call without span")
+        
+        if results['recommendations']:
+            print(f"\nRecommendations:")
+            for rec in results['recommendations']:
+                print(f"  → {rec}")
+        
+        print(f"{'='*60}\n")
+    else:
+        print(json.dumps(results, indent=2))
+
+
+if __name__ == "__main__":
+    main()