@musashishao/agent-kit 1.6.1 → 1.7.0

package/.agent/skills/opentelemetry-expert/SKILL.md

@@ -0,0 +1,738 @@
+---
+name: opentelemetry-expert
+description: Deep-dive OpenTelemetry SDK (Node.js, Python). Custom instrumentation for LLM calls, context propagation, exporter configurations, sampling strategies.
+allowed-tools: Read, Write, Bash, Glob, Grep
+skills:
+  - observability-patterns
+---
+
+# OpenTelemetry Expert
+
+> Master-level OpenTelemetry instrumentation for AI applications.
+
+## 🔧 Runtime Scripts
+
+**Execute for automated analysis:**
+
+| Script | Purpose | Usage |
+|--------|---------|-------|
+| `scripts/trace_analyzer.py` | Analyze trace spans for performance issues | `python scripts/trace_analyzer.py <trace_file.json>` |
+
+---
+
+## 1. SDK Setup
+
+### Node.js (TypeScript)
+
+```typescript
+// otel.ts - Initialize before all imports
+import { NodeSDK } from '@opentelemetry/sdk-node';
+import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node';
+import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
+import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http';
+import { PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics';
+import { Resource } from '@opentelemetry/resources';
+import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions';
+
+const sdk = new NodeSDK({
+  resource: new Resource({
+    [SemanticResourceAttributes.SERVICE_NAME]: 'ai-service',
+    [SemanticResourceAttributes.SERVICE_VERSION]: '1.0.0',
+    'deployment.environment': process.env.NODE_ENV || 'development',
+  }),
+  
+  traceExporter: new OTLPTraceExporter({
+    url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT || 'http://localhost:4318/v1/traces',
+  }),
+  
+  metricReader: new PeriodicExportingMetricReader({
+    exporter: new OTLPMetricExporter({
+      url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT || 'http://localhost:4318/v1/metrics',
+    }),
+    exportIntervalMillis: 60000,
+  }),
+  
+  instrumentations: [
+    getNodeAutoInstrumentations({
+      '@opentelemetry/instrumentation-fs': { enabled: false },
+    }),
+  ],
+});
+
+sdk.start();
+
+// Graceful shutdown
+process.on('SIGTERM', () => {
+  sdk.shutdown().then(() => process.exit(0));
+});
+```
+
+### Python
+
+```python
+# otel_setup.py
+from opentelemetry import trace, metrics
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.metrics import MeterProvider
+from opentelemetry.sdk.resources import Resource, SERVICE_NAME, SERVICE_VERSION
+from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
+import os
+
+def setup_opentelemetry():
+    resource = Resource.create({
+        SERVICE_NAME: "ai-service",
+        SERVICE_VERSION: "1.0.0",
+        "deployment.environment": os.getenv("ENVIRONMENT", "development"),
+    })
+    
+    # Tracing
+    trace_provider = TracerProvider(resource=resource)
+    trace_provider.add_span_processor(
+        BatchSpanProcessor(
+            OTLPSpanExporter(endpoint=os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost:4318/v1/traces"))
+        )
+    )
+    trace.set_tracer_provider(trace_provider)
+    
+    # Metrics
+    metric_reader = PeriodicExportingMetricReader(
+        OTLPMetricExporter(endpoint=os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost:4318/v1/metrics")),
+        export_interval_millis=60000,
+    )
+    metrics.set_meter_provider(MeterProvider(resource=resource, metric_readers=[metric_reader]))
+    
+    return trace.get_tracer("ai-service"), metrics.get_meter("ai-service")
+
+tracer, meter = setup_opentelemetry()
+```
+
+---
+
+## 2. Custom LLM Instrumentation
+
+### OpenAI Wrapper (TypeScript)
+
+```typescript
+import { trace, SpanKind, SpanStatusCode, context } from '@opentelemetry/api';
+import OpenAI from 'openai';
+
+const tracer = trace.getTracer('openai-instrumentation');
+
+interface TracedOpenAIOptions {
+  client: OpenAI;
+  capturePrompts?: boolean;  // Set false in production for privacy
+}
+
+export function createTracedOpenAI({ client, capturePrompts = false }: TracedOpenAIOptions) {
+  return {
+    async createCompletion(params: OpenAI.ChatCompletionCreateParams) {
+      const spanName = `openai.chat.${params.model}`;
+      
+      return tracer.startActiveSpan(spanName, {
+        kind: SpanKind.CLIENT,
+        attributes: {
+          'llm.vendor': 'openai',
+          'llm.request.model': params.model,
+          'llm.request.max_tokens': params.max_tokens || -1,
+          'llm.request.temperature': params.temperature || 1,
+        },
+      }, async (span) => {
+        const startTime = performance.now();
+        
+        try {
+          // Optionally capture prompt (be careful with PII)
+          if (capturePrompts) {
+            span.setAttribute('llm.request.messages', JSON.stringify(params.messages).slice(0, 1000));
+          }
+          
+          const response = await client.chat.completions.create(params);
+          
+          // Record response attributes
+          span.setAttributes({
+            'llm.response.model': response.model,
+            'llm.response.id': response.id,
+            'llm.usage.prompt_tokens': response.usage?.prompt_tokens || 0,
+            'llm.usage.completion_tokens': response.usage?.completion_tokens || 0,
+            'llm.usage.total_tokens': response.usage?.total_tokens || 0,
+            'llm.response.finish_reason': response.choices[0]?.finish_reason || 'unknown',
+            'llm.latency_ms': performance.now() - startTime,
+          });
+          
+          span.setStatus({ code: SpanStatusCode.OK });
+          return response;
+          
+        } catch (error) {
+          span.recordException(error as Error);
+          span.setStatus({
+            code: SpanStatusCode.ERROR,
+            message: error instanceof Error ? error.message : 'Unknown error',
+          });
+          
+          // Add error details
+          if (error instanceof OpenAI.APIError) {
+            span.setAttributes({
+              'error.type': error.type || 'api_error',
+              'error.code': error.code || 'unknown',
+              'error.status': error.status,
+            });
+          }
+          
+          throw error;
+        } finally {
+          span.end();
+        }
+      });
+    },
+    
+    // Add streaming support
+    async *createStreamingCompletion(params: OpenAI.ChatCompletionCreateParams) {
+      const spanName = `openai.chat.stream.${params.model}`;
+      
+      const span = tracer.startSpan(spanName, {
+        kind: SpanKind.CLIENT,
+        attributes: {
+          'llm.vendor': 'openai',
+          'llm.request.model': params.model,
+          'llm.streaming': true,
+        },
+      });
+      
+      const startTime = performance.now();
+      let totalTokens = 0;
+      
+      try {
+        const stream = await client.chat.completions.create({
+          ...params,
+          stream: true,
+        });
+        
+        for await (const chunk of stream) {
+          totalTokens++;  // Approximate
+          yield chunk;
+        }
+        
+        span.setAttributes({
+          'llm.usage.estimated_tokens': totalTokens,
+          'llm.latency_ms': performance.now() - startTime,
+        });
+        span.setStatus({ code: SpanStatusCode.OK });
+        
+      } catch (error) {
+        span.recordException(error as Error);
+        span.setStatus({ code: SpanStatusCode.ERROR });
+        throw error;
+      } finally {
+        span.end();
+      }
+    },
+  };
+}
+```
+
+### Anthropic Wrapper (Python)
+
+```python
+from opentelemetry import trace
+from opentelemetry.trace import SpanKind, Status, StatusCode
+import anthropic
+from functools import wraps
+import time
+
+tracer = trace.get_tracer("anthropic-instrumentation")
+
+def traced_anthropic_call(capture_prompts: bool = False):
+    """Decorator to add tracing to Anthropic API calls."""
+    
+    def decorator(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            model = kwargs.get("model", "claude-3")
+            
+            with tracer.start_as_current_span(
+                f"anthropic.messages.{model}",
+                kind=SpanKind.CLIENT,
+                attributes={
+                    "llm.vendor": "anthropic",
+                    "llm.request.model": model,
+                    "llm.request.max_tokens": kwargs.get("max_tokens", 1024),
+                }
+            ) as span:
+                start_time = time.perf_counter()
+                
+                try:
+                    if capture_prompts:
+                        messages = kwargs.get("messages", [])
+                        span.set_attribute("llm.request.messages", str(messages)[:1000])
+                    
+                    response = func(*args, **kwargs)
+                    
+                    # Record usage
+                    if hasattr(response, "usage"):
+                        span.set_attributes({
+                            "llm.usage.input_tokens": response.usage.input_tokens,
+                            "llm.usage.output_tokens": response.usage.output_tokens,
+                        })
+                    
+                    span.set_attributes({
+                        "llm.response.stop_reason": response.stop_reason,
+                        "llm.latency_ms": (time.perf_counter() - start_time) * 1000,
+                    })
+                    
+                    span.set_status(Status(StatusCode.OK))
+                    return response
+                    
+                except anthropic.APIError as e:
+                    span.record_exception(e)
+                    span.set_status(Status(StatusCode.ERROR, str(e)))
+                    span.set_attributes({
+                        "error.type": type(e).__name__,
+                        "error.message": str(e),
+                    })
+                    raise
+        
+        return wrapper
+    return decorator
+
+
+# Usage
+client = anthropic.Anthropic()
+
+@traced_anthropic_call(capture_prompts=False)
+def call_claude(messages: list, model: str = "claude-3-sonnet-20240229", max_tokens: int = 1024):
+    return client.messages.create(
+        model=model,
+        max_tokens=max_tokens,
+        messages=messages,
+    )
+```
+
+---
+
+## 3. Agent Loop Instrumentation
+
+### Multi-Step Agent Tracing
+
+```typescript
+import { trace, context, SpanKind } from '@opentelemetry/api';
+
+const tracer = trace.getTracer('agent-loop');
+
+interface AgentStep {
+  thought: string;
+  action?: { tool: string; input: any };
+  observation?: string;
+}
+
+export async function runTracedAgent(
+  query: string,
+  maxSteps: number = 10
+): Promise<string> {
+  
+  return tracer.startActiveSpan('agent.run', {
+    attributes: {
+      'agent.query': query.slice(0, 500),
+      'agent.max_steps': maxSteps,
+    },
+  }, async (rootSpan) => {
+    const steps: AgentStep[] = [];
+    let stepCount = 0;
+    let finalAnswer: string | null = null;
+    
+    try {
+      while (!finalAnswer && stepCount < maxSteps) {
+        stepCount++;
+        
+        // Create child span for each step
+        const step = await tracer.startActiveSpan(`agent.step.${stepCount}`, {
+          attributes: { 'agent.step.number': stepCount },
+        }, async (stepSpan) => {
+          
+          // Think phase
+          const thought = await tracer.startActiveSpan('agent.think', async (thinkSpan) => {
+            const result = await this.think(query, steps);
+            thinkSpan.setAttribute('agent.thought.length', result.length);
+            thinkSpan.end();
+            return result;
+          });
+          
+          // Parse action
+          const action = this.parseAction(thought);
+          
+          if (action) {
+            // Tool execution
+            const observation = await tracer.startActiveSpan('agent.tool', {
+              attributes: {
+                'agent.tool.name': action.tool,
+                'agent.tool.input_size': JSON.stringify(action.input).length,
+              },
+            }, async (toolSpan) => {
+              try {
+                const result = await this.executeTool(action);
+                toolSpan.setAttribute('agent.tool.success', true);
+                toolSpan.setAttribute('agent.tool.output_size', result.length);
+                toolSpan.end();
+                return result;
+              } catch (error) {
+                toolSpan.recordException(error as Error);
+                toolSpan.setAttribute('agent.tool.success', false);
+                toolSpan.end();
+                throw error;
+              }
+            });
+            
+            stepSpan.setAttribute('agent.step.has_tool', true);
+            stepSpan.end();
+            return { thought, action, observation };
+          } else {
+            // Final answer
+            finalAnswer = this.extractAnswer(thought);
+            stepSpan.setAttribute('agent.step.is_final', true);
+            stepSpan.end();
+            return { thought };
+          }
+        });
+        
+        steps.push(step);
+      }
+      
+      // Record final metrics
+      rootSpan.setAttributes({
+        'agent.total_steps': stepCount,
+        'agent.completed': finalAnswer !== null,
+        'agent.answer_length': finalAnswer?.length || 0,
+      });
+      
+      return finalAnswer || 'No answer found';
+      
+    } catch (error) {
+      rootSpan.recordException(error as Error);
+      throw error;
+    } finally {
+      rootSpan.end();
+    }
+  });
+}
+```
+
+---
+
+## 4. Context Propagation
+
+### Across HTTP Boundaries
+
+```typescript
+import { propagation, context, trace } from '@opentelemetry/api';
+
+// Client side: Inject context into headers
+async function callService(data: any): Promise<Response> {
+  const headers: Record<string, string> = {
+    'Content-Type': 'application/json',
+  };
+  
+  // Inject current trace context
+  propagation.inject(context.active(), headers);
+  
+  return fetch('https://api.service.com/endpoint', {
+    method: 'POST',
+    headers,
+    body: JSON.stringify(data),
+  });
+}
+
+// Server side: Extract context from headers
+function handleRequest(req: Request): void {
+  const extractedContext = propagation.extract(context.active(), req.headers);
+  
+  // Run handler with extracted context
+  context.with(extractedContext, () => {
+    const span = trace.getTracer('server').startSpan('handle_request');
+    try {
+      // Handler logic - spans created here will be children
+      processRequest(req);
+    } finally {
+      span.end();
+    }
+  });
+}
+```
+
+### Across Message Queues
+
+```typescript
+// Producer: Attach context to message
+async function produceMessage(topic: string, message: any): Promise<void> {
+  const carriers: Record<string, string> = {};
+  propagation.inject(context.active(), carriers);
+  
+  await kafka.produce({
+    topic,
+    messages: [{
+      value: JSON.stringify(message),
+      headers: carriers,  // Context in headers
+    }],
+  });
+}
+
+// Consumer: Extract context from message
+async function consumeMessage(message: KafkaMessage): Promise<void> {
+  const extractedContext = propagation.extract(
+    context.active(),
+    Object.fromEntries(
+      Object.entries(message.headers || {}).map(([k, v]) => [k, v?.toString()])
+    )
+  );
+  
+  await context.with(extractedContext, async () => {
+    // Process with correct parent context
+    await processMessage(JSON.parse(message.value.toString()));
+  });
+}
+```
+
+---
+
+## 5. Exporter Configurations
+
+### Multi-Backend Export
+
+```typescript
+import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
+import { JaegerExporter } from '@opentelemetry/exporter-jaeger';
+import { ZipkinExporter } from '@opentelemetry/exporter-zipkin';
+import { ConsoleSpanExporter, SimpleSpanProcessor, BatchSpanProcessor } from '@opentelemetry/sdk-trace-base';
+
+// Development: Console + Jaeger
+const devExporters = [
+  new SimpleSpanProcessor(new ConsoleSpanExporter()),
+  new BatchSpanProcessor(new JaegerExporter({
+    endpoint: 'http://localhost:14268/api/traces',
+  })),
+];
+
+// Production: OTLP to collector
+const prodExporter = new BatchSpanProcessor(
+  new OTLPTraceExporter({
+    url: process.env.OTEL_COLLECTOR_URL,
+    headers: {
+      'Authorization': `Bearer ${process.env.OTEL_AUTH_TOKEN}`,
+    },
+  }),
+  {
+    maxQueueSize: 2048,
+    maxExportBatchSize: 512,
+    scheduledDelayMillis: 5000,
+  }
+);
+
+// Conditional setup
+const isProduction = process.env.NODE_ENV === 'production';
+const spanProcessor = isProduction ? prodExporter : devExporters[1];
+```
+
+### Environment-Based Configuration
+
+```bash
+# .env
+OTEL_SERVICE_NAME=ai-service
+OTEL_EXPORTER_OTLP_ENDPOINT=https://otel-collector.company.com
+OTEL_EXPORTER_OTLP_HEADERS=Authorization=Bearer abc123
+OTEL_TRACES_SAMPLER=parentbased_traceidratio
+OTEL_TRACES_SAMPLER_ARG=0.1
+OTEL_RESOURCE_ATTRIBUTES=deployment.environment=production,service.version=1.2.3
+```
+
+---
+
+## 6. Sampling Strategies
+
+### Adaptive Sampling
+
+```typescript
+import { 
+  Sampler, 
+  SamplingDecision, 
+  SamplingResult,
+  ParentBasedSampler,
+  TraceIdRatioBasedSampler,
+} from '@opentelemetry/sdk-trace-base';
+import { Context, SpanKind, Attributes } from '@opentelemetry/api';
+
+class AIAwareSampler implements Sampler {
+  private baseRatio: number;
+  
+  constructor(baseRatio: number = 0.1) {
+    this.baseRatio = baseRatio;
+  }
+  
+  shouldSample(
+    context: Context,
+    traceId: string,
+    spanName: string,
+    spanKind: SpanKind,
+    attributes: Attributes
+  ): SamplingResult {
+    
+    // Always sample errors
+    if (attributes['error'] === true) {
+      return { decision: SamplingDecision.RECORD_AND_SAMPLED };
+    }
+    
+    // Always sample high-cost operations
+    if (spanName.includes('gpt-4') || spanName.includes('claude-3-opus')) {
+      return { decision: SamplingDecision.RECORD_AND_SAMPLED };
+    }
+    
+    // Always sample slow operations (based on previous metrics)
+    const estimatedLatency = attributes['estimated_latency_ms'] as number;
+    if (estimatedLatency && estimatedLatency > 5000) {
+      return { decision: SamplingDecision.RECORD_AND_SAMPLED };
+    }
+    
+    // Dynamic sampling based on load
+    const currentLoad = getSystemLoad();  // Your implementation
+    const adjustedRatio = currentLoad > 0.8 ? this.baseRatio / 2 : this.baseRatio;
+    
+    // Hash-based decision for consistency
+    const hash = parseInt(traceId.slice(-8), 16) / 0xffffffff;
+    return {
+      decision: hash < adjustedRatio 
+        ? SamplingDecision.RECORD_AND_SAMPLED 
+        : SamplingDecision.NOT_RECORD
+    };
+  }
+  
+  toString(): string {
+    return `AIAwareSampler{ratio=${this.baseRatio}}`;
+  }
+}
+
+// Use parent-based to respect remote decisions
+const sampler = new ParentBasedSampler({
+  root: new AIAwareSampler(0.1),
+});
+```
+
+---
+
+## 7. Custom Metrics
+
+### AI-Specific Metrics
+
+```typescript
+import { metrics } from '@opentelemetry/api';
+
+const meter = metrics.getMeter('ai-metrics');
+
+// Token usage counter
+const tokenCounter = meter.createCounter('llm.tokens', {
+  description: 'Total tokens used',
+  unit: '{token}',
+});
+
+// Latency histogram with AI-specific buckets
+const latencyHistogram = meter.createHistogram('llm.latency', {
+  description: 'LLM response latency',
+  unit: 'ms',
+  advice: {
+    explicitBucketBoundaries: [100, 250, 500, 1000, 2500, 5000, 10000, 30000],
+  },
+});
+
+// Cost gauge (current daily spend)
+const costGauge = meter.createObservableGauge('llm.cost.daily', {
+  description: 'Estimated daily LLM cost',
+  unit: 'cents',
+});
+costGauge.addCallback((result) => {
+  result.observe(calculateDailyCost());
+});
+
+// Agent success rate
+const agentSuccessRate = meter.createObservableGauge('agent.success_rate', {
+  description: 'Agent task success rate over last hour',
+  unit: '%',
+});
+
+// Quality score distribution
+const qualityHistogram = meter.createHistogram('llm.quality_score', {
+  description: 'Response quality scores',
+  unit: '{score}',
+  advice: {
+    explicitBucketBoundaries: [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0],
+  },
+});
+
+// Record metrics
+function recordLLMCall(result: LLMResult): void {
+  const attributes = {
+    model: result.model,
+    success: result.success,
+  };
+  
+  tokenCounter.add(result.totalTokens, { ...attributes, type: 'total' });
+  tokenCounter.add(result.inputTokens, { ...attributes, type: 'input' });
+  tokenCounter.add(result.outputTokens, { ...attributes, type: 'output' });
+  
+  latencyHistogram.record(result.latencyMs, attributes);
+  
+  if (result.qualityScore) {
+    qualityHistogram.record(result.qualityScore, attributes);
+  }
+}
+```
+
+---
+
+## 8. Troubleshooting
+
+### Debug Mode
+
+```bash
+# Enable debug logging
+export OTEL_LOG_LEVEL=debug
+
+# Verify spans are being created
+export OTEL_TRACES_EXPORTER=console
+
+# Verify metrics
+export OTEL_METRICS_EXPORTER=console
+```
+
+### Common Issues
+
+| Issue | Cause | Solution |
+|-------|-------|----------|
+| No traces | Missing SDK init | Import otel.ts first |
+| Missing child spans | Wrong context | Use `startActiveSpan` |
+| High memory | Too many attributes | Limit attribute sizes |
+| Missing attributes | Set after span ends | Set before `end()` |
+| Dropped spans | Batch too slow | Reduce batch size |
+
+---
+
+## 9. Checklist
+
+### SDK Setup
+- [ ] Resource configured with service info
+- [ ] Trace exporter configured
+- [ ] Metric exporter configured
+- [ ] Graceful shutdown handled
+
+### Instrumentation
+- [ ] All LLM calls have spans
+- [ ] AI-specific attributes set
+- [ ] Errors recorded with context
+- [ ] Token usage metered
+
+### Production
+- [ ] Sampling configured
+- [ ] Export batching tuned
+- [ ] Sensitive data not logged
+- [ ] Cardinality controlled
+
+---
+
+> **Remember:** Good instrumentation is invisible when things work and invaluable when they don't.