blockmine 1.21.0 → 1.23.0

package/.claude/skills/sre/resources/observability-stack.md

@@ -0,0 +1,240 @@
+# Observability Stack
+
+Prometheus + Grafana setup, Loki for logs, Jaeger/Tempo for distributed tracing, and comprehensive observability implementation.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Metrics - Prometheus + Grafana](#metrics---prometheus--grafana)
+- [Logs - Loki](#logs---loki)
+- [Traces - Jaeger/Tempo](#traces---jaegertempo)
+- [Integration](#integration)
+
+## Overview
+
+**Three Pillars of Observability:**
+```
+Metrics → What is happening (aggregated numbers)
+Logs    → Detailed event records
+Traces  → Request flow through system
+```
+
+## Metrics - Prometheus + Grafana
+
+**Prometheus Setup:**
+```yaml
+# prometheus-values.yaml
+prometheus:
+  prometheusSpec:
+    retention: 15d
+    retentionSize: "50GB"
+    storageSpec:
+      volumeClaimTemplate:
+        spec:
+          resources:
+            requests:
+              storage: 100Gi
+
+    serviceMonitorSelectorNilUsesHelmValues: false
+    podMonitorSelectorNilUsesHelmValues: false
+
+    additionalScrapeConfigs:
+      - job_name: 'kubernetes-pods'
+        kubernetes_sd_configs:
+          - role: pod
+        relabel_configs:
+          - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
+            action: keep
+            regex: true
+```
+
+**ServiceMonitor:**
+```yaml
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: api-service
+spec:
+  selector:
+    matchLabels:
+      app: api
+  endpoints:
+  - port: metrics
+    interval: 30s
+    path: /metrics
+```
+
+**Grafana Dashboard:**
+```json
+{
+  "dashboard": {
+    "title": "Service Overview",
+    "panels": [
+      {
+        "title": "Request Rate",
+        "targets": [{
+          "expr": "rate(http_requests_total[5m])"
+        }]
+      },
+      {
+        "title": "Error Rate",
+        "targets": [{
+          "expr": "rate(http_requests_total{status=~\"5..\"}[5m]) / rate(http_requests_total[5m])"
+        }]
+      },
+      {
+        "title": "Latency p95",
+        "targets": [{
+          "expr": "histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m]))"
+        }]
+      }
+    ]
+  }
+}
+```
+
+## Logs - Loki
+
+**Loki Setup:**
+```yaml
+# loki-values.yaml
+loki:
+  auth_enabled: false
+
+  ingester:
+    chunk_idle_period: 3m
+    chunk_retain_period: 1m
+    max_chunk_age: 1h
+
+  limits_config:
+    enforce_metric_name: false
+    reject_old_samples: true
+    reject_old_samples_max_age: 168h
+    ingestion_rate_mb: 10
+    ingestion_burst_size_mb: 20
+
+  schema_config:
+    configs:
+      - from: 2024-01-01
+        store: boltdb-shipper
+        object_store: s3
+        schema: v11
+        index:
+          prefix: loki_index_
+          period: 24h
+
+promtail:
+  config:
+    clients:
+      - url: http://loki:3100/loki/api/v1/push
+```
+
+**Promtail Config:**
+```yaml
+scrape_configs:
+  - job_name: kubernetes-pods
+    kubernetes_sd_configs:
+      - role: pod
+    pipeline_stages:
+      - docker: {}
+      - json:
+          expressions:
+            level: level
+            timestamp: timestamp
+            message: message
+      - labels:
+          level:
+          app:
+```
+
+**LogQL Queries:**
+```
+# Recent errors
+{app="api"} |= "error" | json
+
+# Rate of errors
+rate({app="api"} |= "error" [5m])
+
+# Latency > 1s
+{app="api"} | json | duration > 1s
+```
+
+## Traces - Jaeger/Tempo
+
+**Tempo Setup:**
+```yaml
+tempo:
+  storage:
+    trace:
+      backend: s3
+      s3:
+        bucket: traces
+        endpoint: s3.amazonaws.com
+
+  receivers:
+    jaeger:
+      protocols:
+        grpc:
+        thrift_http:
+    otlp:
+      protocols:
+        grpc:
+        http:
+```
+
+**Application Instrumentation (OpenTelemetry):**
+```typescript
+import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node';
+import { registerInstrumentations } from '@opentelemetry/instrumentation';
+import { HttpInstrumentation } from '@opentelemetry/instrumentation-http';
+import { JaegerExporter } from '@opentelemetry/exporter-jaeger';
+
+const provider = new NodeTracerProvider();
+provider.addSpanProcessor(
+  new BatchSpanProcessor(
+    new JaegerExporter({
+      endpoint: 'http://tempo:14268/api/traces'
+    })
+  )
+);
+
+provider.register();
+
+registerInstrumentations({
+  instrumentations: [
+    new HttpInstrumentation(),
+    new ExpressInstrumentation()
+  ]
+});
+```
+
+## Integration
+
+**Grafana Unified View:**
+```json
+{
+  "panels": [
+    {
+      "title": "Metrics",
+      "datasource": "Prometheus",
+      "targets": [{"expr": "rate(http_requests_total[5m])"}]
+    },
+    {
+      "title": "Logs",
+      "datasource": "Loki",
+      "targets": [{"expr": "{app=\"api\"}"}]
+    },
+    {
+      "title": "Traces",
+      "datasource": "Tempo",
+      "targets": [{"query": "service.name=\"api\""}]
+    }
+  ]
+}
+```
+
+---
+
+**Related Resources:**
+- [alerting-best-practices.md](alerting-best-practices.md)
+- [slo-sli-sla.md](slo-sli-sla.md)

package/.claude/skills/sre/resources/on-call-runbooks.md

@@ -0,0 +1,167 @@
+# On-Call Runbooks
+
+Runbook structure, common scenarios, debugging procedures, escalation paths, and incident response playbooks.
+
+## Runbook Template
+
+```markdown
+# Runbook: [Service Name] - [Issue Type]
+
+## Overview
+**Service:** api-service
+**On-Call:** platform-team
+**Severity:** SEV2
+**Last Updated:** 2024-01-15
+
+## Symptoms
+- High API error rate (> 1%)
+- Increased p95 latency (> 1s)
+- Customer reports of failures
+
+## Impact
+- API requests failing
+- Users unable to complete actions
+- Revenue impact: $X/hour
+
+## Diagnosis
+
+### 1. Check Service Health
+\`\`\`bash
+# Check pod status
+kubectl get pods -n production -l app=api-service
+
+# Check recent logs
+kubectl logs -n production -l app=api-service --tail=100
+
+# Check metrics
+curl https://grafana.example.com/d/api-health
+\`\`\`
+
+### 2. Check Dependencies
+\`\`\`bash
+# Database connectivity
+psql -h db.example.com -U api -c "SELECT 1"
+
+# Redis connectivity
+redis-cli -h cache.example.com ping
+
+# External API
+curl https://external-api.example.com/health
+\`\`\`
+
+### 3. Check Recent Changes
+\`\`\`bash
+# Recent deployments
+kubectl rollout history deployment/api-service -n production
+
+# Recent config changes
+git log --since="2 hours ago" -- config/
+\`\`\`
+
+## Common Causes
+
+### Database Connection Pool Exhausted
+**Symptoms:** Connection timeout errors
+**Fix:**
+\`\`\`bash
+# Scale connection pool
+kubectl set env deployment/api-service DB_POOL_SIZE=100
+
+# Or restart pods to reset connections
+kubectl rollout restart deployment/api-service
+\`\`\`
+
+### High Traffic Spike
+**Symptoms:** All resources at capacity
+**Fix:**
+\`\`\`bash
+# Scale up replicas
+kubectl scale deployment/api-service --replicas=20
+
+# Enable rate limiting
+kubectl apply -f rate-limit-config.yaml
+\`\`\`
+
+### Downstream Service Failure
+**Symptoms:** Timeouts to specific service
+**Fix:**
+\`\`\`bash
+# Enable circuit breaker
+kubectl apply -f circuit-breaker.yaml
+
+# Or disable affected feature
+kubectl set env deployment/api-service FEATURE_X_ENABLED=false
+\`\`\`
+
+## Mitigation Steps
+
+1. **Immediate (< 5 minutes)**
+   - Roll back recent deployment if applicable
+   - Scale resources if needed
+   - Enable circuit breakers
+
+2. **Short-term (< 30 minutes)**
+   - Identify root cause
+   - Apply targeted fix
+   - Monitor recovery
+
+3. **Long-term (follow-up)**
+   - Schedule postmortem
+   - Implement preventive measures
+   - Update runbook
+
+## Escalation
+
+- **L1 (Primary):** platform-team on-call
+- **L2 (Secondary):** platform-team-lead
+- **L3 (Manager):** Engineering Manager
+- **External:** DBA team (for database issues)
+
+## Related Runbooks
+- [Database Connection Issues](runbook-db-connections.md)
+- [High Traffic Handling](runbook-traffic-spike.md)
+- [Deployment Rollback](runbook-rollback.md)
+
+## Validation
+After mitigation:
+- [ ] Error rate < 0.1%
+- [ ] Latency back to normal (< 200ms p95)
+- [ ] All pods healthy
+- [ ] No active alerts
+- [ ] Customer impact resolved
+\`\`\`
+
+## Common Runbooks
+
+**High Memory Usage:**
+```markdown
+# Diagnosis
+kubectl top pods -n production
+kubectl describe pod <pod-name>
+
+# Check for memory leaks
+curl http://pod-ip:9090/debug/pprof/heap
+
+# Mitigation
+kubectl set resources deployment/api-service --limits=memory=2Gi
+kubectl rollout restart deployment/api-service
+```
+
+**Disk Space Full:**
+```markdown
+# Check disk usage
+df -h
+
+# Find large files
+du -sh /* | sort -hr | head -10
+
+# Clean up
+docker system prune -a --volumes -f
+kubectl delete pods --field-selector status.phase=Failed
+```
+
+---
+
+**Related Resources:**
+- [incident-management.md](incident-management.md)
+- [observability-stack.md](observability-stack.md)

package/.claude/skills/sre/resources/performance-optimization.md

@@ -0,0 +1,108 @@
+# Performance Optimization
+
+Profiling techniques, bottleneck identification, database optimization, caching strategies, and performance tuning.
+
+## Profiling
+
+**Application Profiling:**
+```bash
+# Node.js profiling
+node --prof app.js
+node --prof-process isolate-*-v8.log > processed.txt
+
+# Python profiling
+python -m cProfile -o profile.stats app.py
+python -m pstats profile.stats
+
+# Go profiling
+go tool pprof http://localhost:6060/debug/pprof/profile
+```
+
+**Database Query Profiling:**
+```sql
+-- PostgreSQL
+EXPLAIN ANALYZE SELECT * FROM users WHERE email = 'user@example.com';
+
+-- Add index
+CREATE INDEX idx_users_email ON users(email);
+
+-- MySQL
+EXPLAIN SELECT * FROM users WHERE email = 'user@example.com';
+```
+
+## Caching Strategies
+
+**Multi-Layer Caching:**
+```
+Application Cache (in-memory)
+  ↓ miss
+CDN Cache (edge)
+  ↓ miss
+Redis Cache (distributed)
+  ↓ miss
+Database
+```
+
+**Redis Caching:**
+```typescript
+import Redis from 'ioredis';
+const redis = new Redis();
+
+async function getCachedData(key: string) {
+  // Try cache first
+  const cached = await redis.get(key);
+  if (cached) {
+    return JSON.parse(cached);
+  }
+
+  // Fetch from database
+  const data = await database.query(key);
+
+  // Cache for 1 hour
+  await redis.setex(key, 3600, JSON.stringify(data));
+
+  return data;
+}
+```
+
+## Database Optimization
+
+**Indexing Strategy:**
+```sql
+-- Add index for frequent queries
+CREATE INDEX idx_orders_user_created ON orders(user_id, created_at);
+
+-- Partial index for active records
+CREATE INDEX idx_active_users ON users(id) WHERE status = 'active';
+
+-- Covering index
+CREATE INDEX idx_users_lookup ON users(email) INCLUDE (name, created_at);
+```
+
+**Connection Pooling:**
+```typescript
+import { Pool } from 'pg';
+
+const pool = new Pool({
+  max: 20,  // Maximum connections
+  idleTimeoutMillis: 30000,
+  connectionTimeoutMillis: 2000,
+});
+```
+
+## Performance Best Practices
+
+1. **Use CDN for static assets**
+2. **Implement proper caching**
+3. **Optimize database queries**
+4. **Use connection pooling**
+5. **Enable compression**
+6. **Lazy load resources**
+7. **Minimize payload size**
+8. **Use async/parallel processing**
+
+---
+
+**Related Resources:**
+- [capacity-planning.md](capacity-planning.md)
+- [observability-stack.md](observability-stack.md)

package/.claude/skills/sre/resources/reliability-patterns.md

@@ -0,0 +1,183 @@
+# Reliability Patterns
+
+Circuit breakers, retries, timeouts, bulkheads, rate limiting, graceful degradation, and resilience design patterns.
+
+## Circuit Breaker Pattern
+
+**Concept:**
+```
+Closed → Normal operation
+  ↓ (failures exceed threshold)
+Open → Fail fast, don't call service
+  ↓ (after timeout)
+Half-Open → Test if service recovered
+  ↓ (success)
+Closed → Resume normal operation
+```
+
+**Implementation (Resilience4j):**
+```java
+CircuitBreakerConfig config = CircuitBreakerConfig.custom()
+    .failureRateThreshold(50)
+    .waitDurationInOpenState(Duration.ofMillis(30000))
+    .slidingWindowSize(10)
+    .build();
+
+CircuitBreaker circuitBreaker = CircuitBreaker.of("api", config);
+
+Supplier<String> decoratedSupplier = CircuitBreaker
+    .decorateSupplier(circuitBreaker, () -> callExternalService());
+
+String result = Try.ofSupplier(decoratedSupplier)
+    .recover(throwable -> "fallback value")
+    .get();
+```
+
+## Retry Pattern
+
+**Exponential Backoff:**
+```typescript
+async function retryWithBackoff<T>(
+  fn: () => Promise<T>,
+  maxRetries = 3
+): Promise<T> {
+  for (let i = 0; i < maxRetries; i++) {
+    try {
+      return await fn();
+    } catch (error) {
+      if (i === maxRetries - 1) throw error;
+
+      const delay = Math.pow(2, i) * 1000;  // 1s, 2s, 4s
+      await new Promise(resolve => setTimeout(resolve, delay));
+    }
+  }
+  throw new Error('Max retries exceeded');
+}
+```
+
+## Timeout Pattern
+
+**Service Timeouts:**
+```yaml
+timeouts:
+  connection_timeout: 5s
+  read_timeout: 30s
+  write_timeout: 30s
+  idle_timeout: 120s
+```
+
+**Implementation:**
+```typescript
+async function fetchWithTimeout(url: string, timeout = 5000) {
+  const controller = new AbortController();
+  const timeoutId = setTimeout(() => controller.abort(), timeout);
+
+  try {
+    const response = await fetch(url, { signal: controller.signal });
+    return response;
+  } finally {
+    clearTimeout(timeoutId);
+  }
+}
+```
+
+## Bulkhead Pattern
+
+**Resource Isolation:**
+```
+Service A → Connection Pool A (20 connections)
+Service B → Connection Pool B (20 connections)
+Service C → Connection Pool C (10 connections)
+
+Failure in Service A doesn't affect B or C
+```
+
+**Thread Pool Isolation:**
+```java
+ThreadPoolExecutor serviceAPool = new ThreadPoolExecutor(
+    10,  // core size
+    20,  // max size
+    60L, TimeUnit.SECONDS,
+    new LinkedBlockingQueue<>(100)
+);
+```
+
+## Rate Limiting
+
+**Token Bucket:**
+```typescript
+class RateLimiter {
+  private tokens: number;
+  private capacity: number;
+  private refillRate: number;
+
+  async acquire(): Promise<boolean> {
+    if (this.tokens >= 1) {
+      this.tokens--;
+      return true;
+    }
+    return false;
+  }
+
+  refill() {
+    this.tokens = Math.min(
+      this.capacity,
+      this.tokens + this.refillRate
+    );
+  }
+}
+```
+
+**Nginx Rate Limiting:**
+```nginx
+limit_req_zone $binary_remote_addr zone=api:10m rate=10r/s;
+
+server {
+    location /api/ {
+        limit_req zone=api burst=20 nodelay;
+        proxy_pass http://backend;
+    }
+}
+```
+
+## Graceful Degradation
+
+**Feature Flags:**
+```typescript
+async function getRecommendations(userId: string) {
+  if (!featureFlags.isEnabled('ml_recommendations')) {
+    // Fallback to simple algorithm
+    return getPopularItems();
+  }
+
+  try {
+    return await mlService.getRecommendations(userId);
+  } catch (error) {
+    // Degrade gracefully
+    return getPopularItems();
+  }
+}
+```
+
+**Cache Fallback:**
+```typescript
+async function getData(key: string) {
+  try {
+    return await database.get(key);
+  } catch (error) {
+    // Serve stale data from cache
+    const stale = await cache.get(key);
+    if (stale) {
+      logger.warn('Serving stale data due to DB error');
+      return stale;
+    }
+    throw error;
+  }
+}
+```
+
+---
+
+**Related Resources:**
+- [chaos-engineering.md](chaos-engineering.md)
+- [incident-management.md](incident-management.md)