codecruise 0.1.0

package/config/skills/resilience/SKILL.md

@@ -0,0 +1,573 @@
+---
+name: resilience-patterns
+description: Fault tolerance - circuit breakers, retries, timeouts, bulkheads, graceful degradation
+keywords: [resilience, circuit breaker, retry, timeout, bulkhead, fault tolerance, graceful degradation]
+---
+
+# Resilience Patterns
+
+Fault tolerance, retries, and graceful degradation for production systems.
+
+## Core Principles
+
+1. **Fail fast, recover gracefully**
+2. **Isolate failures** — one failing dependency shouldn't bring down the system
+3. **Always have a fallback**
+4. **Make failures visible** — log, metric, alert
+
+## Circuit Breaker
+
+Prevent cascading failures by stopping calls to failing services.
+
+### States
+
+```
+CLOSED → (failures exceed threshold) → OPEN
+                                         ↓
+OPEN → (wait period expires) → HALF-OPEN
+                                    ↓
+HALF-OPEN → (success) → CLOSED
+          → (failure) → OPEN
+```
+
+### Implementation
+
+```typescript
+enum CircuitState {
+  CLOSED = 'CLOSED',
+  OPEN = 'OPEN',
+  HALF_OPEN = 'HALF_OPEN',
+}
+
+interface CircuitBreakerOptions {
+  failureThreshold: number;    // Failures to trip circuit
+  successThreshold: number;    // Successes to close circuit
+  timeout: number;             // Time in OPEN state (ms)
+}
+
+class CircuitBreaker {
+  private state: CircuitState = CircuitState.CLOSED;
+  private failureCount = 0;
+  private successCount = 0;
+  private lastFailureTime = 0;
+
+  constructor(
+    private name: string,
+    private options: CircuitBreakerOptions = {
+      failureThreshold: 5,
+      successThreshold: 3,
+      timeout: 30000,
+    }
+  ) {}
+
+  async execute<T>(operation: () => Promise<T>): Promise<T> {
+    if (this.state === CircuitState.OPEN) {
+      if (Date.now() - this.lastFailureTime >= this.options.timeout) {
+        this.state = CircuitState.HALF_OPEN;
+        this.successCount = 0;
+      } else {
+        throw new CircuitOpenError(this.name);
+      }
+    }
+
+    try {
+      const result = await operation();
+      this.onSuccess();
+      return result;
+    } catch (error) {
+      this.onFailure();
+      throw error;
+    }
+  }
+
+  private onSuccess(): void {
+    this.failureCount = 0;
+
+    if (this.state === CircuitState.HALF_OPEN) {
+      this.successCount++;
+      if (this.successCount >= this.options.successThreshold) {
+        this.state = CircuitState.CLOSED;
+        logger.info('Circuit closed', { circuit: this.name });
+      }
+    }
+  }
+
+  private onFailure(): void {
+    this.failureCount++;
+    this.lastFailureTime = Date.now();
+
+    if (this.state === CircuitState.HALF_OPEN) {
+      this.state = CircuitState.OPEN;
+      logger.warn('Circuit reopened', { circuit: this.name });
+    } else if (this.failureCount >= this.options.failureThreshold) {
+      this.state = CircuitState.OPEN;
+      logger.warn('Circuit opened', {
+        circuit: this.name,
+        failures: this.failureCount
+      });
+    }
+  }
+
+  getState(): CircuitState {
+    return this.state;
+  }
+}
+
+class CircuitOpenError extends AppError {
+  constructor(circuitName: string) {
+    super('CIRCUIT_OPEN', `Circuit ${circuitName} is open`, 503);
+  }
+}
+```
+
+### Usage
+
+```typescript
+// Create circuit breaker per external dependency
+const paymentCircuit = new CircuitBreaker('payment-service', {
+  failureThreshold: 5,
+  successThreshold: 3,
+  timeout: 30000,
+});
+
+async function processPayment(order: Order): Promise<PaymentResult> {
+  return paymentCircuit.execute(async () => {
+    return paymentService.charge(order);
+  });
+}
+```
+
+## Retry with Backoff
+
+### Configuration
+
+| Operation | Timeout | Max Retries | Backoff |
+|-----------|---------|-------------|---------|
+| HTTP (external) | 10s | 3 | Exponential |
+| HTTP (internal) | 5s | 2 | Linear |
+| Database | 5s | 2 | Linear |
+| Cache | 1s | 1 | None |
+| Message queue | 30s | 5 | Exponential |
+
+### Implementation
+
+```typescript
+interface RetryOptions {
+  maxRetries: number;
+  baseDelay: number;      // ms
+  maxDelay: number;       // ms
+  backoff: 'linear' | 'exponential' | 'none';
+  retryOn?: (error: Error) => boolean;
+}
+
+const DEFAULT_RETRY_OPTIONS: RetryOptions = {
+  maxRetries: 3,
+  baseDelay: 1000,
+  maxDelay: 30000,
+  backoff: 'exponential',
+};
+
+async function withRetry<T>(
+  operation: () => Promise<T>,
+  options: Partial<RetryOptions> = {}
+): Promise<T> {
+  const opts = { ...DEFAULT_RETRY_OPTIONS, ...options };
+  let lastError: Error;
+
+  for (let attempt = 1; attempt <= opts.maxRetries + 1; attempt++) {
+    try {
+      return await operation();
+    } catch (error) {
+      lastError = error as Error;
+
+      // Check if error is retryable
+      const shouldRetry = opts.retryOn
+        ? opts.retryOn(lastError)
+        : isRetryable(lastError);
+
+      if (!shouldRetry || attempt > opts.maxRetries) {
+        throw lastError;
+      }
+
+      // Calculate delay
+      const delay = calculateDelay(attempt, opts);
+
+      logger.warn('Retrying operation', {
+        attempt,
+        maxRetries: opts.maxRetries,
+        delay,
+        error: lastError.message,
+      });
+
+      await sleep(delay);
+    }
+  }
+
+  throw lastError!;
+}
+
+function calculateDelay(attempt: number, opts: RetryOptions): number {
+  let delay: number;
+
+  switch (opts.backoff) {
+    case 'none':
+      delay = opts.baseDelay;
+      break;
+    case 'linear':
+      delay = opts.baseDelay * attempt;
+      break;
+    case 'exponential':
+      delay = opts.baseDelay * Math.pow(2, attempt - 1);
+      break;
+  }
+
+  // Add jitter (±10%)
+  const jitter = delay * 0.1 * (Math.random() * 2 - 1);
+  delay = Math.min(delay + jitter, opts.maxDelay);
+
+  return Math.round(delay);
+}
+
+function isRetryable(error: Error): boolean {
+  // Network errors
+  if (error.message.includes('ECONNREFUSED')) return true;
+  if (error.message.includes('ETIMEDOUT')) return true;
+  if (error.message.includes('ENOTFOUND')) return true;
+
+  // HTTP errors
+  if (error instanceof HttpError) {
+    return [408, 429, 500, 502, 503, 504].includes(error.statusCode);
+  }
+
+  // Database errors
+  if (error.message.includes('deadlock')) return true;
+  if (error.message.includes('connection')) return true;
+
+  return false;
+}
+```
+
+### Usage
+
+```typescript
+const result = await withRetry(
+  () => externalApi.fetch(id),
+  {
+    maxRetries: 3,
+    backoff: 'exponential',
+    retryOn: (error) => error instanceof HttpError && error.statusCode >= 500,
+  }
+);
+```
+
+## Timeouts
+
+### Configuration
+
+```typescript
+const TIMEOUTS = {
+  // HTTP requests
+  httpExternal: 10000,    // 10s for external APIs
+  httpInternal: 5000,     // 5s for internal services
+
+  // Database
+  dbQuery: 5000,          // 5s for queries
+  dbTransaction: 30000,   // 30s for transactions
+
+  // Cache
+  cacheRead: 1000,        // 1s
+  cacheWrite: 2000,       // 2s
+
+  // Message queue
+  queuePublish: 5000,     // 5s
+  queueConsume: 30000,    // 30s per message
+} as const;
+```
+
+### Implementation
+
+```typescript
+async function withTimeout<T>(
+  operation: Promise<T>,
+  ms: number,
+  operationName: string
+): Promise<T> {
+  const timeout = new Promise<never>((_, reject) => {
+    setTimeout(() => {
+      reject(new TimeoutError(operationName, ms));
+    }, ms);
+  });
+
+  return Promise.race([operation, timeout]);
+}
+
+class TimeoutError extends AppError {
+  constructor(operation: string, timeout: number) {
+    super(
+      'TIMEOUT',
+      `Operation ${operation} timed out after ${timeout}ms`,
+      504
+    );
+  }
+}
+
+// Usage
+const user = await withTimeout(
+  db.user.findUnique({ where: { id } }),
+  TIMEOUTS.dbQuery,
+  'findUser'
+);
+```
+
+## Graceful Degradation
+
+### Fallback Patterns
+
+```typescript
+// Pattern 1: Return cached/stale data
+async function getProductWithFallback(id: string): Promise<Product> {
+  try {
+    const product = await productService.fetch(id);
+    await cache.set(`product:${id}`, product, 3600);
+    return product;
+  } catch (error) {
+    logger.warn('Product service failed, using cache', { id, error });
+
+    const cached = await cache.get(`product:${id}`);
+    if (cached) {
+      return { ...cached, _stale: true };
+    }
+
+    throw error;
+  }
+}
+
+// Pattern 2: Return default value
+async function getRecommendations(userId: string): Promise<Product[]> {
+  try {
+    return await recommendationService.getFor(userId);
+  } catch (error) {
+    logger.warn('Recommendations failed, returning defaults', { userId });
+    return getDefaultRecommendations();
+  }
+}
+
+// Pattern 3: Disable feature
+async function getFeatureFlags(userId: string): Promise<FeatureFlags> {
+  try {
+    return await featureFlagService.getFor(userId);
+  } catch (error) {
+    logger.warn('Feature flags unavailable, using safe defaults', { userId });
+    return {
+      newCheckout: false,      // Disable experimental features
+      darkMode: true,          // Keep stable features
+      betaFeatures: false,
+    };
+  }
+}
+
+// Pattern 4: Queue for later
+async function sendNotification(notification: Notification): Promise<void> {
+  try {
+    await notificationService.send(notification);
+  } catch (error) {
+    logger.warn('Notification failed, queuing for retry', { notification });
+    await queue.add('notifications:retry', notification, {
+      delay: 60000,
+      attempts: 5,
+    });
+  }
+}
+```
+
+## Bulkhead Pattern
+
+Isolate resources to prevent one consumer from exhausting them.
+
+```typescript
+class Bulkhead {
+  private active = 0;
+  private queue: Array<() => void> = [];
+
+  constructor(
+    private name: string,
+    private maxConcurrent: number,
+    private maxQueue: number = 100
+  ) {}
+
+  async execute<T>(operation: () => Promise<T>): Promise<T> {
+    if (this.active >= this.maxConcurrent) {
+      if (this.queue.length >= this.maxQueue) {
+        throw new BulkheadFullError(this.name);
+      }
+
+      await new Promise<void>((resolve) => {
+        this.queue.push(resolve);
+      });
+    }
+
+    this.active++;
+
+    try {
+      return await operation();
+    } finally {
+      this.active--;
+
+      const next = this.queue.shift();
+      if (next) next();
+    }
+  }
+}
+
+// Separate bulkheads for different concerns
+const dbBulkhead = new Bulkhead('database', 20);
+const apiBulkhead = new Bulkhead('external-api', 10);
+
+// Usage
+const user = await dbBulkhead.execute(() =>
+  db.user.findUnique({ where: { id } })
+);
+```
+
+## Graceful Shutdown
+
+```typescript
+class GracefulShutdown {
+  private isShuttingDown = false;
+  private connections = new Set<Socket>();
+
+  constructor(
+    private server: Server,
+    private cleanup: () => Promise<void>,
+    private timeout = 30000
+  ) {
+    this.setup();
+  }
+
+  private setup(): void {
+    // Track connections
+    this.server.on('connection', (socket) => {
+      this.connections.add(socket);
+      socket.on('close', () => this.connections.delete(socket));
+    });
+
+    // Handle shutdown signals
+    process.on('SIGTERM', () => this.shutdown('SIGTERM'));
+    process.on('SIGINT', () => this.shutdown('SIGINT'));
+  }
+
+  private async shutdown(signal: string): Promise<void> {
+    if (this.isShuttingDown) return;
+    this.isShuttingDown = true;
+
+    logger.info('Shutdown initiated', { signal });
+
+    // Stop accepting new connections
+    this.server.close();
+
+    // Set deadline
+    const deadline = setTimeout(() => {
+      logger.error('Graceful shutdown timed out, forcing exit');
+      process.exit(1);
+    }, this.timeout);
+
+    try {
+      // Wait for existing requests to complete
+      await this.drainConnections();
+
+      // Run cleanup (close DB, flush queues, etc.)
+      await this.cleanup();
+
+      clearTimeout(deadline);
+      logger.info('Graceful shutdown complete');
+      process.exit(0);
+    } catch (error) {
+      logger.error('Shutdown error', { error });
+      process.exit(1);
+    }
+  }
+
+  private async drainConnections(): Promise<void> {
+    // Give existing requests time to complete
+    await sleep(5000);
+
+    // Force close remaining connections
+    for (const socket of this.connections) {
+      socket.destroy();
+    }
+  }
+}
+
+// Usage
+const shutdown = new GracefulShutdown(
+  server,
+  async () => {
+    await db.$disconnect();
+    await redis.quit();
+    await queue.close();
+  },
+  30000
+);
+```
+
+## Health-Based Load Shedding
+
+```typescript
+interface HealthMetrics {
+  cpuUsage: number;
+  memoryUsage: number;
+  eventLoopLag: number;
+  activeConnections: number;
+}
+
+class LoadShedder {
+  private readonly thresholds = {
+    cpuUsage: 0.8,           // 80%
+    memoryUsage: 0.85,       // 85%
+    eventLoopLag: 100,       // 100ms
+    activeConnections: 1000,
+  };
+
+  shouldShed(metrics: HealthMetrics): boolean {
+    return (
+      metrics.cpuUsage > this.thresholds.cpuUsage ||
+      metrics.memoryUsage > this.thresholds.memoryUsage ||
+      metrics.eventLoopLag > this.thresholds.eventLoopLag ||
+      metrics.activeConnections > this.thresholds.activeConnections
+    );
+  }
+}
+
+// Middleware
+function loadSheddingMiddleware(shedder: LoadShedder) {
+  return (req: Request, res: Response, next: NextFunction) => {
+    const metrics = getHealthMetrics();
+
+    if (shedder.shouldShed(metrics)) {
+      logger.warn('Shedding load', { metrics, path: req.path });
+
+      res.status(503).json({
+        code: 'SERVICE_OVERLOADED',
+        message: 'Service temporarily unavailable',
+        retryAfter: 5,
+      });
+      return;
+    }
+
+    next();
+  };
+}
+```
+
+## Quality Checklist
+
+- [ ] Circuit breakers on all external dependencies
+- [ ] Retry with exponential backoff implemented
+- [ ] Timeouts configured for all operations
+- [ ] Fallback strategies for critical paths
+- [ ] Bulkheads isolate resource pools
+- [ ] Graceful shutdown handles SIGTERM/SIGINT
+- [ ] Health checks expose degraded state
+- [ ] Load shedding under pressure
+- [ ] All failures logged with context
+- [ ] Metrics track circuit state and retry counts