aigroup-workflow 2.1.2 → 2.2.1

package/skills/sre-engineer/references/automation-toil.md

@@ -1,492 +0,0 @@
-# Automation and Toil Reduction
-
-## Toil Definition
-
-Toil is manual, repetitive, automatable work that scales linearly with service growth.
-
-```python
-from dataclasses import dataclass
-from enum import Enum
-
-class ToilCategory(Enum):
-    """Categories of operational toil."""
-    MANUAL_INTERVENTION = "manual"
-    REPETITIVE_TASKS = "repetitive"
-    NO_ENDURING_VALUE = "no_value"
-    SCALES_WITH_SERVICE = "scales"
-    INTERRUPT_DRIVEN = "reactive"
-
-@dataclass
-class ToilItem:
-    """Track a specific toil activity."""
-    name: str
-    frequency_per_week: int
-    minutes_per_occurrence: int
-    category: ToilCategory
-    automation_difficulty: str  # 'easy', 'medium', 'hard'
-
-    @property
-    def weekly_hours(self) -> float:
-        """Calculate weekly hours spent on this toil."""
-        return (self.frequency_per_week * self.minutes_per_occurrence) / 60
-
-    @property
-    def annual_hours(self) -> float:
-        """Calculate annual hours spent on this toil."""
-        return self.weekly_hours * 52
-
-    def roi_score(self) -> float:
-        """Calculate ROI score for automation (higher = better).
-
-        Score considers time saved vs. difficulty.
-        """
-        difficulty_multiplier = {
-            'easy': 1.0,
-            'medium': 0.5,
-            'hard': 0.25,
-        }
-        return self.annual_hours * difficulty_multiplier.get(
-            self.automation_difficulty, 0.1
-        )
-
-# Example toil inventory
-toil_items = [
-    ToilItem(
-        name="Manual database failover",
-        frequency_per_week=2,
-        minutes_per_occurrence=30,
-        category=ToilCategory.MANUAL_INTERVENTION,
-        automation_difficulty='medium',
-    ),
-    ToilItem(
-        name="Restarting hung processes",
-        frequency_per_week=5,
-        minutes_per_occurrence=15,
-        category=ToilCategory.REPETITIVE_TASKS,
-        automation_difficulty='easy',
-    ),
-    ToilItem(
-        name="Log file cleanup",
-        frequency_per_week=7,
-        minutes_per_occurrence=10,
-        category=ToilCategory.SCALES_WITH_SERVICE,
-        automation_difficulty='easy',
-    ),
-]
-
-# Calculate total toil and prioritize automation
-total_weekly_hours = sum(item.weekly_hours for item in toil_items)
-print(f"Total weekly toil: {total_weekly_hours:.1f} hours")
-
-# Sort by ROI score to prioritize automation
-sorted_items = sorted(toil_items, key=lambda x: x.roi_score(), reverse=True)
-for item in sorted_items:
-    print(f"{item.name}: {item.roi_score():.1f} ROI score")
-```
-
-## Self-Healing Systems
-
-Automate common failure remediation.
-
-```python
-# auto_healing.py - Self-healing automation examples
-import subprocess
-import logging
-from typing import Callable, Dict
-from dataclasses import dataclass
-
-logger = logging.getLogger(__name__)
-
-@dataclass
-class HealthCheck:
-    """Define a health check and remediation."""
-    name: str
-    check: Callable[[], bool]
-    remediate: Callable[[], bool]
-    max_retries: int = 3
-
-class SelfHealer:
-    """Automatically remediate common failures."""
-
-    def __init__(self):
-        self.health_checks: Dict[str, HealthCheck] = {}
-
-    def register(self, check: HealthCheck):
-        """Register a health check with remediation."""
-        self.health_checks[check.name] = check
-
-    def run(self):
-        """Run all health checks and remediate failures."""
-        for name, check in self.health_checks.items():
-            if not check.check():
-                logger.warning(f"Health check failed: {name}")
-                self._remediate(check)
-
-    def _remediate(self, check: HealthCheck):
-        """Attempt remediation with retries."""
-        for attempt in range(check.max_retries):
-            logger.info(f"Remediation attempt {attempt + 1}/{check.max_retries}")
-
-            if check.remediate():
-                logger.info(f"Remediation successful: {check.name}")
-                return
-
-            if check.check():
-                logger.info(f"Health check passed after remediation: {check.name}")
-                return
-
-        logger.error(f"Remediation failed after {check.max_retries} attempts")
-        self._escalate(check)
-
-    def _escalate(self, check: HealthCheck):
-        """Escalate to on-call when auto-remediation fails."""
-        # Send alert to on-call
-        logger.error(f"ESCALATING: {check.name} - auto-remediation failed")
-
-# Example health checks
-def check_disk_space() -> bool:
-    """Check if disk space is above 20%."""
-    result = subprocess.run(
-        ["df", "-h", "/"],
-        capture_output=True,
-        text=True
-    )
-    # Parse df output and check available space
-    lines = result.stdout.strip().split('\n')
-    if len(lines) > 1:
-        fields = lines[1].split()
-        use_percent = int(fields[4].rstrip('%'))
-        return use_percent < 80
-    return True
-
-def cleanup_disk() -> bool:
-    """Clean up old log files."""
-    try:
-        # Delete logs older than 7 days
-        subprocess.run(
-            ["find", "/var/log", "-name", "*.log", "-mtime", "+7", "-delete"],
-            check=True
-        )
-        return True
-    except subprocess.CalledProcessError:
-        return False
-
-def check_service_responsive() -> bool:
-    """Check if service responds to health endpoint."""
-    try:
-        result = subprocess.run(
-            ["curl", "-f", "http://localhost:8080/health"],
-            capture_output=True,
-            timeout=5
-        )
-        return result.returncode == 0
-    except subprocess.TimeoutExpired:
-        return False
-
-def restart_service() -> bool:
-    """Restart the service."""
-    try:
-        subprocess.run(
-            ["systemctl", "restart", "myservice"],
-            check=True
-        )
-        return True
-    except subprocess.CalledProcessError:
-        return False
-
-# Set up self-healing
-healer = SelfHealer()
-healer.register(HealthCheck(
-    name="disk_space",
-    check=check_disk_space,
-    remediate=cleanup_disk,
-))
-healer.register(HealthCheck(
-    name="service_health",
-    check=check_service_responsive,
-    remediate=restart_service,
-))
-
-# Run as cron job or systemd timer
-if __name__ == "__main__":
-    healer.run()
-```
-
-## Runbook Automation
-
-Convert manual runbooks to automated scripts.
-
-```python
-# runbook_automation.py
-from typing import List, Tuple
-from dataclasses import dataclass
-import subprocess
-import json
-
-@dataclass
-class RunbookStep:
-    """A single step in a runbook."""
-    description: str
-    command: str
-    critical: bool = True  # Stop on failure?
-    verify: str | None = None  # Optional verification command
-
-class AutomatedRunbook:
-    """Execute runbook steps automatically."""
-
-    def __init__(self, name: str):
-        self.name = name
-        self.steps: List[RunbookStep] = []
-
-    def add_step(self, step: RunbookStep):
-        """Add a step to the runbook."""
-        self.steps.append(step)
-
-    def execute(self, dry_run: bool = False) -> Tuple[bool, List[str]]:
-        """Execute all runbook steps.
-
-        Args:
-            dry_run: If True, only print commands without executing
-
-        Returns:
-            tuple: (success, output_lines)
-        """
-        outputs = []
-
-        for i, step in enumerate(self.steps, 1):
-            outputs.append(f"\n[Step {i}/{len(self.steps)}] {step.description}")
-
-            if dry_run:
-                outputs.append(f"Would run: {step.command}")
-                continue
-
-            # Execute command
-            try:
-                result = subprocess.run(
-                    step.command,
-                    shell=True,
-                    capture_output=True,
-                    text=True,
-                    timeout=300,
-                )
-
-                if result.returncode != 0:
-                    outputs.append(f"ERROR: {result.stderr}")
-                    if step.critical:
-                        return False, outputs
-                else:
-                    outputs.append(result.stdout)
-
-                # Run verification if specified
-                if step.verify:
-                    verify_result = subprocess.run(
-                        step.verify,
-                        shell=True,
-                        capture_output=True,
-                        text=True,
-                    )
-                    if verify_result.returncode != 0:
-                        outputs.append(f"VERIFICATION FAILED: {verify_result.stderr}")
-                        if step.critical:
-                            return False, outputs
-
-            except subprocess.TimeoutExpired:
-                outputs.append(f"ERROR: Command timed out")
-                if step.critical:
-                    return False, outputs
-
-        return True, outputs
-
-# Example: Database failover runbook
-failover_runbook = AutomatedRunbook("Database Failover")
-
-failover_runbook.add_step(RunbookStep(
-    description="Stop writes to primary database",
-    command="kubectl exec -it postgres-primary-0 -- psql -c 'ALTER SYSTEM SET default_transaction_read_only = on;'",
-    critical=True,
-))
-
-failover_runbook.add_step(RunbookStep(
-    description="Wait for replication lag to clear",
-    command="sleep 10",
-    critical=False,
-))
-
-failover_runbook.add_step(RunbookStep(
-    description="Promote replica to primary",
-    command="kubectl exec -it postgres-replica-0 -- pg_ctl promote",
-    critical=True,
-    verify="kubectl exec -it postgres-replica-0 -- psql -c 'SELECT pg_is_in_recovery();' | grep -q 'f'",
-))
-
-failover_runbook.add_step(RunbookStep(
-    description="Update service to point to new primary",
-    command="kubectl patch service postgres -p '{\"spec\":{\"selector\":{\"role\":\"replica\"}}}'",
-    critical=True,
-))
-
-# Execute
-success, output = failover_runbook.execute(dry_run=False)
-print('\n'.join(output))
-```
-
-## Capacity Planning Automation
-
-```python
-# capacity_planner.py - Automated capacity planning
-from dataclasses import dataclass
-from datetime import datetime, timedelta
-import numpy as np
-
-@dataclass
-class CapacityMetrics:
-    """Historical capacity metrics."""
-    timestamp: datetime
-    requests_per_second: float
-    cpu_utilization: float
-    memory_utilization: float
-
-class CapacityPlanner:
-    """Automated capacity planning and forecasting."""
-
-    def __init__(self, metrics: list[CapacityMetrics]):
-        self.metrics = metrics
-
-    def forecast_growth(self, days_ahead: int = 90) -> dict:
-        """Forecast resource usage growth.
-
-        Uses linear regression on historical data.
-        """
-        # Extract time series
-        timestamps = [(m.timestamp - self.metrics[0].timestamp).days
-                      for m in self.metrics]
-        cpu_values = [m.cpu_utilization for m in self.metrics]
-        mem_values = [m.memory_utilization for m in self.metrics]
-
-        # Fit linear trend
-        cpu_trend = np.polyfit(timestamps, cpu_values, deg=1)
-        mem_trend = np.polyfit(timestamps, mem_values, deg=1)
-
-        # Forecast
-        future_day = timestamps[-1] + days_ahead
-        cpu_forecast = np.polyval(cpu_trend, future_day)
-        mem_forecast = np.polyval(mem_trend, future_day)
-
-        return {
-            'days_ahead': days_ahead,
-            'cpu_forecast': min(cpu_forecast, 1.0),
-            'memory_forecast': min(mem_forecast, 1.0),
-            'cpu_threshold_breach': cpu_forecast > 0.8,
-            'memory_threshold_breach': mem_forecast > 0.8,
-        }
-
-    def recommend_scaling(self, forecast: dict) -> str:
-        """Recommend scaling action based on forecast."""
-        if forecast['cpu_threshold_breach'] or forecast['memory_threshold_breach']:
-            return f"SCALE UP: Forecast shows >80% utilization in {forecast['days_ahead']} days"
-
-        return "OK: No scaling needed"
-
-# Example usage
-historical_metrics = [
-    CapacityMetrics(
-        timestamp=datetime.now() - timedelta(days=30),
-        requests_per_second=1000,
-        cpu_utilization=0.45,
-        memory_utilization=0.50,
-    ),
-    CapacityMetrics(
-        timestamp=datetime.now() - timedelta(days=15),
-        requests_per_second=1200,
-        cpu_utilization=0.55,
-        memory_utilization=0.60,
-    ),
-    CapacityMetrics(
-        timestamp=datetime.now(),
-        requests_per_second=1500,
-        cpu_utilization=0.65,
-        memory_utilization=0.70,
-    ),
-]
-
-planner = CapacityPlanner(historical_metrics)
-forecast = planner.forecast_growth(days_ahead=90)
-recommendation = planner.recommend_scaling(forecast)
-
-print(f"90-day forecast: CPU={forecast['cpu_forecast']:.1%}, Memory={forecast['memory_forecast']:.1%}")
-print(recommendation)
-```
-
-## Automation Testing
-
-```python
-# test_automation.py - Test automation scripts before production
-import unittest
-from unittest.mock import patch, MagicMock
-
-class TestSelfHealing(unittest.TestCase):
-    """Test self-healing automation."""
-
-    @patch('subprocess.run')
-    def test_disk_cleanup_success(self, mock_run):
-        """Test successful disk cleanup."""
-        mock_run.return_value = MagicMock(returncode=0)
-
-        result = cleanup_disk()
-
-        self.assertTrue(result)
-        mock_run.assert_called_once()
-
-    @patch('subprocess.run')
-    def test_service_restart_with_retry(self, mock_run):
-        """Test service restart retries on failure."""
-        # First attempt fails, second succeeds
-        mock_run.side_effect = [
-            MagicMock(returncode=1),  # First restart fails
-            MagicMock(returncode=0),  # Second restart succeeds
-        ]
-
-        # Implementation would retry on failure
-        # Assert retry logic works correctly
-```
-
-## Toil Reduction Metrics
-
-```python
-# Track toil reduction progress
-class ToilTracker:
-    """Track toil reduction over time."""
-
-    def __init__(self):
-        self.snapshots = []
-
-    def record_snapshot(self, week: int, toil_hours: float, team_hours: float):
-        """Record toil snapshot for a week."""
-        self.snapshots.append({
-            'week': week,
-            'toil_hours': toil_hours,
-            'team_hours': team_hours,
-            'toil_percentage': (toil_hours / team_hours * 100) if team_hours > 0 else 0,
-        })
-
-    def toil_trend(self) -> str:
-        """Calculate if toil is increasing or decreasing."""
-        if len(self.snapshots) < 2:
-            return "insufficient data"
-
-        first_pct = self.snapshots[0]['toil_percentage']
-        last_pct = self.snapshots[-1]['toil_percentage']
-
-        if last_pct < first_pct:
-            return f"improving ({first_pct:.1f}% → {last_pct:.1f}%)"
-        else:
-            return f"worsening ({first_pct:.1f}% → {last_pct:.1f}%)"
-
-# Target: <50% toil, ideally <30%
-tracker = ToilTracker()
-tracker.record_snapshot(week=1, toil_hours=30, team_hours=40)  # 75% toil
-tracker.record_snapshot(week=4, toil_hours=20, team_hours=40)  # 50% toil
-tracker.record_snapshot(week=8, toil_hours=12, team_hours=40)  # 30% toil
-
-print(f"Toil trend: {tracker.toil_trend()}")
-```