nucleus-mcp 1.0.7 → 1.2.0

package/src/mcp_server_nucleus/runtime/dashboard.py

@@ -1,1087 +0,0 @@
-"""
-DashboardEngine - Enterprise-Grade Orchestration Dashboard
-Real-time visibility into all NOP V3.1 components.
-
-Features:
-- 6 metric categories, 25+ metrics
-- ASCII, JSON, Mermaid output formats
-- Alert engine with configurable thresholds
-- Trend analysis with 7-day JSONL persistence
-- Snapshot creation and comparison
-- <100ms render time for 10K tasks
-- Graceful degradation on component failure
-
-Scales: 10K tasks, 100 agents, 100 concurrent viewers
-
-Author: NOP V3.1 - January 2026
-"""
-
-import json
-import time
-import threading
-import hashlib
-from pathlib import Path
-from typing import Dict, List, Optional, Any, Tuple
-from dataclasses import dataclass, field
-from enum import Enum
-from datetime import datetime, timedelta
-
-
-class AlertLevel(str, Enum):
-    """Alert severity levels."""
-    CRITICAL = "critical"
-    WARNING = "warning"
-    INFO = "info"
-
-
-class MetricCategory(str, Enum):
-    """Metric category types."""
-    AGENTS = "agents"
-    TASKS = "tasks"
-    INGESTION = "ingestion"
-    COST = "cost"
-    DEPS = "deps"
-    SYSTEM = "system"
-
-
-class OutputFormat(str, Enum):
-    """Supported output formats."""
-    ASCII = "ascii"
-    JSON = "json"
-    MERMAID = "mermaid"
-
-
-@dataclass
-class Alert:
-    """Alert data structure."""
-    level: AlertLevel
-    metric: str
-    message: str
-    value: float
-    threshold: float
-    timestamp: str = field(default_factory=lambda: time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()))
-
-    def to_dict(self) -> Dict:
-        return {
-            "level": self.level.value,
-            "metric": self.metric,
-            "message": self.message,
-            "value": self.value,
-            "threshold": self.threshold,
-            "timestamp": self.timestamp,
-        }
-
-
-@dataclass
-class Snapshot:
-    """Dashboard snapshot for comparison."""
-    id: str
-    name: str
-    timestamp: str
-    metrics: Dict[str, Any]
-    alerts: List[Dict]
-
-    def to_dict(self) -> Dict:
-        return {
-            "id": self.id,
-            "name": self.name,
-            "timestamp": self.timestamp,
-            "metrics": self.metrics,
-            "alerts": self.alerts,
-        }
-
-
-class MetricsCache:
-    """Cache with configurable TTL for dashboard metrics."""
-    
-    def __init__(self, ttl_ms: int = 100):
-        self.ttl_ms = ttl_ms
-        self.cache: Dict[str, Tuple[float, Any]] = {}
-        self.lock = threading.RLock()
-    
-    def get(self, key: str) -> Optional[Any]:
-        """Get cached value if not expired."""
-        with self.lock:
-            if key in self.cache:
-                ts, value = self.cache[key]
-                if (time.time() - ts) * 1000 < self.ttl_ms:
-                    return value
-                del self.cache[key]
-            return None
-    
-    def set(self, key: str, value: Any) -> None:
-        """Set cached value with current timestamp."""
-        with self.lock:
-            self.cache[key] = (time.time(), value)
-    
-    def invalidate(self, key: str = None) -> None:
-        """Invalidate cache entry or all entries."""
-        with self.lock:
-            if key:
-                self.cache.pop(key, None)
-            else:
-                self.cache.clear()
-
-
-class AlertEngine:
-    """Alert engine with configurable thresholds."""
-    
-    DEFAULT_THRESHOLDS = {
-        "agents.exhausted_ratio": {"warning": 0.5, "critical": 0.9},
-        "agents.utilization": {"warning": 0.9, "critical": 0.99},
-        "tasks.pending": {"warning": 100, "critical": 500},
-        "tasks.blocked_ratio": {"warning": 0.3, "critical": 0.5},
-        "cost.budget_remaining_ratio": {"warning": 0.2, "critical": 0.05},
-        "deps.max_depth": {"warning": 5, "critical": 10},
-        "deps.circular": {"warning": 1, "critical": 1},
-        "system.error_rate": {"warning": 5, "critical": 20},
-    }
-    
-    def __init__(self):
-        self.thresholds = dict(self.DEFAULT_THRESHOLDS)
-        self.active_alerts: List[Alert] = []
-    
-    def set_threshold(self, metric: str, level: str, value: float) -> None:
-        """Set custom threshold for metric."""
-        if metric not in self.thresholds:
-            self.thresholds[metric] = {}
-        self.thresholds[metric][level] = value
-    
-    def check(self, metrics: Dict[str, Any]) -> List[Alert]:
-        """Check all metrics against thresholds."""
-        alerts = []
-        
-        # Agent alerts
-        if "agents" in metrics:
-            agents = metrics["agents"]
-            total = agents.get("total", 0)
-            exhausted = agents.get("exhausted", 0)
-            
-            if total > 0:
-                exhausted_ratio = exhausted / total
-                alert = self._check_threshold(
-                    "agents.exhausted_ratio", exhausted_ratio,
-                    f"{int(exhausted_ratio*100)}% of agents exhausted"
-                )
-                if alert:
-                    alerts.append(alert)
-            
-            utilization = agents.get("utilization", 0)
-            alert = self._check_threshold(
-                "agents.utilization", utilization,
-                f"Agent pool at {int(utilization*100)}% capacity"
-            )
-            if alert:
-                alerts.append(alert)
-        
-        # Task alerts
-        if "tasks" in metrics:
-            tasks = metrics["tasks"]
-            pending = tasks.get("pending", 0)
-            
-            alert = self._check_threshold(
-                "tasks.pending", pending,
-                f"{pending} tasks pending in queue"
-            )
-            if alert:
-                alerts.append(alert)
-            
-            total = tasks.get("total", 0)
-            blocked = tasks.get("blocked", 0)
-            if total > 0:
-                blocked_ratio = blocked / total
-                alert = self._check_threshold(
-                    "tasks.blocked_ratio", blocked_ratio,
-                    f"{int(blocked_ratio*100)}% of tasks blocked"
-                )
-                if alert:
-                    alerts.append(alert)
-        
-        # Cost alerts
-        if "cost" in metrics:
-            cost = metrics["cost"]
-            budget = cost.get("budget", 0)
-            remaining = cost.get("remaining", 0)
-            
-            if budget > 0:
-                remaining_ratio = remaining / budget
-                alert = self._check_threshold(
-                    "cost.budget_remaining_ratio", remaining_ratio,
-                    f"Only {int(remaining_ratio*100)}% budget remaining",
-                    inverse=True  # Lower is worse
-                )
-                if alert:
-                    alerts.append(alert)
-        
-        # Dependency alerts
-        if "deps" in metrics:
-            deps = metrics["deps"]
-            max_depth = deps.get("max_depth", 0)
-            
-            alert = self._check_threshold(
-                "deps.max_depth", max_depth,
-                f"Dependency chain depth of {max_depth}"
-            )
-            if alert:
-                alerts.append(alert)
-            
-            circular = deps.get("circular", 0)
-            if circular > 0:
-                alerts.append(Alert(
-                    level=AlertLevel.CRITICAL,
-                    metric="deps.circular",
-                    message=f"{circular} circular dependencies detected!",
-                    value=circular,
-                    threshold=0,
-                ))
-        
-        self.active_alerts = alerts
-        return alerts
-    
-    def _check_threshold(
-        self, metric: str, value: float, message: str, inverse: bool = False
-    ) -> Optional[Alert]:
-        """Check single metric against thresholds."""
-        thresholds = self.thresholds.get(metric, {})
-        
-        critical_threshold = thresholds.get("critical")
-        warning_threshold = thresholds.get("warning")
-        
-        if inverse:
-            # Lower values are worse (e.g., budget remaining)
-            if critical_threshold is not None and value <= critical_threshold:
-                return Alert(AlertLevel.CRITICAL, metric, message, value, critical_threshold)
-            if warning_threshold is not None and value <= warning_threshold:
-                return Alert(AlertLevel.WARNING, metric, message, value, warning_threshold)
-        else:
-            # Higher values are worse (e.g., pending tasks)
-            if critical_threshold is not None and value >= critical_threshold:
-                return Alert(AlertLevel.CRITICAL, metric, message, value, critical_threshold)
-            if warning_threshold is not None and value >= warning_threshold:
-                return Alert(AlertLevel.WARNING, metric, message, value, warning_threshold)
-        
-        return None
-    
-    def get_active_alerts(self) -> List[Alert]:
-        """Get currently active alerts."""
-        return self.active_alerts
-
-
-class TrendAnalyzer:
-    """Trend analyzer with JSONL persistence."""
-    
-    def __init__(self, brain_path: Path = None, retention_days: int = 7):
-        self.brain_path = brain_path
-        self.retention_days = retention_days
-        self.metrics_file = None
-        if brain_path:
-            self.metrics_file = brain_path / "ledger" / "metrics.jsonl"
-    
-    def record_metrics(self, metrics: Dict[str, Any], interval: str = "hourly") -> None:
-        """Record metrics to JSONL file."""
-        if not self.metrics_file:
-            return
-        
-        self.metrics_file.parent.mkdir(parents=True, exist_ok=True)
-        
-        entry = {
-            "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
-            "interval": interval,
-            "metrics": self._flatten_metrics(metrics),
-        }
-        
-        with open(self.metrics_file, "a") as f:
-            f.write(json.dumps(entry) + "\n")
-        
-        self._cleanup_old_entries()
-    
-    def get_trends(self, metric: str, hours: int = 24) -> List[Dict]:
-        """Get trend data for a specific metric."""
-        if not self.metrics_file or not self.metrics_file.exists():
-            return []
-        
-        cutoff = datetime.utcnow() - timedelta(hours=hours)
-        trends = []
-        
-        with open(self.metrics_file) as f:
-            for line in f:
-                try:
-                    entry = json.loads(line.strip())
-                    ts = datetime.fromisoformat(entry["timestamp"].replace("Z", "+00:00"))
-                    if ts.replace(tzinfo=None) >= cutoff:
-                        value = entry["metrics"].get(metric)
-                        if value is not None:
-                            trends.append({
-                                "timestamp": entry["timestamp"],
-                                "value": value,
-                            })
-                except (json.JSONDecodeError, KeyError, ValueError):
-                    continue
-        
-        return trends
-    
-    def get_velocity(self, hours: int = 24) -> float:
-        """Calculate task completion velocity (tasks/hour)."""
-        trends = self.get_trends("tasks.done", hours)
-        if len(trends) < 2:
-            return 0.0
-        
-        first = trends[0]["value"]
-        last = trends[-1]["value"]
-        delta = last - first
-        
-        return delta / hours if hours > 0 else 0.0
-    
-    def _flatten_metrics(self, metrics: Dict, prefix: str = "") -> Dict[str, Any]:
-        """Flatten nested metrics dict to dot-notation keys."""
-        result = {}
-        for key, value in metrics.items():
-            full_key = f"{prefix}.{key}" if prefix else key
-            if isinstance(value, dict):
-                result.update(self._flatten_metrics(value, full_key))
-            else:
-                result[full_key] = value
-        return result
-    
-    def _cleanup_old_entries(self) -> None:
-        """Remove entries older than retention period."""
-        if not self.metrics_file or not self.metrics_file.exists():
-            return
-        
-        cutoff = datetime.utcnow() - timedelta(days=self.retention_days)
-        kept_lines = []
-        
-        with open(self.metrics_file) as f:
-            for line in f:
-                try:
-                    entry = json.loads(line.strip())
-                    ts = datetime.fromisoformat(entry["timestamp"].replace("Z", "+00:00"))
-                    if ts.replace(tzinfo=None) >= cutoff:
-                        kept_lines.append(line)
-                except (json.JSONDecodeError, KeyError, ValueError):
-                    continue
-        
-        with open(self.metrics_file, "w") as f:
-            f.writelines(kept_lines)
-
-
-class SnapshotManager:
-    """Manage dashboard snapshots for comparison."""
-    
-    def __init__(self, brain_path: Path = None, max_snapshots: int = 100):
-        self.brain_path = brain_path
-        self.max_snapshots = max_snapshots
-        self.snapshots_dir = None
-        if brain_path:
-            self.snapshots_dir = brain_path / "snapshots" / "dashboard"
-    
-    def create(self, metrics: Dict, alerts: List[Alert], name: str = None) -> Snapshot:
-        """Create a new snapshot."""
-        snapshot_id = f"snap_{int(time.time())}_{hashlib.sha256(str(time.time()).encode()).hexdigest()[:6]}"
-        timestamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
-        
-        snapshot = Snapshot(
-            id=snapshot_id,
-            name=name or f"Snapshot {timestamp}",
-            timestamp=timestamp,
-            metrics=metrics,
-            alerts=[a.to_dict() for a in alerts],
-        )
-        
-        if self.snapshots_dir:
-            self.snapshots_dir.mkdir(parents=True, exist_ok=True)
-            snapshot_file = self.snapshots_dir / f"{snapshot_id}.json"
-            with open(snapshot_file, "w") as f:
-                json.dump(snapshot.to_dict(), f, indent=2)
-            
-            self._cleanup_old_snapshots()
-        
-        return snapshot
-    
-    def get(self, snapshot_id: str) -> Optional[Snapshot]:
-        """Get a snapshot by ID."""
-        if not self.snapshots_dir:
-            return None
-        
-        snapshot_file = self.snapshots_dir / f"{snapshot_id}.json"
-        if not snapshot_file.exists():
-            return None
-        
-        with open(snapshot_file) as f:
-            data = json.load(f)
-        
-        return Snapshot(**data)
-    
-    def list(self, limit: int = 10) -> List[Dict]:
-        """List recent snapshots."""
-        if not self.snapshots_dir or not self.snapshots_dir.exists():
-            return []
-        
-        snapshots = []
-        for f in sorted(self.snapshots_dir.glob("snap_*.json"), reverse=True)[:limit]:
-            with open(f) as file:
-                data = json.load(file)
-                snapshots.append({
-                    "id": data["id"],
-                    "name": data["name"],
-                    "timestamp": data["timestamp"],
-                })
-        
-        return snapshots
-    
-    def compare(self, snapshot_a: str, snapshot_b: str) -> Dict:
-        """Compare two snapshots."""
-        a = self.get(snapshot_a)
-        b = self.get(snapshot_b)
-        
-        if not a or not b:
-            return {"error": "Snapshot not found"}
-        
-        comparison = {
-            "a": {"id": a.id, "timestamp": a.timestamp},
-            "b": {"id": b.id, "timestamp": b.timestamp},
-            "deltas": {},
-        }
-        
-        # Compare flat metrics
-        a_flat = self._flatten(a.metrics)
-        b_flat = self._flatten(b.metrics)
-        
-        all_keys = set(a_flat.keys()) | set(b_flat.keys())
-        for key in all_keys:
-            a_val = a_flat.get(key, 0)
-            b_val = b_flat.get(key, 0)
-            
-            if isinstance(a_val, (int, float)) and isinstance(b_val, (int, float)):
-                delta = b_val - a_val
-                if delta != 0:
-                    comparison["deltas"][key] = {
-                        "a": a_val,
-                        "b": b_val,
-                        "delta": delta,
-                        "direction": "up" if delta > 0 else "down",
-                    }
-        
-        return comparison
-    
-    def _flatten(self, d: Dict, prefix: str = "") -> Dict:
-        """Flatten nested dict."""
-        result = {}
-        for k, v in d.items():
-            key = f"{prefix}.{k}" if prefix else k
-            if isinstance(v, dict):
-                result.update(self._flatten(v, key))
-            else:
-                result[key] = v
-        return result
-    
-    def _cleanup_old_snapshots(self) -> None:
-        """Remove old snapshots beyond max limit."""
-        if not self.snapshots_dir:
-            return
-        
-        snapshots = sorted(self.snapshots_dir.glob("snap_*.json"))
-        while len(snapshots) > self.max_snapshots:
-            oldest = snapshots.pop(0)
-            oldest.unlink()
-
-
-class ASCIIFormatter:
-    """Format dashboard as ASCII art for terminal display."""
-    
-    LEVEL_ICONS = {
-        "minimal": "📊",
-        "standard": "🚀",
-        "verbose": "📈",
-        "full": "🔬",
-    }
-    
-    def format(
-        self,
-        metrics: Dict[str, Any],
-        alerts: List[Alert],
-        detail_level: str = "standard",
-    ) -> str:
-        """Format metrics as ASCII dashboard."""
-        lines = []
-        now = time.strftime("%Y-%m-%d %H:%M:%S")
-        icon = self.LEVEL_ICONS.get(detail_level, "📊")
-        
-        lines.append(f"{icon} NOP Status Dashboard - {now}")
-        lines.append("═" * 60)
-        
-        # Alerts section (always shown if any)
-        if alerts:
-            lines.append("")
-            lines.append("⚠️  ALERTS")
-            lines.append("─" * 40)
-            for alert in alerts:
-                icon = "🔴" if alert.level == AlertLevel.CRITICAL else "🟡"
-                lines.append(f"   {icon} [{alert.level.value.upper()}] {alert.message}")
-            lines.append("")
-        
-        # Agent Pool Health
-        if "agents" in metrics:
-            lines.extend(self._format_agents(metrics["agents"], detail_level))
-        
-        # Task Queue
-        if "tasks" in metrics:
-            lines.extend(self._format_tasks(metrics["tasks"], detail_level))
-        
-        # Ingestion (verbose/full only)
-        if "ingestion" in metrics and detail_level in ["verbose", "full"]:
-            lines.extend(self._format_ingestion(metrics["ingestion"]))
-        
-        # Cost Tracking
-        if "cost" in metrics:
-            lines.extend(self._format_cost(metrics["cost"], detail_level))
-        
-        # Dependencies (verbose/full only)
-        if "deps" in metrics and detail_level in ["verbose", "full"]:
-            lines.extend(self._format_deps(metrics["deps"]))
-        
-        # System Health (full only)
-        if "system" in metrics and detail_level == "full":
-            lines.extend(self._format_system(metrics["system"]))
-        
-        return "\n".join(lines)
-    
-    def _format_agents(self, agents: Dict, detail_level: str) -> List[str]:
-        """Format agent pool section."""
-        lines = ["", "📊 AGENT POOL HEALTH"]
-        lines.append("─" * 40)
-        
-        total = agents.get("total", 0)
-        active = agents.get("active", 0)
-        idle = agents.get("idle", 0)
-        exhausted = agents.get("exhausted", 0)
-        utilization = agents.get("utilization", 0)
-        
-        pct = f"({int(utilization*100)}%)" if total > 0 else ""
-        lines.append(f"   ├── Active: {active}/{total} {pct}")
-        lines.append(f"   ├── Idle: {idle}")
-        lines.append(f"   ├── Exhausted: {exhausted}")
-        
-        if detail_level in ["verbose", "full"]:
-            reset_warnings = agents.get("reset_warnings", [])
-            if reset_warnings:
-                lines.append(f"   └── Reset Warnings: {len(reset_warnings)}")
-                for w in reset_warnings[:3]:
-                    lines.append(f"       └── {w.get('slot_id', 'unknown')} in {w.get('minutes', '?')}m")
-            else:
-                lines.append("   └── Reset Warnings: 0")
-        else:
-            lines.append(f"   └── Capacity: {int(utilization*100)}%")
-        
-        return lines
-    
-    def _format_tasks(self, tasks: Dict, detail_level: str) -> List[str]:
-        """Format task queue section."""
-        lines = ["", "📋 TASK QUEUE"]
-        lines.append("─" * 40)
-        
-        pending = tasks.get("pending", 0)
-        in_progress = tasks.get("in_progress", 0)
-        blocked = tasks.get("blocked", 0)
-        done = tasks.get("done", 0)
-        failed = tasks.get("failed", 0)
-        velocity = tasks.get("velocity", 0)
-        
-        lines.append(f"   ├── Pending: {pending}")
-        lines.append(f"   ├── In Progress: {in_progress}")
-        lines.append(f"   ├── Blocked: {blocked}")
-        
-        if detail_level in ["verbose", "full"]:
-            lines.append(f"   ├── Done (24h): {done}")
-            lines.append(f"   ├── Failed: {failed}")
-            lines.append(f"   └── Velocity: {velocity:.1f}/hr")
-        else:
-            lines.append(f"   └── Done (24h): {done}")
-        
-        return lines
-    
-    def _format_ingestion(self, ingestion: Dict) -> List[str]:
-        """Format ingestion section."""
-        lines = ["", "📥 INGESTION"]
-        lines.append("─" * 40)
-        
-        total = ingestion.get("total", 0)
-        skipped = ingestion.get("skipped", 0)
-        failed = ingestion.get("failed", 0)
-        batches = ingestion.get("batches", 0)
-        
-        lines.append(f"   ├── Total Ingested: {total}")
-        lines.append(f"   ├── Skipped (Dupes): {skipped}")
-        lines.append(f"   ├── Failed: {failed}")
-        lines.append(f"   └── Batches: {batches}")
-        
-        return lines
-    
-    def _format_cost(self, cost: Dict, detail_level: str) -> List[str]:
-        """Format cost tracking section."""
-        lines = ["", "💰 COST TRACKING"]
-        lines.append("─" * 40)
-        
-        tokens = cost.get("tokens", 0)
-        usd = cost.get("usd", 0)
-        budget = cost.get("budget", 0)
-        remaining = cost.get("remaining", 0)
-        burn_rate = cost.get("burn_rate", 0)
-        
-        # Format tokens
-        if tokens >= 1_000_000:
-            tokens_str = f"{tokens/1_000_000:.1f}M"
-        elif tokens >= 1_000:
-            tokens_str = f"{tokens/1_000:.1f}K"
-        else:
-            tokens_str = str(tokens)
-        
-        lines.append(f"   ├── Tokens: {tokens_str}")
-        lines.append(f"   ├── Estimated: ${usd:.2f}")
-        
-        if budget > 0:
-            pct = int((remaining / budget) * 100)
-            lines.append(f"   ├── Budget: ${budget:.2f} ({pct}% remaining)")
-        
-        if detail_level in ["verbose", "full"]:
-            lines.append(f"   └── Burn Rate: ${burn_rate:.2f}/hr")
-        else:
-            lines.append(f"   └── Remaining: ${remaining:.2f}")
-        
-        return lines
-    
-    def _format_deps(self, deps: Dict) -> List[str]:
-        """Format dependency section."""
-        lines = ["", "🔗 DEPENDENCIES"]
-        lines.append("─" * 40)
-        
-        max_depth = deps.get("max_depth", 0)
-        blocked_chains = deps.get("blocked_chains", 0)
-        circular = deps.get("circular", 0)
-        
-        lines.append(f"   ├── Max Depth: {max_depth}")
-        lines.append(f"   ├── Blocked Chains: {blocked_chains}")
-        lines.append(f"   └── Circular: {circular}")
-        
-        return lines
-    
-    def _format_system(self, system: Dict) -> List[str]:
-        """Format system health section."""
-        lines = ["", "🖥️  SYSTEM HEALTH"]
-        lines.append("─" * 40)
-        
-        uptime = system.get("uptime", "N/A")
-        last_activity = system.get("last_activity", "N/A")
-        error_rate = system.get("error_rate", 0)
-        
-        lines.append(f"   ├── Uptime: {uptime}")
-        lines.append(f"   ├── Last Activity: {last_activity}")
-        lines.append(f"   └── Error Rate: {error_rate}/hr")
-        
-        return lines
-
-
-class JSONFormatter:
-    """Format dashboard as JSON for API consumption."""
-    
-    def format(
-        self,
-        metrics: Dict[str, Any],
-        alerts: List[Alert],
-        detail_level: str = "standard",
-    ) -> str:
-        """Format metrics as JSON."""
-        output = {
-            "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
-            "detail_level": detail_level,
-            "metrics": metrics,
-            "alerts": [a.to_dict() for a in alerts],
-        }
-        return json.dumps(output, indent=2)
-
-
-class MermaidFormatter:
-    """Format dependency graph as Mermaid diagram."""
-    
-    def format(self, deps: Dict) -> str:
-        """Format dependencies as Mermaid diagram."""
-        lines = ["```mermaid", "graph TD"]
-        
-        forward_deps = deps.get("forward_deps", {})
-        depths = deps.get("depths", {})
-        
-        # Add nodes with depth-based styling
-        nodes_added = set()
-        for task_id, task_deps in forward_deps.items():
-            if task_id not in nodes_added:
-                depth = depths.get(task_id, 0)
-                style = self._get_node_style(depth)
-                lines.append(f"    {self._safe_id(task_id)}{style}")
-                nodes_added.add(task_id)
-            
-            for dep_id in task_deps:
-                if dep_id not in nodes_added:
-                    depth = depths.get(dep_id, 0)
-                    style = self._get_node_style(depth)
-                    lines.append(f"    {self._safe_id(dep_id)}{style}")
-                    nodes_added.add(dep_id)
-                
-                lines.append(f"    {self._safe_id(dep_id)} --> {self._safe_id(task_id)}")
-        
-        lines.append("```")
-        return "\n".join(lines)
-    
-    def _safe_id(self, task_id: str) -> str:
-        """Convert task ID to safe Mermaid node ID."""
-        return task_id.replace("-", "_").replace(".", "_")
-    
-    def _get_node_style(self, depth: int) -> str:
-        """Get node style based on depth."""
-        if depth == 0:
-            return "[🟢 Root]"
-        elif depth <= 2:
-            return f"[Depth {depth}]"
-        elif depth <= 4:
-            return f"[⚠️ Depth {depth}]"
-        else:
-            return f"[🔴 Depth {depth}]"
-
-
-class MetricsCollector:
-    """Collect metrics from all NOP V3.1 components."""
-    
-    def __init__(self, orchestrator=None, brain_path: Path = None):
-        self.orch = orchestrator
-        self.brain_path = brain_path
-        self.cache = MetricsCache(ttl_ms=100)
-    
-    def collect_all(self, use_cache: bool = True) -> Dict[str, Any]:
-        """Collect all metrics from all components."""
-        if use_cache:
-            cached = self.cache.get("all_metrics")
-            if cached:
-                return cached
-        
-        metrics = {
-            "agents": self.collect_agent_metrics(),
-            "tasks": self.collect_task_metrics(),
-            "ingestion": self.collect_ingestion_metrics(),
-            "cost": self.collect_cost_metrics(),
-            "deps": self.collect_dependency_metrics(),
-            "system": self.collect_system_metrics(),
-        }
-        
-        if use_cache:
-            self.cache.set("all_metrics", metrics)
-        
-        return metrics
-    
-    def collect_agent_metrics(self) -> Dict:
-        """Collect agent pool metrics."""
-        try:
-            if self.orch:
-                pool = self.orch.get_agent_pool()
-                if pool:
-                    pool_metrics = pool.get_pool_metrics() if hasattr(pool, 'get_pool_metrics') else {}
-                    return {
-                        "total": pool_metrics.get("total_agents", 0),
-                        "active": pool_metrics.get("active_agents", 0),
-                        "idle": pool_metrics.get("idle_agents", 0),
-                        "exhausted": pool_metrics.get("exhausted_agents", 0),
-                        "utilization": pool_metrics.get("utilization", 0),
-                        "reset_warnings": [],
-                    }
-            
-            # Fallback: read from registry
-            if self.brain_path:
-                registry_path = self.brain_path / "slots" / "registry.json"
-                if registry_path.exists():
-                    with open(registry_path) as f:
-                        registry = json.load(f)
-                    
-                    slots = registry.get("slots", {})
-                    total = len(slots)
-                    active = sum(1 for s in slots.values() if s.get("status") == "active")
-                    exhausted = sum(1 for s in slots.values() if s.get("status") == "exhausted")
-                    idle = active  # Approximate
-                    
-                    return {
-                        "total": total,
-                        "active": active,
-                        "idle": idle,
-                        "exhausted": exhausted,
-                        "utilization": active / total if total > 0 else 0,
-                        "reset_warnings": [],
-                    }
-            
-            return {"error": "Agent metrics unavailable"}
-        except Exception as e:
-            return {"error": str(e)}
-    
-    def collect_task_metrics(self) -> Dict:
-        """Collect task queue metrics."""
-        try:
-            if self.orch:
-                pool_metrics = self.orch.get_pool_metrics()
-                return {
-                    "total": pool_metrics.get("total_tasks", 0),
-                    "pending": pool_metrics.get("pending", 0),
-                    "in_progress": pool_metrics.get("in_progress", 0),
-                    "blocked": pool_metrics.get("blocked", 0),
-                    "done": pool_metrics.get("done", 0),
-                    "failed": pool_metrics.get("failed", 0),
-                    "velocity": 0,  # Calculated from trends
-                }
-            
-            # Fallback: read from tasks.json
-            if self.brain_path:
-                tasks_path = self.brain_path / "ledger" / "tasks.json"
-                if tasks_path.exists():
-                    with open(tasks_path) as f:
-                        data = json.load(f)
-                    
-                    tasks = data.get("tasks", [])
-                    return {
-                        "total": len(tasks),
-                        "pending": sum(1 for t in tasks if t.get("status") in ["PENDING", "READY"]),
-                        "in_progress": sum(1 for t in tasks if t.get("status") == "IN_PROGRESS"),
-                        "blocked": sum(1 for t in tasks if t.get("status") == "BLOCKED"),
-                        "done": sum(1 for t in tasks if t.get("status") == "DONE"),
-                        "failed": sum(1 for t in tasks if t.get("status") == "FAILED"),
-                        "velocity": 0,
-                    }
-            
-            return {"error": "Task metrics unavailable"}
-        except Exception as e:
-            return {"error": str(e)}
-    
-    def collect_ingestion_metrics(self) -> Dict:
-        """Collect ingestion statistics."""
-        try:
-            if self.orch:
-                stats = self.orch.get_ingestion_stats()
-                return {
-                    "total": stats.get("total_ingested", 0),
-                    "skipped": stats.get("total_skipped", 0),
-                    "failed": stats.get("total_failed", 0),
-                    "batches": stats.get("batches_count", 0),
-                    "by_source": stats.get("by_source", {}),
-                }
-            
-            return {
-                "total": 0,
-                "skipped": 0,
-                "failed": 0,
-                "batches": 0,
-                "by_source": {},
-            }
-        except Exception as e:
-            return {"error": str(e)}
-    
-    def collect_cost_metrics(self) -> Dict:
-        """Collect cost tracking metrics."""
-        # This would integrate with LLM client cost tracking
-        return {
-            "tokens": 0,
-            "usd": 0.0,
-            "budget": 10.0,
-            "remaining": 10.0,
-            "burn_rate": 0.0,
-        }
-    
-    def collect_dependency_metrics(self) -> Dict:
-        """Collect dependency graph metrics."""
-        try:
-            if self.orch:
-                graph = self.orch.get_dependency_graph()
-                depths = graph.get("depths", {})
-                
-                max_depth = max(depths.values()) if depths else 0
-                circular = sum(1 for d in depths.values() if d < 0)
-                blocked_chains = sum(1 for d in depths.values() if d > 3)
-                
-                return {
-                    "max_depth": max_depth,
-                    "blocked_chains": blocked_chains,
-                    "circular": circular,
-                }
-            
-            return {"max_depth": 0, "blocked_chains": 0, "circular": 0}
-        except Exception as e:
-            return {"error": str(e)}
-    
-    def collect_system_metrics(self) -> Dict:
-        """Collect system health metrics."""
-        return {
-            "uptime": "N/A",
-            "last_activity": time.strftime("%Y-%m-%d %H:%M:%S"),
-            "error_rate": 0,
-        }
-
-
-class DashboardEngine:
-    """
-    Enterprise-grade orchestration dashboard.
-    
-    Provides real-time visibility into all NOP V3.1 components
-    with multiple output formats, alerting, and trend analysis.
-    """
-    
-    def __init__(
-        self,
-        orchestrator=None,
-        brain_path: Path = None,
-    ):
-        self.orch = orchestrator
-        self.brain_path = brain_path
-        
-        # Initialize components
-        self.collector = MetricsCollector(orchestrator, brain_path)
-        self.alert_engine = AlertEngine()
-        self.trend_analyzer = TrendAnalyzer(brain_path)
-        self.snapshot_manager = SnapshotManager(brain_path)
-        
-        # Initialize formatters
-        self.formatters = {
-            OutputFormat.ASCII: ASCIIFormatter(),
-            OutputFormat.JSON: JSONFormatter(),
-            OutputFormat.MERMAID: MermaidFormatter(),
-        }
-    
-    def render(
-        self,
-        detail_level: str = "standard",
-        format: str = "ascii",
-        include_alerts: bool = True,
-        include_trends: bool = False,
-        compare_to: str = None,
-        category: str = None,
-    ) -> str:
-        """
-        Render the dashboard in the specified format.
-        
-        Args:
-            detail_level: "minimal", "standard", "verbose", "full"
-            format: "ascii", "json", "mermaid"
-            include_alerts: Include alert section
-            include_trends: Include trend data
-            compare_to: Snapshot ID or time offset ("1h", "24h")
-            category: Filter to specific category
-        
-        Returns:
-            Formatted dashboard string
-        """
-        # Collect metrics
-        metrics = self.collector.collect_all()
-        
-        # Filter by category if specified
-        if category and category in metrics:
-            metrics = {category: metrics[category]}
-        
-        # Add trends if requested
-        if include_trends:
-            velocity = self.trend_analyzer.get_velocity(hours=24)
-            if "tasks" in metrics:
-                metrics["tasks"]["velocity"] = velocity
-        
-        # Check alerts
-        alerts = self.alert_engine.check(metrics) if include_alerts else []
-        
-        # Handle comparison
-        if compare_to:
-            return self._render_comparison(metrics, compare_to)
-        
-        # Format output
-        output_format = OutputFormat(format.lower())
-        
-        if output_format == OutputFormat.MERMAID:
-            deps = metrics.get("deps", {})
-            if self.orch:
-                deps = self.orch.get_dependency_graph()
-            return self.formatters[output_format].format(deps)
-        
-        formatter = self.formatters.get(output_format, self.formatters[OutputFormat.ASCII])
-        return formatter.format(metrics, alerts, detail_level)
-    
-    def _render_comparison(self, current_metrics: Dict, compare_to: str) -> str:
-        """Render comparison view."""
-        # Parse compare_to
-        if compare_to.endswith("h"):
-            # Time offset comparison
-            hours = int(compare_to[:-1])
-            # For now, just show current - would need historical snapshots
-            lines = [
-                f"📊 Dashboard Comparison: Now vs {hours}h ago",
-                "═" * 50,
-                "(Historical comparison requires snapshot data)",
-            ]
-            return "\n".join(lines)
-        else:
-            # Snapshot comparison
-            comparison = self.snapshot_manager.compare("current", compare_to)
-            if "error" in comparison:
-                return f"❌ {comparison['error']}"
-            
-            lines = [
-                "📊 Dashboard Comparison",
-                "═" * 50,
-            ]
-            for key, delta in comparison.get("deltas", {}).items():
-                direction = "✅" if delta["direction"] == "up" else "⬇️"
-                lines.append(f"   {key}: {delta['a']} → {delta['b']} ({direction} {abs(delta['delta'])})")
-            
-            return "\n".join(lines)
-    
-    def get_metrics(self, category: str = None) -> Dict:
-        """Get raw metrics dictionary."""
-        metrics = self.collector.collect_all()
-        if category and category in metrics:
-            return {category: metrics[category]}
-        return metrics
-    
-    def get_alerts(self) -> List[Alert]:
-        """Get current active alerts."""
-        metrics = self.collector.collect_all()
-        return self.alert_engine.check(metrics)
-    
-    def create_snapshot(self, name: str = None) -> Snapshot:
-        """Create a manual snapshot."""
-        metrics = self.collector.collect_all(use_cache=False)
-        alerts = self.alert_engine.check(metrics)
-        return self.snapshot_manager.create(metrics, alerts, name)
-    
-    def compare_snapshots(self, snapshot_a: str, snapshot_b: str) -> Dict:
-        """Compare two snapshots."""
-        return self.snapshot_manager.compare(snapshot_a, snapshot_b)
-    
-    def list_snapshots(self, limit: int = 10) -> List[Dict]:
-        """List available snapshots."""
-        return self.snapshot_manager.list(limit)
-    
-    def set_alert_threshold(self, metric: str, level: str, value: float) -> None:
-        """Set custom alert threshold."""
-        self.alert_engine.set_threshold(metric, level, value)
-    
-    def record_hourly_metrics(self) -> None:
-        """Record current metrics for trend analysis."""
-        metrics = self.collector.collect_all(use_cache=False)
-        self.trend_analyzer.record_metrics(metrics, interval="hourly")
-    
-    def get_trends(self, metric: str, hours: int = 24) -> List[Dict]:
-        """Get trend data for a metric."""
-        return self.trend_analyzer.get_trends(metric, hours)
-
-
-def format_dashboard(
-    metrics: Dict,
-    alerts: List[Alert],
-    detail_level: str = "standard",
-    format: str = "ascii",
-) -> str:
-    """Standalone function for formatting dashboard output."""
-    if format.lower() == "ascii":
-        return ASCIIFormatter().format(metrics, alerts, detail_level)
-    elif format.lower() == "json":
-        return JSONFormatter().format(metrics, alerts, detail_level)
-    else:
-        return ASCIIFormatter().format(metrics, alerts, detail_level)