anvil-dev-framework 0.1.7 → 0.1.9

package/global/lib/token_analyzer.py

@@ -0,0 +1,1383 @@
+"""
+Token Analyzer Service for Anvil Framework.
+
+Provides analysis and recommendation generation for token consumption data.
+Used by /audit and /efficiency commands.
+
+Usage:
+    from token_analyzer import TokenAnalyzer
+
+    analyzer = TokenAnalyzer()
+    analysis = analyzer.analyze_session()
+    recommendations = analyzer.generate_recommendations(analysis)
+"""
+
+import logging
+from dataclasses import dataclass, field
+from typing import Optional, Dict, Any, List
+from datetime import datetime, timezone
+
+from token_metrics import TokenMetrics, SessionSummary
+
+# Module logger
+logger = logging.getLogger(__name__)
+
+
+# Claude context limits (approximate)
+CONTEXT_LIMIT_TOKENS = 200000
+EFFECTIVE_LIMIT_TOKENS = 150000  # Leave room for responses
+
+# Component types that provide value through execution, not reference.
+# These should not be flagged as "unused" even if was_used=FALSE.
+EXECUTION_VALUE_TYPES = {"hook", "context"}
+
+
+@dataclass
+class WastePattern:
+    """A detected inefficiency pattern."""
+    pattern_type: str  # redundant_load, unused_component, high_cost_low_value
+    component_name: str
+    component_type: str
+    tokens_wasted: int
+    occurrences: int
+    description: str
+
+
+@dataclass
+class Recommendation:
+    """An actionable optimization suggestion."""
+    priority: int  # 1=high, 2=medium, 3=low
+    category: str  # reduce, defer, remove, optimize
+    title: str
+    description: str
+    estimated_savings: int  # tokens
+    action: Optional[str] = None  # specific action to take
+
+
+@dataclass
+class SessionAnalysis:
+    """Complete analysis of a session's token consumption."""
+    session_id: str
+    summary: SessionSummary
+    context_percent_used: float
+    breakdown_by_type: Dict[str, Dict[str, Any]]
+    top_consumers: List[Dict[str, Any]]
+    waste_patterns: List[WastePattern]
+    efficiency_score: float  # 0-100
+    analyzed_at: datetime = field(default_factory=datetime.now)
+
+
+@dataclass
+class ComponentEfficiency:
+    """Efficiency metrics for a single component."""
+    component_name: str
+    component_type: str
+    total_tokens: int
+    load_count: int
+    used_count: int
+    utilization_rate: float
+    efficiency_score: float
+    avg_tokens_per_load: float
+    trend: str  # improving, stable, degrading, new
+
+
+@dataclass
+class EfficiencyReport:
+    """Historical efficiency analysis report."""
+    period_days: int
+    generated_at: datetime
+    total_sessions: int
+    total_tokens: int
+    avg_tokens_per_session: float
+    overall_efficiency_score: float
+    component_scores: List["ComponentEfficiency"]
+    trends: Dict[str, str]
+    recommendations: List[Recommendation]
+    comparison_to_previous: Optional[Dict[str, float]] = None
+
+
+class TokenAnalyzer:
+    """
+    Token consumption analysis service.
+
+    Analyzes session data to detect waste patterns and generate
+    actionable optimization recommendations.
+    """
+
+    def __init__(self, metrics: Optional[TokenMetrics] = None):
+        """
+        Initialize the analyzer.
+
+        Args:
+            metrics: TokenMetrics instance to use (creates one if not provided)
+        """
+        self.metrics = metrics or TokenMetrics()
+
+    # =========================================================================
+    # Session Analysis
+    # =========================================================================
+
+    def analyze_session(self, session_id: Optional[str] = None) -> Optional[SessionAnalysis]:
+        """
+        Perform comprehensive analysis of a session.
+
+        Args:
+            session_id: Session to analyze (defaults to current)
+
+        Returns:
+            SessionAnalysis with all metrics and patterns
+        """
+        summary = self.metrics.get_session_summary(session_id)
+        if not summary:
+            return None
+
+        # Calculate context percentage
+        context_percent = (summary.total_tokens / EFFECTIVE_LIMIT_TOKENS) * 100
+
+        # Build detailed breakdown by type
+        breakdown = self._build_type_breakdown(summary)
+
+        # Detect waste patterns
+        waste_patterns = self.detect_waste_patterns(summary.session_id)
+
+        # Calculate efficiency score
+        efficiency_score = self._calculate_efficiency_score(summary, waste_patterns)
+
+        return SessionAnalysis(
+            session_id=summary.session_id,
+            summary=summary,
+            context_percent_used=round(context_percent, 1),
+            breakdown_by_type=breakdown,
+            top_consumers=summary.top_consumers,
+            waste_patterns=waste_patterns,
+            efficiency_score=efficiency_score
+        )
+
+    def analyze_by_component_type(
+        self,
+        component_type: str,
+        session_id: Optional[str] = None
+    ) -> Dict[str, Any]:
+        """
+        Analyze token consumption for a specific component type.
+
+        Args:
+            component_type: Type to analyze (command, hook, tool, etc.)
+            session_id: Session to analyze
+
+        Returns:
+            Dict with type-specific metrics
+        """
+        sid = session_id or self.metrics.current_session_id
+        if not sid:
+            return {"error": "No active session"}
+
+        with self.metrics._get_connection() as conn:
+            cursor = conn.cursor()
+
+            # Get all loads of this type
+            cursor.execute("""
+                SELECT component_name, tokens, source, was_used, loaded_at
+                FROM component_loads
+                WHERE session_id = ? AND component_type = ?
+                ORDER BY tokens DESC
+            """, (sid, component_type))
+
+            components = []
+            total_tokens = 0
+            used_count = 0
+
+            for row in cursor.fetchall():
+                components.append({
+                    "name": row["component_name"],
+                    "tokens": row["tokens"],
+                    "source": row["source"],
+                    "was_used": bool(row["was_used"]),
+                    "loaded_at": row["loaded_at"]
+                })
+                total_tokens += row["tokens"]
+                if row["was_used"]:
+                    used_count += 1
+
+            utilization = (used_count / len(components) * 100) if components else 0
+
+            return {
+                "component_type": component_type,
+                "total_tokens": total_tokens,
+                "component_count": len(components),
+                "used_count": used_count,
+                "utilization_percent": round(utilization, 1),
+                "components": components
+            }
+
+    def _build_type_breakdown(self, summary: SessionSummary) -> Dict[str, Dict[str, Any]]:
+        """Build detailed breakdown for each component type."""
+        breakdown = {}
+        total = summary.total_tokens or 1  # Avoid division by zero
+
+        for comp_type, tokens in summary.component_breakdown.items():
+            percent = (tokens / total) * 100
+            breakdown[comp_type] = {
+                "tokens": tokens,
+                "percent": round(percent, 1),
+                "count": self._count_components_of_type(
+                    summary.session_id, comp_type
+                )
+            }
+
+        # Add tool calls as a type
+        if summary.tool_tokens > 0:
+            breakdown["tools"] = {
+                "tokens": summary.tool_tokens,
+                "percent": round((summary.tool_tokens / total) * 100, 1),
+                "count": summary.tool_calls_count
+            }
+
+        return breakdown
+
+    def _count_components_of_type(self, session_id: str, comp_type: str) -> int:
+        """Count unique components of a type in a session."""
+        with self.metrics._get_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute("""
+                SELECT COUNT(DISTINCT component_name)
+                FROM component_loads
+                WHERE session_id = ? AND component_type = ?
+            """, (session_id, comp_type))
+            return cursor.fetchone()[0]
+
+    # =========================================================================
+    # Waste Detection
+    # =========================================================================
+
+    def detect_waste_patterns(self, session_id: Optional[str] = None) -> List[WastePattern]:
+        """
+        Detect inefficiency patterns in a session.
+
+        Patterns detected:
+        - Redundant loads: Same component loaded multiple times
+        - Unused components: Loaded but never used
+        - High-cost low-value: Large components with low utilization
+
+        Args:
+            session_id: Session to analyze
+
+        Returns:
+            List of detected waste patterns
+        """
+        sid = session_id or self.metrics.current_session_id
+        if not sid:
+            return []
+
+        patterns = []
+
+        # Detect redundant loads
+        patterns.extend(self._detect_redundant_loads(sid))
+
+        # Detect unused components
+        patterns.extend(self._detect_unused_components(sid))
+
+        # Detect high-cost low-value
+        patterns.extend(self._detect_high_cost_low_value(sid))
+
+        # Sort by tokens wasted (highest first)
+        patterns.sort(key=lambda p: p.tokens_wasted, reverse=True)
+
+        return patterns
+
+    def _detect_redundant_loads(self, session_id: str) -> List[WastePattern]:
+        """Find components loaded multiple times in same session."""
+        patterns = []
+
+        with self.metrics._get_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute("""
+                SELECT component_type, component_name,
+                       COUNT(*) as load_count, SUM(tokens) as total_tokens
+                FROM component_loads
+                WHERE session_id = ?
+                GROUP BY component_type, component_name
+                HAVING COUNT(*) > 1
+            """, (session_id,))
+
+            for row in cursor.fetchall():
+                # Wasted = total - (one load's worth)
+                avg_tokens = row["total_tokens"] // row["load_count"]
+                wasted = row["total_tokens"] - avg_tokens
+
+                patterns.append(WastePattern(
+                    pattern_type="redundant_load",
+                    component_name=row["component_name"],
+                    component_type=row["component_type"],
+                    tokens_wasted=wasted,
+                    occurrences=row["load_count"],
+                    description=f"Loaded {row['load_count']} times, costing {wasted:,} extra tokens"
+                ))
+
+        return patterns
+
+    def _detect_unused_components(self, session_id: str) -> List[WastePattern]:
+        """Find components that were loaded but never used."""
+        patterns = []
+
+        with self.metrics._get_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute("""
+                SELECT component_type, component_name, SUM(tokens) as total_tokens
+                FROM component_loads
+                WHERE session_id = ? AND was_used = FALSE
+                GROUP BY component_type, component_name
+            """, (session_id,))
+
+            for row in cursor.fetchall():
+                # Skip execution-value components (hooks, context)
+                # These provide value through running, not through being referenced
+                if row["component_type"] in EXECUTION_VALUE_TYPES:
+                    continue
+
+                patterns.append(WastePattern(
+                    pattern_type="unused_component",
+                    component_name=row["component_name"],
+                    component_type=row["component_type"],
+                    tokens_wasted=row["total_tokens"],
+                    occurrences=1,
+                    description=f"Loaded but never used, wasting {row['total_tokens']:,} tokens"
+                ))
+
+        return patterns
+
+    def _detect_high_cost_low_value(self, session_id: str) -> List[WastePattern]:
+        """Find large components with low utilization (historical analysis)."""
+        patterns = []
+
+        # Get component stats over 30 days
+        stats = self.metrics.get_component_stats(days=30)
+
+        for stat in stats:
+            # Skip if not enough data
+            if stat["load_count"] < 5:
+                continue
+
+            # Calculate utilization rate
+            utilization = stat["used_count"] / stat["load_count"]
+
+            # Flag if utilization < 30% and avg tokens > 500
+            if utilization < 0.30 and stat["avg_tokens"] > 500:
+                potential_waste = int(stat["avg_tokens"] * (1 - utilization))
+
+                patterns.append(WastePattern(
+                    pattern_type="high_cost_low_value",
+                    component_name=stat["component_name"],
+                    component_type=stat["component_type"],
+                    tokens_wasted=potential_waste,
+                    occurrences=stat["load_count"],
+                    description=f"Only used {utilization*100:.0f}% of the time, avg {stat['avg_tokens']:.0f} tokens"
+                ))
+
+        return patterns
+
+    # =========================================================================
+    # Recommendations
+    # =========================================================================
+
+    def generate_recommendations(
+        self,
+        analysis: Optional[SessionAnalysis] = None
+    ) -> List[Recommendation]:
+        """
+        Generate actionable optimization recommendations.
+
+        Args:
+            analysis: SessionAnalysis to base recommendations on
+
+        Returns:
+            List of prioritized recommendations
+        """
+        if analysis is None:
+            analysis = self.analyze_session()
+
+        if analysis is None:
+            return []
+
+        recommendations = []
+
+        # Add recommendations based on waste patterns
+        for pattern in analysis.waste_patterns:
+            rec = self._recommendation_for_pattern(pattern)
+            if rec:
+                recommendations.append(rec)
+
+        # Add recommendations based on context size
+        if analysis.context_percent_used > 60:
+            recommendations.append(Recommendation(
+                priority=1 if analysis.context_percent_used > 80 else 2,
+                category="reduce",
+                title="High context utilization",
+                description=f"Context is {analysis.context_percent_used:.0f}% full. "
+                           f"Consider running /clear or compacting soon.",
+                estimated_savings=int(analysis.summary.total_tokens * 0.3),
+                action="Run /clear to reset context, or compact conversation"
+            ))
+
+        # Add recommendations based on tool usage
+        if analysis.summary.tool_tokens > 50000:
+            recommendations.append(Recommendation(
+                priority=2,
+                category="optimize",
+                title="High tool token consumption",
+                description=f"Tools have consumed {analysis.summary.tool_tokens:,} tokens. "
+                           f"Consider batching operations.",
+                estimated_savings=int(analysis.summary.tool_tokens * 0.2),
+                action="Batch file reads and writes when possible"
+            ))
+
+        # Sort by priority then estimated savings
+        recommendations.sort(key=lambda r: (r.priority, -r.estimated_savings))
+
+        return recommendations
+
+    def _recommendation_for_pattern(self, pattern: WastePattern) -> Optional[Recommendation]:
+        """Generate a recommendation for a waste pattern."""
+        if pattern.pattern_type == "redundant_load":
+            return Recommendation(
+                priority=1,
+                category="reduce",
+                title=f"Remove redundant loads of {pattern.component_name}",
+                description=pattern.description,
+                estimated_savings=pattern.tokens_wasted,
+                action=f"Ensure {pattern.component_name} is only loaded once per session"
+            )
+
+        elif pattern.pattern_type == "unused_component":
+            return Recommendation(
+                priority=2,
+                category="defer",
+                title=f"Defer loading {pattern.component_name}",
+                description=pattern.description,
+                estimated_savings=pattern.tokens_wasted,
+                action=f"Load {pattern.component_name} on-demand instead of at startup"
+            )
+
+        elif pattern.pattern_type == "high_cost_low_value":
+            return Recommendation(
+                priority=3,
+                category="optimize",
+                title=f"Optimize {pattern.component_name}",
+                description=pattern.description,
+                estimated_savings=pattern.tokens_wasted,
+                action=f"Consider making {pattern.component_name} smaller or on-demand"
+            )
+
+        return None
+
+    # =========================================================================
+    # Efficiency Scoring
+    # =========================================================================
+
+    def _calculate_efficiency_score(
+        self,
+        summary: SessionSummary,
+        waste_patterns: List[WastePattern]
+    ) -> float:
+        """
+        Calculate an efficiency score for the session.
+
+        Score is 0-100 based on:
+        - Context utilization (not too high or too low)
+        - Waste as percentage of total tokens
+        - Tool efficiency (reasonable input/output ratios)
+
+        Returns:
+            Efficiency score 0-100
+        """
+        score = 100.0
+
+        # Penalize for waste
+        total_waste = sum(p.tokens_wasted for p in waste_patterns)
+        if summary.total_tokens > 0:
+            waste_percent = (total_waste / summary.total_tokens) * 100
+            # Deduct up to 40 points for waste
+            score -= min(40, waste_percent * 2)
+
+        # Penalize for very high context usage (risk of truncation)
+        context_percent = (summary.total_tokens / EFFECTIVE_LIMIT_TOKENS) * 100
+        if context_percent > 80:
+            score -= (context_percent - 80)  # Deduct 1 point per % over 80
+
+        # Penalize for redundant loads (extra penalty on top of waste)
+        redundant_count = sum(
+            1 for p in waste_patterns if p.pattern_type == "redundant_load"
+        )
+        score -= redundant_count * 2  # 2 points per redundant pattern
+
+        return max(0, min(100, round(score, 1)))
+
+    # =========================================================================
+    # Historical Efficiency Analysis (Phase 3)
+    # =========================================================================
+
+    def generate_efficiency_report(
+        self,
+        period_days: int = 7,
+        compare_to_previous: bool = True
+    ) -> Optional[EfficiencyReport]:
+        """
+        Generate a historical efficiency report for the specified period.
+
+        Args:
+            period_days: Number of days to analyze (default 7 for weekly)
+            compare_to_previous: Whether to compare to the previous period
+
+        Returns:
+            EfficiencyReport with component scores and recommendations
+        """
+        # Get component stats for the period
+        stats = self.metrics.get_component_stats(days=period_days)
+
+        if not stats:
+            return None
+
+        # Get session history for the period
+        sessions = self.metrics.get_session_history(days=period_days)
+        if not sessions:
+            return None
+
+        # Calculate component efficiency scores
+        component_scores = [
+            self._calculate_component_efficiency(stat)
+            for stat in stats
+        ]
+
+        # Sort by efficiency score (lowest first to highlight problem areas)
+        component_scores.sort(key=lambda c: c.efficiency_score)
+
+        # Detect trends
+        trends = self._detect_component_trends(period_days)
+
+        # Update component scores with trends
+        for comp in component_scores:
+            comp_key = f"{comp.component_type}:{comp.component_name}"
+            if comp_key in trends:
+                comp.trend = trends[comp_key]
+
+        # Calculate overall metrics
+        total_tokens = sum(s.get("total_tokens", 0) for s in sessions)
+        avg_tokens = total_tokens / len(sessions) if sessions else 0
+
+        # Calculate overall efficiency score
+        if component_scores:
+            overall_score = sum(c.efficiency_score for c in component_scores) / len(component_scores)
+        else:
+            overall_score = 100.0
+
+        # Generate recommendations based on component scores
+        recommendations = self._generate_efficiency_recommendations(component_scores)
+
+        # Compare to previous period
+        comparison = None
+        if compare_to_previous:
+            comparison = self._compare_to_previous_period(period_days, total_tokens, overall_score)
+
+        return EfficiencyReport(
+            period_days=period_days,
+            generated_at=datetime.now(),
+            total_sessions=len(sessions),
+            total_tokens=total_tokens,
+            avg_tokens_per_session=round(avg_tokens, 0),
+            overall_efficiency_score=round(overall_score, 1),
+            component_scores=component_scores,
+            trends=trends,
+            recommendations=recommendations,
+            comparison_to_previous=comparison
+        )
+
+    def _calculate_component_efficiency(self, stat: Dict[str, Any]) -> ComponentEfficiency:
+        """
+        Calculate efficiency score for a single component.
+
+        Score based on:
+        - Utilization (50 points): % of loads where component was used
+        - Token cost (30 points): Lower avg tokens = higher score
+        - Consistency (20 points): Frequent use with high utilization
+
+        Args:
+            stat: Component statistics dict from get_component_stats
+
+        Returns:
+            ComponentEfficiency with calculated score
+        """
+        load_count = stat.get("load_count", 0)
+        used_count = stat.get("used_count", 0)
+        avg_tokens = stat.get("avg_tokens", 0)
+        total_tokens = stat.get("total_tokens", 0)
+
+        # Calculate utilization rate
+        utilization = used_count / load_count if load_count > 0 else 0
+
+        # Utilization score (0-50 points)
+        utilization_score = utilization * 50
+
+        # Token cost score (0-30 points)
+        # Score higher for smaller components
+        if avg_tokens < 200:
+            token_score = 30
+        elif avg_tokens < 500:
+            token_score = 25
+        elif avg_tokens < 1000:
+            token_score = 20
+        elif avg_tokens < 2000:
+            token_score = 15
+        elif avg_tokens < 5000:
+            token_score = 10
+        else:
+            token_score = 5
+
+        # Consistency score (0-20 points)
+        # High utilization with frequent use = consistent value
+        if load_count >= 10 and utilization >= 0.8:
+            consistency_score = 20
+        elif load_count >= 5 and utilization >= 0.6:
+            consistency_score = 15
+        elif load_count >= 3 and utilization >= 0.4:
+            consistency_score = 10
+        else:
+            consistency_score = 5
+
+        efficiency_score = utilization_score + token_score + consistency_score
+
+        return ComponentEfficiency(
+            component_name=stat.get("component_name", "unknown"),
+            component_type=stat.get("component_type", "unknown"),
+            total_tokens=total_tokens,
+            load_count=load_count,
+            used_count=used_count,
+            utilization_rate=round(utilization, 3),
+            efficiency_score=round(efficiency_score, 1),
+            avg_tokens_per_load=round(avg_tokens, 0),
+            trend="new"  # Will be updated by detect_component_trends
+        )
+
+    def _detect_component_trends(self, period_days: int) -> Dict[str, str]:
+        """
+        Detect trends by comparing current period to previous period.
+
+        Args:
+            period_days: Length of current period in days
+
+        Returns:
+            Dict mapping component key to trend (improving/stable/degrading/new)
+        """
+        trends = {}
+
+        # Get current period stats
+        current_stats = self.metrics.get_component_stats(days=period_days)
+
+        # Get previous period stats (same length, offset by current period)
+        previous_stats = self.metrics.get_component_stats(
+            days=period_days,
+            offset_days=period_days
+        )
+
+        # Build lookup for previous period
+        previous_lookup = {}
+        for stat in previous_stats:
+            key = f"{stat['component_type']}:{stat['component_name']}"
+            previous_lookup[key] = stat
+
+        # Compare each current component to previous
+        for stat in current_stats:
+            key = f"{stat['component_type']}:{stat['component_name']}"
+
+            if key not in previous_lookup:
+                trends[key] = "new"
+                continue
+
+            prev = previous_lookup[key]
+
+            # Compare utilization rates
+            curr_util = stat.get("used_count", 0) / stat.get("load_count", 1)
+            prev_util = prev.get("used_count", 0) / prev.get("load_count", 1)
+
+            # Determine trend based on utilization change
+            util_change = curr_util - prev_util
+
+            if util_change > 0.1:  # More than 10% improvement
+                trends[key] = "improving"
+            elif util_change < -0.1:  # More than 10% decline
+                trends[key] = "degrading"
+            else:
+                trends[key] = "stable"
+
+        return trends
+
+    def _generate_efficiency_recommendations(
+        self,
+        component_scores: List[ComponentEfficiency]
+    ) -> List[Recommendation]:
+        """
+        Generate recommendations based on component efficiency scores.
+
+        Args:
+            component_scores: List of component efficiency scores
+
+        Returns:
+            List of prioritized recommendations
+        """
+        recommendations = []
+
+        for comp in component_scores:
+            # Skip components with good efficiency
+            if comp.efficiency_score >= 70:
+                continue
+
+            # Skip execution-value components (hooks, context)
+            # These provide value through running, not through being referenced
+            if comp.component_type in EXECUTION_VALUE_TYPES:
+                continue
+
+            # Low utilization = defer loading
+            if comp.utilization_rate < 0.3:
+                potential_savings = int(comp.avg_tokens_per_load * (1 - comp.utilization_rate))
+                recommendations.append(Recommendation(
+                    priority=1 if comp.efficiency_score < 40 else 2,
+                    category="defer",
+                    title=f"Defer loading {comp.component_name}",
+                    description=f"Used only {comp.utilization_rate*100:.0f}% of the time, "
+                                f"avg {comp.avg_tokens_per_load:.0f} tokens per load",
+                    estimated_savings=potential_savings,
+                    action=f"Move {comp.component_name} to on-demand loading via trigger table"
+                ))
+
+            # High cost = optimize
+            elif comp.avg_tokens_per_load > 2000:
+                potential_savings = int(comp.avg_tokens_per_load * 0.3)  # Assume 30% reduction possible
+                recommendations.append(Recommendation(
+                    priority=2,
+                    category="optimize",
+                    title=f"Optimize {comp.component_name} size",
+                    description=f"Averaging {comp.avg_tokens_per_load:.0f} tokens per load. "
+                                f"Consider splitting or reducing content.",
+                    estimated_savings=potential_savings,
+                    action=f"Review and condense {comp.component_name} content"
+                ))
+
+            # Degrading trend = review
+            if comp.trend == "degrading":
+                recommendations.append(Recommendation(
+                    priority=3,
+                    category="review",
+                    title=f"Review declining {comp.component_name}",
+                    description="Utilization is decreasing. May no longer be needed.",
+                    estimated_savings=int(comp.total_tokens * 0.5),
+                    action=f"Evaluate if {comp.component_name} is still required"
+                ))
+
+        # Sort by estimated savings first (highest savings first), then priority
+        recommendations.sort(key=lambda r: (-r.estimated_savings, r.priority))
+
+        return recommendations
+
+    def _compare_to_previous_period(
+        self,
+        period_days: int,
+        current_total_tokens: int,
+        current_efficiency_score: float
+    ) -> Dict[str, float]:
+        """
+        Compare current metrics to the previous period.
+
+        Args:
+            period_days: Length of period in days
+            current_total_tokens: Total tokens in current period
+            current_efficiency_score: Overall efficiency score for current period
+
+        Returns:
+            Dict with comparison metrics (change percentages)
+        """
+        # Get previous period sessions (same length, offset by current period)
+        sessions = self.metrics.get_session_history(
+            days=period_days,
+            offset_days=period_days
+        )
+
+        if not sessions:
+            return {"data_available": False}
+
+        prev_total_tokens = sum(s.get("total_tokens", 0) for s in sessions)
+
+        # Calculate previous period efficiency from session scores
+        prev_efficiency_scores = [s.get("efficiency_score", 0) for s in sessions if s.get("efficiency_score")]
+        prev_efficiency = sum(prev_efficiency_scores) / len(prev_efficiency_scores) if prev_efficiency_scores else 0
+
+        # Calculate changes
+        token_change = 0
+        if prev_total_tokens > 0:
+            token_change = ((current_total_tokens - prev_total_tokens) / prev_total_tokens) * 100
+
+        efficiency_change = 0
+        if prev_efficiency > 0:
+            efficiency_change = current_efficiency_score - prev_efficiency
+
+        return {
+            "token_change_percent": round(token_change, 1),
+            "previous_total_tokens": prev_total_tokens,
+            "sessions_in_previous": len(sessions),
+            "previous_efficiency_score": round(prev_efficiency, 1),
+            "efficiency_change_percent": round(efficiency_change, 1)
+        }
+
+    # =========================================================================
+    # Formatting Helpers
+    # =========================================================================
+
+    def format_analysis_report(self, analysis: Optional[SessionAnalysis]) -> str:
+        """
+        Format an analysis as a readable report.
+
+        Args:
+            analysis: SessionAnalysis to format (can be None)
+
+        Returns:
+            Formatted markdown report
+        """
+        if analysis is None:
+            logger.debug("format_analysis_report called with None analysis - no session data available")
+            return (
+                "## Token Audit Report\n\n"
+                "**Status**: No session data available\n\n"
+                "The current session has no recorded token metrics. This can happen when:\n"
+                "- The session just started and no tool calls have been tracked yet\n"
+                "- Token tracking instrumentation is not fully configured\n"
+                "- The session ID could not be resolved\n\n"
+                "Try making some tool calls first, then run `/audit` again."
+            )
+
+        lines = [
+            "## Token Audit Report",
+            "",
+            f"**Session**: `{analysis.session_id[:8]}...`",
+            f"**Analyzed**: {analysis.analyzed_at.strftime('%Y-%m-%d %H:%M')}",
+            f"**Efficiency Score**: {analysis.efficiency_score}/100",
+            "",
+            "### Context Usage",
+            "",
+            f"- **Total Tokens**: {analysis.summary.total_tokens:,}",
+            f"- **Context Used**: {analysis.context_percent_used:.1f}% of effective limit",
+            f"- **Peak Tokens**: {analysis.summary.peak_context_tokens:,}",
+            "",
+        ]
+
+        # Budget status
+        if analysis.summary.budget_tokens:
+            lines.extend([
+                f"- **Budget**: {analysis.summary.budget_tokens:,} tokens",
+                f"- **Budget Used**: {analysis.summary.budget_percent_used:.1f}%",
+                "",
+            ])
+
+        # Breakdown by type
+        lines.extend([
+            "### Breakdown by Type",
+            "",
+            "| Type | Tokens | % of Total | Count |",
+            "|------|--------|------------|-------|",
+        ])
+
+        for comp_type, data in sorted(
+            analysis.breakdown_by_type.items(),
+            key=lambda x: x[1]["tokens"],
+            reverse=True
+        ):
+            lines.append(
+                f"| {comp_type} | {data['tokens']:,} | {data['percent']:.1f}% | {data['count']} |"
+            )
+
+        lines.append("")
+
+        # Top consumers
+        if analysis.top_consumers:
+            lines.extend([
+                "### Top Token Consumers",
+                "",
+                "| Component | Type | Tokens |",
+                "|-----------|------|--------|",
+            ])
+
+            for consumer in analysis.top_consumers[:5]:
+                lines.append(
+                    f"| {consumer['name']} | {consumer['type']} | {consumer['tokens']:,} |"
+                )
+
+            lines.append("")
+
+        # Waste patterns
+        if analysis.waste_patterns:
+            lines.extend([
+                "### Detected Waste Patterns",
+                "",
+            ])
+
+            for pattern in analysis.waste_patterns[:5]:
+                icon = {
+                    "redundant_load": "🔄",
+                    "unused_component": "⚠️",
+                    "high_cost_low_value": "📉"
+                }.get(pattern.pattern_type, "•")
+
+                lines.append(f"- {icon} **{pattern.component_name}**: {pattern.description}")
+
+            lines.append("")
+
+        return "\n".join(lines)
+
+    def format_recommendations(self, recommendations: List[Recommendation]) -> str:
+        """
+        Format recommendations as a readable list.
+
+        Args:
+            recommendations: List of recommendations to format
+
+        Returns:
+            Formatted markdown recommendations
+        """
+        if not recommendations:
+            return "No recommendations at this time."
+
+        lines = [
+            "### Recommendations",
+            "",
+        ]
+
+        priority_icons = {1: "🔴", 2: "🟡", 3: "🟢"}
+
+        for rec in recommendations:
+            icon = priority_icons.get(rec.priority, "•")
+            lines.extend([
+                f"#### {icon} {rec.title}",
+                "",
+                f"{rec.description}",
+                "",
+                f"**Estimated Savings**: ~{rec.estimated_savings:,} tokens",
+            ])
+
+            if rec.action:
+                lines.append(f"**Action**: {rec.action}")
+
+            lines.append("")
+
+        return "\n".join(lines)
+
+    def format_efficiency_report(self, report: EfficiencyReport) -> str:
+        """
+        Format an efficiency report as a readable markdown report.
+
+        Args:
+            report: EfficiencyReport to format
+
+        Returns:
+            Formatted markdown report
+        """
+        period_label = "Weekly" if report.period_days == 7 else f"{report.period_days}-Day"
+
+        lines = [
+            f"## {period_label} Efficiency Report",
+            "",
+            f"**Period**: Last {report.period_days} days",
+            f"**Generated**: {report.generated_at.strftime('%Y-%m-%d %H:%M')}",
+            f"**Overall Efficiency**: {report.overall_efficiency_score:.0f}/100",
+            "",
+            "### Summary",
+            "",
+            f"- **Sessions Analyzed**: {report.total_sessions}",
+            f"- **Total Tokens**: {report.total_tokens:,}",
+            f"- **Avg per Session**: {report.avg_tokens_per_session:,.0f}",
+            "",
+        ]
+
+        # Comparison to previous period
+        if report.comparison_to_previous and report.comparison_to_previous.get("data_available", True):
+            change = report.comparison_to_previous.get("token_change_percent", 0)
+            change_icon = "📈" if change > 0 else "📉" if change < 0 else "➡️"
+            lines.extend([
+                "### Period Comparison",
+                "",
+                f"- **Token Change**: {change_icon} {change:+.1f}% vs previous {report.period_days} days",
+                "",
+            ])
+
+        # Component efficiency scores table
+        lines.extend([
+            "### Component Efficiency Scores",
+            "",
+            "| Component | Type | Score | Utilization | Trend |",
+            "|-----------|------|-------|-------------|-------|",
+        ])
+
+        trend_icons = {
+            "improving": "↑",
+            "stable": "→",
+            "degrading": "↓",
+            "new": "★"
+        }
+
+        for comp in report.component_scores[:10]:  # Top 10 worst performing
+            trend_icon = trend_icons.get(comp.trend, "?")
+            lines.append(
+                f"| {comp.component_name} | {comp.component_type} | "
+                f"{comp.efficiency_score:.0f} | {comp.utilization_rate*100:.0f}% | {trend_icon} |"
+            )
+
+        lines.append("")
+
+        # Recommendations
+        if report.recommendations:
+            lines.extend([
+                "### Top Recommendations",
+                "",
+            ])
+
+            priority_icons = {1: "🔴", 2: "🟡", 3: "🟢"}
+
+            for rec in report.recommendations[:5]:  # Top 5 recommendations
+                icon = priority_icons.get(rec.priority, "•")
+                lines.append(
+                    f"- {icon} **{rec.title}**: {rec.description}"
+                )
+                lines.append(f"  - Potential savings: ~{rec.estimated_savings:,} tokens")
+
+            lines.append("")
+
+        return "\n".join(lines)
+
+    # =========================================================================
+    # Self-Improvement Loop (Phase 6)
+    # =========================================================================
+
+    def analyze_usage_patterns(
+        self,
+        days: int = 30
+    ) -> Dict[str, Any]:
+        """
+        Analyze long-term usage patterns to identify optimization opportunities.
+
+        Args:
+            days: Number of days to analyze (default 30)
+
+        Returns:
+            Dict with usage pattern analysis including:
+            - never_used: Components loaded but never marked as used
+            - rarely_used: Components used < 10% of times loaded
+            - high_cost_low_use: High-token components with low utilization
+            - pruning_candidates: Recommended items to remove
+        """
+        # Get component stats for the period
+        stats = self.metrics.get_component_stats(days=days)
+
+        never_used = []
+        rarely_used = []
+        high_cost_low_use = []
+        pruning_candidates = []
+
+        for stat in stats:
+            load_count = stat.get("load_count", 0)
+            used_count = stat.get("used_count", 0)
+            total_tokens = stat.get("total_tokens", 0)
+            avg_tokens = stat.get("avg_tokens", 0)
+
+            if load_count == 0:
+                continue
+
+            utilization = used_count / load_count
+
+            component_info = {
+                "component_name": stat.get("component_name"),
+                "component_type": stat.get("component_type"),
+                "load_count": load_count,
+                "used_count": used_count,
+                "utilization_rate": round(utilization, 2),
+                "total_tokens": total_tokens,
+                "avg_tokens": avg_tokens
+            }
+
+            # Never used (0% utilization)
+            if used_count == 0:
+                never_used.append(component_info)
+                # High token cost never-used = definite prune candidate
+                if total_tokens > 1000:
+                    pruning_candidates.append({
+                        **component_info,
+                        "reason": "Never used, high token cost",
+                        "savings": total_tokens
+                    })
+
+            # Rarely used (< 10% utilization)
+            elif utilization < 0.1:
+                rarely_used.append(component_info)
+                # High token cost rarely-used = consider pruning
+                if total_tokens > 2000:
+                    pruning_candidates.append({
+                        **component_info,
+                        "reason": "Rarely used, high token cost",
+                        "savings": int(total_tokens * 0.8)  # Estimate 80% savings
+                    })
+
+            # High cost, low utilization (< 30%, > 500 tokens avg)
+            elif utilization < 0.3 and avg_tokens > 500:
+                high_cost_low_use.append(component_info)
+                pruning_candidates.append({
+                    **component_info,
+                    "reason": "Low utilization, consider deferring",
+                    "savings": int(total_tokens * 0.5)  # Defer = 50% savings
+                })
+
+        # Sort pruning candidates by potential savings
+        pruning_candidates.sort(key=lambda x: x["savings"], reverse=True)
+
+        return {
+            "period_days": days,
+            "components_analyzed": len(stats),
+            "never_used": never_used,
+            "rarely_used": rarely_used,
+            "high_cost_low_use": high_cost_low_use,
+            "pruning_candidates": pruning_candidates[:10],  # Top 10
+            "total_potential_savings": sum(p["savings"] for p in pruning_candidates)
+        }
+
+    def generate_optimization_suggestions(
+        self,
+        usage_analysis: Optional[Dict[str, Any]] = None
+    ) -> List[Dict[str, Any]]:
+        """
+        Generate specific optimization suggestions based on usage analysis.
+
+        Args:
+            usage_analysis: Result from analyze_usage_patterns() or None to run fresh
+
+        Returns:
+            List of actionable suggestions with estimated savings
+        """
+        if usage_analysis is None:
+            usage_analysis = self.analyze_usage_patterns()
+
+        suggestions = []
+        suggestion_id = 1
+
+        # Suggestions for never-used components
+        for comp in usage_analysis.get("never_used", [])[:5]:
+            suggestions.append({
+                "id": suggestion_id,
+                "type": "remove_unused_pattern",
+                "priority": 1,
+                "title": f"Remove unused {comp['component_type']}: {comp['component_name']}",
+                "description": (
+                    f"'{comp['component_name']}' has been loaded {comp['load_count']} times "
+                    f"but never used. Total cost: {comp['total_tokens']:,} tokens."
+                ),
+                "estimated_savings": comp["total_tokens"],
+                "target_files": self._get_component_files(
+                    comp["component_type"],
+                    comp["component_name"]
+                ),
+                "changes": {
+                    "action": "remove",
+                    "component_type": comp["component_type"],
+                    "component_name": comp["component_name"]
+                },
+                "risk_level": "low",
+                "reversible": True
+            })
+            suggestion_id += 1
+
+        # Suggestions for deferring rarely-used components
+        for comp in usage_analysis.get("high_cost_low_use", [])[:5]:
+            suggestions.append({
+                "id": suggestion_id,
+                "type": "defer_loading",
+                "priority": 2,
+                "title": f"Defer loading of {comp['component_name']}",
+                "description": (
+                    f"'{comp['component_name']}' is only used {comp['utilization_rate']*100:.0f}% "
+                    f"of the time but costs {comp['avg_tokens']:,} tokens per load. "
+                    f"Consider on-demand loading."
+                ),
+                "estimated_savings": int(comp["total_tokens"] * 0.6),
+                "target_files": self._get_component_files(
+                    comp["component_type"],
+                    comp["component_name"]
+                ),
+                "changes": {
+                    "action": "defer",
+                    "component_type": comp["component_type"],
+                    "component_name": comp["component_name"],
+                    "triggers": [comp["component_name"].lower()]
+                },
+                "risk_level": "medium",
+                "reversible": True
+            })
+            suggestion_id += 1
+
+        # Context reduction suggestions based on session analysis
+        recent_sessions = self.metrics.get_session_history(days=7)
+        if recent_sessions:
+            avg_tokens = sum(s.get("total_tokens", 0) for s in recent_sessions) / len(recent_sessions)
+            if avg_tokens > 10000:  # High average consumption
+                suggestions.append({
+                    "id": suggestion_id,
+                    "type": "reduce_context",
+                    "priority": 2,
+                    "title": "Reduce initial context size",
+                    "description": (
+                        f"Average session uses {avg_tokens:,.0f} tokens. "
+                        f"Consider moving detailed sections to on-demand commands."
+                    ),
+                    "estimated_savings": int(avg_tokens * 0.3),
+                    "target_files": [".claude/CLAUDE.md"],
+                    "changes": {
+                        "action": "move_to_on_demand",
+                        "sections": ["Anti-Patterns", "Project-Learned Patterns"]
+                    },
+                    "risk_level": "medium",
+                    "reversible": True
+                })
+                suggestion_id += 1
+
+        # Sort by priority and savings
+        suggestions.sort(key=lambda s: (s["priority"], -s["estimated_savings"]))
+
+        return suggestions
+
+    def _get_component_files(
+        self,
+        component_type: str,
+        component_name: str
+    ) -> List[str]:
+        """
+        Get the files associated with a component.
+
+        Args:
+            component_type: Type of component (command, hook, etc.)
+            component_name: Name of the component
+
+        Returns:
+            List of file paths that define or use the component
+        """
+        files = []
+
+        if component_type == "command":
+            # Commands are in global/commands/ or .claude/commands/
+            files.extend([
+                f"global/commands/{component_name}.md",
+                f".claude/commands/{component_name}.md"
+            ])
+        elif component_type == "hook":
+            # Hooks are in .claude/hooks/ or project/hooks/
+            files.extend([
+                f".claude/hooks/{component_name}.py",
+                f"project/hooks/{component_name}.py"
+            ])
+        elif component_type == "context":
+            # Context files
+            files.append(".claude/CLAUDE.md")
+
+        # Filter to existing files (would need filesystem access)
+        # For now, return all potential paths
+        return files
+
+    def track_optimization_impact(
+        self,
+        optimization_id: int,
+        before_tokens: int,
+        after_tokens: int
+    ) -> Dict[str, Any]:
+        """
+        Track the impact of an applied optimization.
+
+        Args:
+            optimization_id: ID of the optimization
+            before_tokens: Token count before optimization
+            after_tokens: Token count after optimization
+
+        Returns:
+            Dict with impact metrics
+        """
+        savings = before_tokens - after_tokens
+        savings_percent = (savings / before_tokens * 100) if before_tokens > 0 else 0
+
+        impact = {
+            "optimization_id": optimization_id,
+            "before_tokens": before_tokens,
+            "after_tokens": after_tokens,
+            "tokens_saved": savings,
+            "savings_percent": round(savings_percent, 1),
+            "effective": savings > 0,
+            "timestamp": datetime.now(timezone.utc).isoformat()
+        }
+
+        # If effectiveness is negative, suggest reversal
+        if savings < 0:
+            impact["recommendation"] = "Consider reverting - optimization increased token usage"
+
+        return impact
+
+    def format_suggestions_report(
+        self,
+        suggestions: List[Dict[str, Any]]
+    ) -> str:
+        """
+        Format optimization suggestions as a readable report.
+
+        Args:
+            suggestions: List of suggestions from generate_optimization_suggestions()
+
+        Returns:
+            Formatted markdown report
+        """
+        if not suggestions:
+            return "## Optimization Suggestions\n\n*No optimization opportunities found.*"
+
+        total_savings = sum(s["estimated_savings"] for s in suggestions)
+
+        lines = [
+            "## Optimization Suggestions",
+            "",
+            f"**Total Potential Savings**: ~{total_savings:,} tokens",
+            f"**Suggestions**: {len(suggestions)}",
+            "",
+            "### Recommendations",
+            ""
+        ]
+
+        priority_labels = {1: "High", 2: "Medium", 3: "Low"}
+        risk_emojis = {"low": "🟢", "medium": "🟡", "high": "🔴"}
+
+        for suggestion in suggestions:
+            priority = priority_labels.get(suggestion["priority"], "?")
+            risk = risk_emojis.get(suggestion.get("risk_level", "medium"), "⚪")
+
+            lines.extend([
+                f"#### [{suggestion['id']}] {suggestion['title']}",
+                "",
+                f"**Priority**: {priority} | **Risk**: {risk} | "
+                f"**Savings**: ~{suggestion['estimated_savings']:,} tokens",
+                "",
+                suggestion["description"],
+                "",
+                f"**Apply with**: `/efficiency --apply {suggestion['id']}`",
+                "",
+                "---",
+                ""
+            ])
+
+        return "\n".join(lines)
+
+
+# Singleton instance for convenience
+_analyzer_instance: Optional[TokenAnalyzer] = None
+
+
+def get_analyzer() -> TokenAnalyzer:
+    """Get or create the singleton analyzer instance."""
+    global _analyzer_instance
+    if _analyzer_instance is None:
+        _analyzer_instance = TokenAnalyzer()
+    return _analyzer_instance