codymaster 4.1.3 → 4.3.0

package/skills/cm-content-factory/scripts/state_manager.py

@@ -0,0 +1,320 @@
+#!/usr/bin/env python3
+"""
+State Manager — Central state management for Content Factory dashboard.
+
+All pipeline scripts write state here; dashboard reads it.
+State persisted to `.content-factory-state.json` at project root.
+Events logged to `logs/events.jsonl` (append-only).
+
+Usage:
+    from state_manager import StateManager
+    sm = StateManager("/path/to/project")
+    sm.update_phase("write", "running", progress=0.5)
+    sm.add_task("write-article-1", "writing", {"topic": "SEO Tips"})
+    sm.log_event("info", "Article 1 started")
+"""
+
+import json
+import os
+import time
+import threading
+from pathlib import Path
+from datetime import datetime
+
+
+class StateManager:
+    """Thread-safe state manager for Content Factory pipeline."""
+
+    STATE_FILE = ".content-factory-state.json"
+    EVENTS_FILE = "logs/events.jsonl"
+
+    def __init__(self, project_root: str = None):
+        self.project_root = Path(project_root or os.getcwd()).resolve()
+        self.state_path = self.project_root / self.STATE_FILE
+        self.events_path = self.project_root / self.EVENTS_FILE
+        self._lock = threading.Lock()
+        self._ensure_dirs()
+
+    def _ensure_dirs(self):
+        """Create logs directory if needed."""
+        self.events_path.parent.mkdir(parents=True, exist_ok=True)
+
+    def _default_state(self) -> dict:
+        """Return default empty state."""
+        return {
+            "version": "2.0",
+            "started_at": datetime.now().isoformat(),
+            "updated_at": datetime.now().isoformat(),
+            "status": "idle",
+            "pipeline": {
+                "current_phase": None,
+                "phases": {
+                    "extract": {"status": "pending", "progress": 0, "started_at": None, "finished_at": None},
+                    "plan": {"status": "pending", "progress": 0, "started_at": None, "finished_at": None},
+                    "write": {"status": "pending", "progress": 0, "started_at": None, "finished_at": None},
+                    "audit": {"status": "pending", "progress": 0, "started_at": None, "finished_at": None},
+                    "seo": {"status": "pending", "progress": 0, "started_at": None, "finished_at": None},
+                    "publish": {"status": "pending", "progress": 0, "started_at": None, "finished_at": None},
+                },
+            },
+            "tasks": [],
+            "tokens": {
+                "total_input": 0,
+                "total_output": 0,
+                "total_cost_usd": 0.0,
+                "budget_limit_usd": None,
+                "providers": {},
+            },
+            "errors": [],
+            "stats": {
+                "articles_total": 0,
+                "articles_written": 0,
+                "articles_passed_audit": 0,
+                "articles_failed": 0,
+            },
+            "agents": [],
+        }
+
+    def load(self) -> dict:
+        """Load current state from disk."""
+        if not self.state_path.exists():
+            return self._default_state()
+        try:
+            with open(self.state_path, "r", encoding="utf-8") as f:
+                return json.load(f)
+        except (json.JSONDecodeError, IOError):
+            return self._default_state()
+
+    def save(self, state: dict):
+        """Save state to disk (thread-safe)."""
+        state["updated_at"] = datetime.now().isoformat()
+        with self._lock:
+            tmp = self.state_path.with_suffix(".tmp")
+            with open(tmp, "w", encoding="utf-8") as f:
+                json.dump(state, f, indent=2, ensure_ascii=False)
+            tmp.replace(self.state_path)
+
+    def update_phase(self, phase: str, status: str, progress: float = None, error: str = None):
+        """Update phase status and optional progress (0.0 - 1.0)."""
+        state = self.load()
+        phase_data = state["pipeline"]["phases"].get(phase, {})
+
+        phase_data["status"] = status
+        if progress is not None:
+            phase_data["progress"] = min(1.0, max(0.0, progress))
+        if status == "running" and not phase_data.get("started_at"):
+            phase_data["started_at"] = datetime.now().isoformat()
+        if status in ("done", "failed"):
+            phase_data["finished_at"] = datetime.now().isoformat()
+
+        state["pipeline"]["phases"][phase] = phase_data
+        state["pipeline"]["current_phase"] = phase if status == "running" else state["pipeline"]["current_phase"]
+
+        # Update overall status
+        if status == "running":
+            state["status"] = "running"
+        elif all(p["status"] == "done" for p in state["pipeline"]["phases"].values()):
+            state["status"] = "completed"
+        elif any(p["status"] == "failed" for p in state["pipeline"]["phases"].values()):
+            state["status"] = "error"
+
+        if error:
+            self._add_error(state, phase, error)
+
+        self.save(state)
+        self.log_event("info" if status != "failed" else "error",
+                       f"Phase {phase}: {status}" + (f" ({progress*100:.0f}%)" if progress else ""),
+                       {"phase": phase, "status": status})
+
+    def add_task(self, task_id: str, status: str, meta: dict = None):
+        """Add or update a task in the queue."""
+        state = self.load()
+        existing = next((t for t in state["tasks"] if t["id"] == task_id), None)
+        now = datetime.now().isoformat()
+
+        if existing:
+            existing["status"] = status
+            existing["updated_at"] = now
+            if meta:
+                existing["meta"].update(meta)
+        else:
+            state["tasks"].append({
+                "id": task_id,
+                "status": status,
+                "created_at": now,
+                "updated_at": now,
+                "meta": meta or {},
+            })
+
+        # Keep last 200 tasks max
+        if len(state["tasks"]) > 200:
+            done = [t for t in state["tasks"] if t["status"] in ("done", "failed")]
+            active = [t for t in state["tasks"] if t["status"] not in ("done", "failed")]
+            state["tasks"] = active + done[-100:]
+
+        self.save(state)
+
+    def update_tokens(self, provider: str, input_tokens: int = 0, output_tokens: int = 0, cost_usd: float = 0.0):
+        """Track token usage by provider."""
+        state = self.load()
+        tokens = state["tokens"]
+        tokens["total_input"] += input_tokens
+        tokens["total_output"] += output_tokens
+        tokens["total_cost_usd"] = round(tokens["total_cost_usd"] + cost_usd, 6)
+
+        if provider not in tokens["providers"]:
+            tokens["providers"][provider] = {"input": 0, "output": 0, "cost_usd": 0.0, "requests": 0}
+
+        p = tokens["providers"][provider]
+        p["input"] += input_tokens
+        p["output"] += output_tokens
+        p["cost_usd"] = round(p["cost_usd"] + cost_usd, 6)
+        p["requests"] += 1
+
+        self.save(state)
+
+    def check_budget(self) -> bool:
+        """Check if spending is within budget. Returns True if OK."""
+        state = self.load()
+        limit = state["tokens"].get("budget_limit_usd")
+        if limit is None:
+            return True
+        return state["tokens"]["total_cost_usd"] < limit
+
+    def set_budget(self, budget_usd: float):
+        """Set budget limit."""
+        state = self.load()
+        state["tokens"]["budget_limit_usd"] = budget_usd
+        self.save(state)
+
+    def update_stats(self, **kwargs):
+        """Update article statistics."""
+        state = self.load()
+        for k, v in kwargs.items():
+            if k in state["stats"]:
+                state["stats"][k] = v
+        self.save(state)
+
+    def increment_stat(self, key: str, amount: int = 1):
+        """Increment a stat counter."""
+        state = self.load()
+        if key in state["stats"]:
+            state["stats"][key] += amount
+        self.save(state)
+
+    def register_agent(self, agent_id: str, agent_type: str):
+        """Register an active agent."""
+        state = self.load()
+        existing = next((a for a in state["agents"] if a["id"] == agent_id), None)
+        now = datetime.now().isoformat()
+        if existing:
+            existing["last_seen"] = now
+        else:
+            state["agents"].append({
+                "id": agent_id,
+                "type": agent_type,
+                "registered_at": now,
+                "last_seen": now,
+            })
+        self.save(state)
+
+    def _add_error(self, state: dict, source: str, message: str):
+        """Add error to state (max 50)."""
+        state["errors"].append({
+            "source": source,
+            "message": message,
+            "timestamp": datetime.now().isoformat(),
+        })
+        state["errors"] = state["errors"][-50:]
+
+    def add_error(self, source: str, message: str):
+        """Public method to add an error."""
+        state = self.load()
+        self._add_error(state, source, message)
+        self.save(state)
+        self.log_event("error", message, {"source": source})
+
+    def log_event(self, level: str, message: str, data: dict = None):
+        """Append event to JSONL log file."""
+        event = {
+            "ts": datetime.now().isoformat(),
+            "level": level,
+            "msg": message,
+        }
+        if data:
+            event["data"] = data
+
+        with self._lock:
+            with open(self.events_path, "a", encoding="utf-8") as f:
+                f.write(json.dumps(event, ensure_ascii=False) + "\n")
+
+    def get_recent_events(self, n: int = 100) -> list:
+        """Read last N events from log."""
+        if not self.events_path.exists():
+            return []
+        try:
+            with open(self.events_path, "r", encoding="utf-8") as f:
+                lines = f.readlines()
+            events = []
+            for line in lines[-n:]:
+                line = line.strip()
+                if line:
+                    try:
+                        events.append(json.loads(line))
+                    except json.JSONDecodeError:
+                        pass
+            return events
+        except IOError:
+            return []
+
+    def reset(self):
+        """Reset state to default (for new pipeline run)."""
+        state = self._default_state()
+        self.save(state)
+        self.log_event("info", "State reset — new pipeline run")
+
+    def get_snapshot(self) -> dict:
+        """Get full state snapshot including recent events."""
+        state = self.load()
+        state["recent_events"] = self.get_recent_events(50)
+        return state
+
+
+# CLI interface
+if __name__ == "__main__":
+    import sys
+    sm = StateManager()
+
+    if len(sys.argv) < 2:
+        print("Usage: python3 state_manager.py <command>")
+        print("  status  — Show current state")
+        print("  reset   — Reset state")
+        print("  events  — Show recent events")
+        sys.exit(0)
+
+    cmd = sys.argv[1]
+    if cmd == "status":
+        state = sm.load()
+        print(f"\n🏭 Content Factory State")
+        print(f"  Status: {state['status']}")
+        print(f"  Updated: {state['updated_at']}")
+        print(f"\n  Pipeline:")
+        for phase, data in state["pipeline"]["phases"].items():
+            icon = {"pending": "⬜", "running": "🔄", "done": "✅", "failed": "❌"}.get(data["status"], "❓")
+            progress = f" ({data['progress']*100:.0f}%)" if data["progress"] > 0 else ""
+            print(f"    {icon} {phase}{progress}")
+        print(f"\n  Tokens: {state['tokens']['total_input']}in / {state['tokens']['total_output']}out")
+        print(f"  Cost: ${state['tokens']['total_cost_usd']:.4f}")
+        print(f"  Tasks: {len(state['tasks'])}")
+        print(f"  Errors: {len(state['errors'])}")
+        print(f"  Agents: {len(state['agents'])}")
+
+    elif cmd == "reset":
+        sm.reset()
+        print("✅ State reset")
+
+    elif cmd == "events":
+        events = sm.get_recent_events(20)
+        for e in events:
+            icon = {"info": "ℹ️", "warn": "⚠️", "error": "❌"}.get(e["level"], "•")
+            print(f"  {icon} [{e['ts'][-8:]}] {e['msg']}")

package/skills/cm-content-factory/scripts/token_manager.py

@@ -0,0 +1,268 @@
+#!/usr/bin/env python3
+"""
+Token Manager — Track token usage, costs, rate limits, and budget for AI providers.
+
+Monitors API consumption across providers (Gemini, Claude, OpenAI).
+Implements circuit breaker pattern and exponential backoff.
+
+Usage:
+    from token_manager import TokenManager
+    tm = TokenManager("/path/to/project")
+    tm.record_usage("gemini", input_tokens=500, output_tokens=200)
+    if not tm.check_budget():
+        print("Budget exceeded!")
+    tm.wait_if_rate_limited("gemini")
+"""
+
+import json
+import time
+import math
+from pathlib import Path
+from datetime import datetime
+
+
+# Cost per 1M tokens (USD) — 2025 pricing estimates
+PROVIDER_COSTS = {
+    "gemini": {"input": 0.15, "output": 0.60},
+    "gemini-pro": {"input": 1.25, "output": 5.00},
+    "claude-sonnet": {"input": 3.00, "output": 15.00},
+    "claude-haiku": {"input": 0.25, "output": 1.25},
+    "openai-gpt4o": {"input": 2.50, "output": 10.00},
+    "openai-gpt4o-mini": {"input": 0.15, "output": 0.60},
+}
+
+# Default rate limits (requests per minute)
+RATE_LIMITS = {
+    "gemini": 60,
+    "gemini-pro": 30,
+    "claude-sonnet": 50,
+    "claude-haiku": 100,
+    "openai-gpt4o": 60,
+    "openai-gpt4o-mini": 100,
+}
+
+
+class TokenManager:
+    """Token usage tracker with rate limiting and circuit breaker."""
+
+    USAGE_FILE = "logs/token_usage.json"
+    MAX_CONSECUTIVE_FAILURES = 5
+    BACKOFF_BASE_SECONDS = 2.0
+    BACKOFF_MAX_SECONDS = 120.0
+
+    def __init__(self, project_root: str = None, budget_usd: float = None):
+        self.project_root = Path(project_root or ".").resolve()
+        self.usage_path = self.project_root / self.USAGE_FILE
+        self.budget_usd = budget_usd
+        self._request_timestamps: dict[str, list] = {}
+        self._consecutive_failures: dict[str, int] = {}
+        self._ensure_dirs()
+
+    def _ensure_dirs(self):
+        self.usage_path.parent.mkdir(parents=True, exist_ok=True)
+
+    def _load_usage(self) -> dict:
+        if not self.usage_path.exists():
+            return self._default_usage()
+        try:
+            with open(self.usage_path, "r", encoding="utf-8") as f:
+                return json.load(f)
+        except (json.JSONDecodeError, IOError):
+            return self._default_usage()
+
+    def _default_usage(self) -> dict:
+        return {
+            "session_id": datetime.now().strftime("%Y%m%d_%H%M%S"),
+            "started_at": datetime.now().isoformat(),
+            "updated_at": datetime.now().isoformat(),
+            "total_input_tokens": 0,
+            "total_output_tokens": 0,
+            "total_cost_usd": 0.0,
+            "budget_limit_usd": self.budget_usd,
+            "budget_remaining_usd": self.budget_usd,
+            "providers": {},
+            "requests": [],
+        }
+
+    def _save_usage(self, usage: dict):
+        usage["updated_at"] = datetime.now().isoformat()
+        tmp = self.usage_path.with_suffix(".tmp")
+        with open(tmp, "w", encoding="utf-8") as f:
+            json.dump(usage, f, indent=2, ensure_ascii=False)
+        tmp.replace(self.usage_path)
+
+    def estimate_cost(self, provider: str, input_tokens: int, output_tokens: int) -> float:
+        """Estimate cost for a given request."""
+        costs = PROVIDER_COSTS.get(provider, {"input": 1.0, "output": 3.0})
+        input_cost = (input_tokens / 1_000_000) * costs["input"]
+        output_cost = (output_tokens / 1_000_000) * costs["output"]
+        return round(input_cost + output_cost, 6)
+
+    def record_usage(self, provider: str, input_tokens: int = 0, output_tokens: int = 0,
+                     success: bool = True, task_id: str = None):
+        """Record a completed API request."""
+        cost = self.estimate_cost(provider, input_tokens, output_tokens)
+        usage = self._load_usage()
+
+        usage["total_input_tokens"] += input_tokens
+        usage["total_output_tokens"] += output_tokens
+        usage["total_cost_usd"] = round(usage["total_cost_usd"] + cost, 6)
+
+        if usage["budget_limit_usd"] is not None:
+            usage["budget_remaining_usd"] = round(
+                usage["budget_limit_usd"] - usage["total_cost_usd"], 6
+            )
+
+        # Per-provider stats
+        if provider not in usage["providers"]:
+            usage["providers"][provider] = {
+                "input_tokens": 0, "output_tokens": 0,
+                "cost_usd": 0.0, "requests": 0,
+                "failures": 0,
+            }
+        p = usage["providers"][provider]
+        p["input_tokens"] += input_tokens
+        p["output_tokens"] += output_tokens
+        p["cost_usd"] = round(p["cost_usd"] + cost, 6)
+        p["requests"] += 1
+        if not success:
+            p["failures"] += 1
+
+        # Append request entry (keep last 500)
+        usage["requests"].append({
+            "ts": datetime.now().isoformat(),
+            "provider": provider,
+            "input": input_tokens,
+            "output": output_tokens,
+            "cost": cost,
+            "success": success,
+            "task_id": task_id,
+        })
+        usage["requests"] = usage["requests"][-500:]
+
+        self._save_usage(usage)
+
+        # Track for circuit breaker
+        if success:
+            self._consecutive_failures[provider] = 0
+        else:
+            self._consecutive_failures[provider] = \
+                self._consecutive_failures.get(provider, 0) + 1
+
+        return cost
+
+    def check_budget(self) -> bool:
+        """Returns True if within budget (or no budget set)."""
+        usage = self._load_usage()
+        limit = usage.get("budget_limit_usd") or self.budget_usd
+        if limit is None:
+            return True
+        return usage["total_cost_usd"] < limit
+
+    def get_budget_status(self) -> dict:
+        """Get budget status summary."""
+        usage = self._load_usage()
+        limit = usage.get("budget_limit_usd") or self.budget_usd
+        spent = usage["total_cost_usd"]
+        return {
+            "limit_usd": limit,
+            "spent_usd": spent,
+            "remaining_usd": round((limit or 0) - spent, 6) if limit else None,
+            "percentage_used": round((spent / limit) * 100, 1) if limit else 0,
+            "within_budget": spent < limit if limit else True,
+        }
+
+    def is_circuit_open(self, provider: str) -> bool:
+        """Check if circuit breaker is tripped (too many failures)."""
+        return self._consecutive_failures.get(provider, 0) >= self.MAX_CONSECUTIVE_FAILURES
+
+    def get_backoff_seconds(self, provider: str) -> float:
+        """Calculate exponential backoff delay."""
+        failures = self._consecutive_failures.get(provider, 0)
+        if failures == 0:
+            return 0.0
+        delay = min(
+            self.BACKOFF_BASE_SECONDS * math.pow(2, failures - 1),
+            self.BACKOFF_MAX_SECONDS,
+        )
+        return delay
+
+    def wait_if_rate_limited(self, provider: str):
+        """Wait if needed to respect rate limits."""
+        now = time.time()
+        limit = RATE_LIMITS.get(provider, 60)
+        window = 60.0  # 1 minute window
+
+        if provider not in self._request_timestamps:
+            self._request_timestamps[provider] = []
+
+        # Clean old timestamps
+        self._request_timestamps[provider] = [
+            ts for ts in self._request_timestamps[provider]
+            if now - ts < window
+        ]
+
+        if len(self._request_timestamps[provider]) >= limit:
+            oldest = self._request_timestamps[provider][0]
+            wait_time = window - (now - oldest) + 0.5
+            if wait_time > 0:
+                time.sleep(wait_time)
+
+        self._request_timestamps[provider].append(time.time())
+
+    def reset_circuit(self, provider: str):
+        """Manually reset circuit breaker for a provider."""
+        self._consecutive_failures[provider] = 0
+
+    def get_summary(self) -> dict:
+        """Get usage summary."""
+        usage = self._load_usage()
+        return {
+            "total_input_tokens": usage["total_input_tokens"],
+            "total_output_tokens": usage["total_output_tokens"],
+            "total_cost_usd": usage["total_cost_usd"],
+            "budget": self.get_budget_status(),
+            "providers": usage["providers"],
+            "total_requests": len(usage["requests"]),
+        }
+
+    def reset(self):
+        """Reset usage for new session."""
+        usage = self._default_usage()
+        self._save_usage(usage)
+        self._request_timestamps.clear()
+        self._consecutive_failures.clear()
+
+
+# CLI interface
+if __name__ == "__main__":
+    import sys
+    tm = TokenManager()
+
+    if len(sys.argv) < 2:
+        print("Usage: python3 token_manager.py <command>")
+        print("  status  — Show token usage summary")
+        print("  reset   — Reset usage for new session")
+        sys.exit(0)
+
+    cmd = sys.argv[1]
+    if cmd == "status":
+        s = tm.get_summary()
+        print(f"\n💰 Token Usage Summary")
+        print(f"  Input:  {s['total_input_tokens']:,} tokens")
+        print(f"  Output: {s['total_output_tokens']:,} tokens")
+        print(f"  Cost:   ${s['total_cost_usd']:.4f}")
+        print(f"  Requests: {s['total_requests']}")
+        b = s["budget"]
+        if b["limit_usd"]:
+            bar_len = 20
+            filled = int(bar_len * b["percentage_used"] / 100)
+            bar = "█" * filled + "░" * (bar_len - filled)
+            print(f"  Budget: [{bar}] {b['percentage_used']}% (${b['spent_usd']:.4f}/${b['limit_usd']:.2f})")
+        print(f"\n  Per Provider:")
+        for name, data in s["providers"].items():
+            print(f"    {name}: {data['requests']} req, ${data['cost_usd']:.4f}, {data['failures']} failures")
+
+    elif cmd == "reset":
+        tm.reset()
+        print("✅ Token usage reset")