codymaster 4.8.0 → 7.0.0

package/skills/cm-content-factory/scripts/state_manager.py

@@ -1,320 +0,0 @@
-#!/usr/bin/env python3
-"""
-State Manager — Central state management for Content Factory dashboard.
-
-All pipeline scripts write state here; dashboard reads it.
-State persisted to `.content-factory-state.json` at project root.
-Events logged to `logs/events.jsonl` (append-only).
-
-Usage:
-    from state_manager import StateManager
-    sm = StateManager("/path/to/project")
-    sm.update_phase("write", "running", progress=0.5)
-    sm.add_task("write-article-1", "writing", {"topic": "SEO Tips"})
-    sm.log_event("info", "Article 1 started")
-"""
-
-import json
-import os
-import time
-import threading
-from pathlib import Path
-from datetime import datetime
-
-
-class StateManager:
-    """Thread-safe state manager for Content Factory pipeline."""
-
-    STATE_FILE = ".content-factory-state.json"
-    EVENTS_FILE = "logs/events.jsonl"
-
-    def __init__(self, project_root: str = None):
-        self.project_root = Path(project_root or os.getcwd()).resolve()
-        self.state_path = self.project_root / self.STATE_FILE
-        self.events_path = self.project_root / self.EVENTS_FILE
-        self._lock = threading.Lock()
-        self._ensure_dirs()
-
-    def _ensure_dirs(self):
-        """Create logs directory if needed."""
-        self.events_path.parent.mkdir(parents=True, exist_ok=True)
-
-    def _default_state(self) -> dict:
-        """Return default empty state."""
-        return {
-            "version": "2.0",
-            "started_at": datetime.now().isoformat(),
-            "updated_at": datetime.now().isoformat(),
-            "status": "idle",
-            "pipeline": {
-                "current_phase": None,
-                "phases": {
-                    "extract": {"status": "pending", "progress": 0, "started_at": None, "finished_at": None},
-                    "plan": {"status": "pending", "progress": 0, "started_at": None, "finished_at": None},
-                    "write": {"status": "pending", "progress": 0, "started_at": None, "finished_at": None},
-                    "audit": {"status": "pending", "progress": 0, "started_at": None, "finished_at": None},
-                    "seo": {"status": "pending", "progress": 0, "started_at": None, "finished_at": None},
-                    "publish": {"status": "pending", "progress": 0, "started_at": None, "finished_at": None},
-                },
-            },
-            "tasks": [],
-            "tokens": {
-                "total_input": 0,
-                "total_output": 0,
-                "total_cost_usd": 0.0,
-                "budget_limit_usd": None,
-                "providers": {},
-            },
-            "errors": [],
-            "stats": {
-                "articles_total": 0,
-                "articles_written": 0,
-                "articles_passed_audit": 0,
-                "articles_failed": 0,
-            },
-            "agents": [],
-        }
-
-    def load(self) -> dict:
-        """Load current state from disk."""
-        if not self.state_path.exists():
-            return self._default_state()
-        try:
-            with open(self.state_path, "r", encoding="utf-8") as f:
-                return json.load(f)
-        except (json.JSONDecodeError, IOError):
-            return self._default_state()
-
-    def save(self, state: dict):
-        """Save state to disk (thread-safe)."""
-        state["updated_at"] = datetime.now().isoformat()
-        with self._lock:
-            tmp = self.state_path.with_suffix(".tmp")
-            with open(tmp, "w", encoding="utf-8") as f:
-                json.dump(state, f, indent=2, ensure_ascii=False)
-            tmp.replace(self.state_path)
-
-    def update_phase(self, phase: str, status: str, progress: float = None, error: str = None):
-        """Update phase status and optional progress (0.0 - 1.0)."""
-        state = self.load()
-        phase_data = state["pipeline"]["phases"].get(phase, {})
-
-        phase_data["status"] = status
-        if progress is not None:
-            phase_data["progress"] = min(1.0, max(0.0, progress))
-        if status == "running" and not phase_data.get("started_at"):
-            phase_data["started_at"] = datetime.now().isoformat()
-        if status in ("done", "failed"):
-            phase_data["finished_at"] = datetime.now().isoformat()
-
-        state["pipeline"]["phases"][phase] = phase_data
-        state["pipeline"]["current_phase"] = phase if status == "running" else state["pipeline"]["current_phase"]
-
-        # Update overall status
-        if status == "running":
-            state["status"] = "running"
-        elif all(p["status"] == "done" for p in state["pipeline"]["phases"].values()):
-            state["status"] = "completed"
-        elif any(p["status"] == "failed" for p in state["pipeline"]["phases"].values()):
-            state["status"] = "error"
-
-        if error:
-            self._add_error(state, phase, error)
-
-        self.save(state)
-        self.log_event("info" if status != "failed" else "error",
-                       f"Phase {phase}: {status}" + (f" ({progress*100:.0f}%)" if progress else ""),
-                       {"phase": phase, "status": status})
-
-    def add_task(self, task_id: str, status: str, meta: dict = None):
-        """Add or update a task in the queue."""
-        state = self.load()
-        existing = next((t for t in state["tasks"] if t["id"] == task_id), None)
-        now = datetime.now().isoformat()
-
-        if existing:
-            existing["status"] = status
-            existing["updated_at"] = now
-            if meta:
-                existing["meta"].update(meta)
-        else:
-            state["tasks"].append({
-                "id": task_id,
-                "status": status,
-                "created_at": now,
-                "updated_at": now,
-                "meta": meta or {},
-            })
-
-        # Keep last 200 tasks max
-        if len(state["tasks"]) > 200:
-            done = [t for t in state["tasks"] if t["status"] in ("done", "failed")]
-            active = [t for t in state["tasks"] if t["status"] not in ("done", "failed")]
-            state["tasks"] = active + done[-100:]
-
-        self.save(state)
-
-    def update_tokens(self, provider: str, input_tokens: int = 0, output_tokens: int = 0, cost_usd: float = 0.0):
-        """Track token usage by provider."""
-        state = self.load()
-        tokens = state["tokens"]
-        tokens["total_input"] += input_tokens
-        tokens["total_output"] += output_tokens
-        tokens["total_cost_usd"] = round(tokens["total_cost_usd"] + cost_usd, 6)
-
-        if provider not in tokens["providers"]:
-            tokens["providers"][provider] = {"input": 0, "output": 0, "cost_usd": 0.0, "requests": 0}
-
-        p = tokens["providers"][provider]
-        p["input"] += input_tokens
-        p["output"] += output_tokens
-        p["cost_usd"] = round(p["cost_usd"] + cost_usd, 6)
-        p["requests"] += 1
-
-        self.save(state)
-
-    def check_budget(self) -> bool:
-        """Check if spending is within budget. Returns True if OK."""
-        state = self.load()
-        limit = state["tokens"].get("budget_limit_usd")
-        if limit is None:
-            return True
-        return state["tokens"]["total_cost_usd"] < limit
-
-    def set_budget(self, budget_usd: float):
-        """Set budget limit."""
-        state = self.load()
-        state["tokens"]["budget_limit_usd"] = budget_usd
-        self.save(state)
-
-    def update_stats(self, **kwargs):
-        """Update article statistics."""
-        state = self.load()
-        for k, v in kwargs.items():
-            if k in state["stats"]:
-                state["stats"][k] = v
-        self.save(state)
-
-    def increment_stat(self, key: str, amount: int = 1):
-        """Increment a stat counter."""
-        state = self.load()
-        if key in state["stats"]:
-            state["stats"][key] += amount
-        self.save(state)
-
-    def register_agent(self, agent_id: str, agent_type: str):
-        """Register an active agent."""
-        state = self.load()
-        existing = next((a for a in state["agents"] if a["id"] == agent_id), None)
-        now = datetime.now().isoformat()
-        if existing:
-            existing["last_seen"] = now
-        else:
-            state["agents"].append({
-                "id": agent_id,
-                "type": agent_type,
-                "registered_at": now,
-                "last_seen": now,
-            })
-        self.save(state)
-
-    def _add_error(self, state: dict, source: str, message: str):
-        """Add error to state (max 50)."""
-        state["errors"].append({
-            "source": source,
-            "message": message,
-            "timestamp": datetime.now().isoformat(),
-        })
-        state["errors"] = state["errors"][-50:]
-
-    def add_error(self, source: str, message: str):
-        """Public method to add an error."""
-        state = self.load()
-        self._add_error(state, source, message)
-        self.save(state)
-        self.log_event("error", message, {"source": source})
-
-    def log_event(self, level: str, message: str, data: dict = None):
-        """Append event to JSONL log file."""
-        event = {
-            "ts": datetime.now().isoformat(),
-            "level": level,
-            "msg": message,
-        }
-        if data:
-            event["data"] = data
-
-        with self._lock:
-            with open(self.events_path, "a", encoding="utf-8") as f:
-                f.write(json.dumps(event, ensure_ascii=False) + "\n")
-
-    def get_recent_events(self, n: int = 100) -> list:
-        """Read last N events from log."""
-        if not self.events_path.exists():
-            return []
-        try:
-            with open(self.events_path, "r", encoding="utf-8") as f:
-                lines = f.readlines()
-            events = []
-            for line in lines[-n:]:
-                line = line.strip()
-                if line:
-                    try:
-                        events.append(json.loads(line))
-                    except json.JSONDecodeError:
-                        pass
-            return events
-        except IOError:
-            return []
-
-    def reset(self):
-        """Reset state to default (for new pipeline run)."""
-        state = self._default_state()
-        self.save(state)
-        self.log_event("info", "State reset — new pipeline run")
-
-    def get_snapshot(self) -> dict:
-        """Get full state snapshot including recent events."""
-        state = self.load()
-        state["recent_events"] = self.get_recent_events(50)
-        return state
-
-
-# CLI interface
-if __name__ == "__main__":
-    import sys
-    sm = StateManager()
-
-    if len(sys.argv) < 2:
-        print("Usage: python3 state_manager.py <command>")
-        print("  status  — Show current state")
-        print("  reset   — Reset state")
-        print("  events  — Show recent events")
-        sys.exit(0)
-
-    cmd = sys.argv[1]
-    if cmd == "status":
-        state = sm.load()
-        print(f"\n🏭 Content Factory State")
-        print(f"  Status: {state['status']}")
-        print(f"  Updated: {state['updated_at']}")
-        print(f"\n  Pipeline:")
-        for phase, data in state["pipeline"]["phases"].items():
-            icon = {"pending": "⬜", "running": "🔄", "done": "✅", "failed": "❌"}.get(data["status"], "❓")
-            progress = f" ({data['progress']*100:.0f}%)" if data["progress"] > 0 else ""
-            print(f"    {icon} {phase}{progress}")
-        print(f"\n  Tokens: {state['tokens']['total_input']}in / {state['tokens']['total_output']}out")
-        print(f"  Cost: ${state['tokens']['total_cost_usd']:.4f}")
-        print(f"  Tasks: {len(state['tasks'])}")
-        print(f"  Errors: {len(state['errors'])}")
-        print(f"  Agents: {len(state['agents'])}")
-
-    elif cmd == "reset":
-        sm.reset()
-        print("✅ State reset")
-
-    elif cmd == "events":
-        events = sm.get_recent_events(20)
-        for e in events:
-            icon = {"info": "ℹ️", "warn": "⚠️", "error": "❌"}.get(e["level"], "•")
-            print(f"  {icon} [{e['ts'][-8:]}] {e['msg']}")

package/skills/cm-content-factory/scripts/token_manager.py

@@ -1,268 +0,0 @@
-#!/usr/bin/env python3
-"""
-Token Manager — Track token usage, costs, rate limits, and budget for AI providers.
-
-Monitors API consumption across providers (Gemini, Claude, OpenAI).
-Implements circuit breaker pattern and exponential backoff.
-
-Usage:
-    from token_manager import TokenManager
-    tm = TokenManager("/path/to/project")
-    tm.record_usage("gemini", input_tokens=500, output_tokens=200)
-    if not tm.check_budget():
-        print("Budget exceeded!")
-    tm.wait_if_rate_limited("gemini")
-"""
-
-import json
-import time
-import math
-from pathlib import Path
-from datetime import datetime
-
-
-# Cost per 1M tokens (USD) — 2025 pricing estimates
-PROVIDER_COSTS = {
-    "gemini": {"input": 0.15, "output": 0.60},
-    "gemini-pro": {"input": 1.25, "output": 5.00},
-    "claude-sonnet": {"input": 3.00, "output": 15.00},
-    "claude-haiku": {"input": 0.25, "output": 1.25},
-    "openai-gpt4o": {"input": 2.50, "output": 10.00},
-    "openai-gpt4o-mini": {"input": 0.15, "output": 0.60},
-}
-
-# Default rate limits (requests per minute)
-RATE_LIMITS = {
-    "gemini": 60,
-    "gemini-pro": 30,
-    "claude-sonnet": 50,
-    "claude-haiku": 100,
-    "openai-gpt4o": 60,
-    "openai-gpt4o-mini": 100,
-}
-
-
-class TokenManager:
-    """Token usage tracker with rate limiting and circuit breaker."""
-
-    USAGE_FILE = "logs/token_usage.json"
-    MAX_CONSECUTIVE_FAILURES = 5
-    BACKOFF_BASE_SECONDS = 2.0
-    BACKOFF_MAX_SECONDS = 120.0
-
-    def __init__(self, project_root: str = None, budget_usd: float = None):
-        self.project_root = Path(project_root or ".").resolve()
-        self.usage_path = self.project_root / self.USAGE_FILE
-        self.budget_usd = budget_usd
-        self._request_timestamps: dict[str, list] = {}
-        self._consecutive_failures: dict[str, int] = {}
-        self._ensure_dirs()
-
-    def _ensure_dirs(self):
-        self.usage_path.parent.mkdir(parents=True, exist_ok=True)
-
-    def _load_usage(self) -> dict:
-        if not self.usage_path.exists():
-            return self._default_usage()
-        try:
-            with open(self.usage_path, "r", encoding="utf-8") as f:
-                return json.load(f)
-        except (json.JSONDecodeError, IOError):
-            return self._default_usage()
-
-    def _default_usage(self) -> dict:
-        return {
-            "session_id": datetime.now().strftime("%Y%m%d_%H%M%S"),
-            "started_at": datetime.now().isoformat(),
-            "updated_at": datetime.now().isoformat(),
-            "total_input_tokens": 0,
-            "total_output_tokens": 0,
-            "total_cost_usd": 0.0,
-            "budget_limit_usd": self.budget_usd,
-            "budget_remaining_usd": self.budget_usd,
-            "providers": {},
-            "requests": [],
-        }
-
-    def _save_usage(self, usage: dict):
-        usage["updated_at"] = datetime.now().isoformat()
-        tmp = self.usage_path.with_suffix(".tmp")
-        with open(tmp, "w", encoding="utf-8") as f:
-            json.dump(usage, f, indent=2, ensure_ascii=False)
-        tmp.replace(self.usage_path)
-
-    def estimate_cost(self, provider: str, input_tokens: int, output_tokens: int) -> float:
-        """Estimate cost for a given request."""
-        costs = PROVIDER_COSTS.get(provider, {"input": 1.0, "output": 3.0})
-        input_cost = (input_tokens / 1_000_000) * costs["input"]
-        output_cost = (output_tokens / 1_000_000) * costs["output"]
-        return round(input_cost + output_cost, 6)
-
-    def record_usage(self, provider: str, input_tokens: int = 0, output_tokens: int = 0,
-                     success: bool = True, task_id: str = None):
-        """Record a completed API request."""
-        cost = self.estimate_cost(provider, input_tokens, output_tokens)
-        usage = self._load_usage()
-
-        usage["total_input_tokens"] += input_tokens
-        usage["total_output_tokens"] += output_tokens
-        usage["total_cost_usd"] = round(usage["total_cost_usd"] + cost, 6)
-
-        if usage["budget_limit_usd"] is not None:
-            usage["budget_remaining_usd"] = round(
-                usage["budget_limit_usd"] - usage["total_cost_usd"], 6
-            )
-
-        # Per-provider stats
-        if provider not in usage["providers"]:
-            usage["providers"][provider] = {
-                "input_tokens": 0, "output_tokens": 0,
-                "cost_usd": 0.0, "requests": 0,
-                "failures": 0,
-            }
-        p = usage["providers"][provider]
-        p["input_tokens"] += input_tokens
-        p["output_tokens"] += output_tokens
-        p["cost_usd"] = round(p["cost_usd"] + cost, 6)
-        p["requests"] += 1
-        if not success:
-            p["failures"] += 1
-
-        # Append request entry (keep last 500)
-        usage["requests"].append({
-            "ts": datetime.now().isoformat(),
-            "provider": provider,
-            "input": input_tokens,
-            "output": output_tokens,
-            "cost": cost,
-            "success": success,
-            "task_id": task_id,
-        })
-        usage["requests"] = usage["requests"][-500:]
-
-        self._save_usage(usage)
-
-        # Track for circuit breaker
-        if success:
-            self._consecutive_failures[provider] = 0
-        else:
-            self._consecutive_failures[provider] = \
-                self._consecutive_failures.get(provider, 0) + 1
-
-        return cost
-
-    def check_budget(self) -> bool:
-        """Returns True if within budget (or no budget set)."""
-        usage = self._load_usage()
-        limit = usage.get("budget_limit_usd") or self.budget_usd
-        if limit is None:
-            return True
-        return usage["total_cost_usd"] < limit
-
-    def get_budget_status(self) -> dict:
-        """Get budget status summary."""
-        usage = self._load_usage()
-        limit = usage.get("budget_limit_usd") or self.budget_usd
-        spent = usage["total_cost_usd"]
-        return {
-            "limit_usd": limit,
-            "spent_usd": spent,
-            "remaining_usd": round((limit or 0) - spent, 6) if limit else None,
-            "percentage_used": round((spent / limit) * 100, 1) if limit else 0,
-            "within_budget": spent < limit if limit else True,
-        }
-
-    def is_circuit_open(self, provider: str) -> bool:
-        """Check if circuit breaker is tripped (too many failures)."""
-        return self._consecutive_failures.get(provider, 0) >= self.MAX_CONSECUTIVE_FAILURES
-
-    def get_backoff_seconds(self, provider: str) -> float:
-        """Calculate exponential backoff delay."""
-        failures = self._consecutive_failures.get(provider, 0)
-        if failures == 0:
-            return 0.0
-        delay = min(
-            self.BACKOFF_BASE_SECONDS * math.pow(2, failures - 1),
-            self.BACKOFF_MAX_SECONDS,
-        )
-        return delay
-
-    def wait_if_rate_limited(self, provider: str):
-        """Wait if needed to respect rate limits."""
-        now = time.time()
-        limit = RATE_LIMITS.get(provider, 60)
-        window = 60.0  # 1 minute window
-
-        if provider not in self._request_timestamps:
-            self._request_timestamps[provider] = []
-
-        # Clean old timestamps
-        self._request_timestamps[provider] = [
-            ts for ts in self._request_timestamps[provider]
-            if now - ts < window
-        ]
-
-        if len(self._request_timestamps[provider]) >= limit:
-            oldest = self._request_timestamps[provider][0]
-            wait_time = window - (now - oldest) + 0.5
-            if wait_time > 0:
-                time.sleep(wait_time)
-
-        self._request_timestamps[provider].append(time.time())
-
-    def reset_circuit(self, provider: str):
-        """Manually reset circuit breaker for a provider."""
-        self._consecutive_failures[provider] = 0
-
-    def get_summary(self) -> dict:
-        """Get usage summary."""
-        usage = self._load_usage()
-        return {
-            "total_input_tokens": usage["total_input_tokens"],
-            "total_output_tokens": usage["total_output_tokens"],
-            "total_cost_usd": usage["total_cost_usd"],
-            "budget": self.get_budget_status(),
-            "providers": usage["providers"],
-            "total_requests": len(usage["requests"]),
-        }
-
-    def reset(self):
-        """Reset usage for new session."""
-        usage = self._default_usage()
-        self._save_usage(usage)
-        self._request_timestamps.clear()
-        self._consecutive_failures.clear()
-
-
-# CLI interface
-if __name__ == "__main__":
-    import sys
-    tm = TokenManager()
-
-    if len(sys.argv) < 2:
-        print("Usage: python3 token_manager.py <command>")
-        print("  status  — Show token usage summary")
-        print("  reset   — Reset usage for new session")
-        sys.exit(0)
-
-    cmd = sys.argv[1]
-    if cmd == "status":
-        s = tm.get_summary()
-        print(f"\n💰 Token Usage Summary")
-        print(f"  Input:  {s['total_input_tokens']:,} tokens")
-        print(f"  Output: {s['total_output_tokens']:,} tokens")
-        print(f"  Cost:   ${s['total_cost_usd']:.4f}")
-        print(f"  Requests: {s['total_requests']}")
-        b = s["budget"]
-        if b["limit_usd"]:
-            bar_len = 20
-            filled = int(bar_len * b["percentage_used"] / 100)
-            bar = "█" * filled + "░" * (bar_len - filled)
-            print(f"  Budget: [{bar}] {b['percentage_used']}% (${b['spent_usd']:.4f}/${b['limit_usd']:.2f})")
-        print(f"\n  Per Provider:")
-        for name, data in s["providers"].items():
-            print(f"    {name}: {data['requests']} req, ${data['cost_usd']:.4f}, {data['failures']} failures")
-
-    elif cmd == "reset":
-        tm.reset()
-        print("✅ Token usage reset")