codymaster 4.1.4 → 4.3.0

package/skills/cm-content-factory/scripts/memory.py

@@ -0,0 +1,521 @@
+#!/usr/bin/env python3
+"""
+Memory Engine — 3-layer memory system for self-learning content factory.
+
+Architecture:
+  1. Semantic Memory — Long-term patterns, rules, writing style (persistent)
+  2. Episodic Memory — Per-session experiences with outcomes (time-bounded)
+  3. Working Memory — Current session context (ephemeral)
+
+Usage:
+    python3 memory.py --config content-factory.config.json --status
+    python3 memory.py --config content-factory.config.json --learn
+    python3 memory.py --config content-factory.config.json --test
+"""
+
+import json
+import sys
+import os
+import argparse
+from pathlib import Path
+from datetime import datetime, timedelta
+
+
+MEMORY_SCHEMA = {
+    "semantic": {
+        "writing_patterns.json": {
+            "description": "Accumulated writing style preferences",
+            "patterns": {},
+            "style_preferences": {
+                "tone": [],
+                "sentence_length": "medium",
+                "paragraph_style": "concise",
+                "heading_style": "question-based",
+                "cta_style": "soft"
+            },
+            "updated_at": None
+        },
+        "seo_patterns.json": {
+            "description": "SEO rules proven effective",
+            "patterns": {},
+            "effective_titles": [],
+            "effective_descriptions": [],
+            "keyword_strategies": [],
+            "updated_at": None
+        },
+        "niche_knowledge.json": {
+            "description": "Domain knowledge accumulated from research",
+            "domains": {},
+            "sources": [],
+            "confidence_scores": {},
+            "updated_at": None
+        },
+        "mistakes.json": {
+            "description": "Anti-patterns learned from user corrections",
+            "patterns": {},
+            "common_errors": [],
+            "deleted_content_reasons": [],
+            "updated_at": None
+        }
+    },
+    "episodic": {},
+    "working": {
+        "current_session.json": {
+            "session_id": None,
+            "started_at": None,
+            "phase": None,
+            "articles_processed": 0,
+            "errors": [],
+            "notes": []
+        }
+    }
+}
+
+
+class MemoryEngine:
+    """Three-layer memory system for the content factory."""
+
+    def __init__(self, config_path: str):
+        with open(config_path, "r", encoding="utf-8") as f:
+            self.config = json.load(f)
+
+        self.project_root = Path(config_path).resolve().parent
+        mem_cfg = self.config.get("memory", {})
+        self.memory_dir = self.project_root / mem_cfg.get("memory_dir", "memory")
+        self.auto_learn = mem_cfg.get("auto_learn", True)
+        self.max_episodic_days = mem_cfg.get("max_episodic_days", 90)
+
+        # Ensure directory structure
+        self._init_dirs()
+
+    def _init_dirs(self):
+        """Create memory directory structure if not exists."""
+        for subdir in ["semantic", "episodic", "working"]:
+            (self.memory_dir / subdir).mkdir(parents=True, exist_ok=True)
+
+        # Initialize semantic files if missing
+        for filename, schema in MEMORY_SCHEMA["semantic"].items():
+            filepath = self.memory_dir / "semantic" / filename
+            if not filepath.exists():
+                data = dict(schema)
+                data["updated_at"] = datetime.now().isoformat()
+                self._save_json(filepath, data)
+
+        # Initialize working memory
+        working_file = self.memory_dir / "working" / "current_session.json"
+        if not working_file.exists():
+            self._save_json(working_file, MEMORY_SCHEMA["working"]["current_session.json"])
+
+    # ──────────────────────────────────────────────
+    # SEMANTIC MEMORY — Long-term patterns & rules
+    # ──────────────────────────────────────────────
+
+    def load_semantic(self, category: str) -> dict:
+        """Load a semantic memory file."""
+        filepath = self.memory_dir / "semantic" / f"{category}.json"
+        if filepath.exists():
+            return self._load_json(filepath)
+        return {}
+
+    def save_semantic(self, category: str, data: dict):
+        """Save to semantic memory."""
+        data["updated_at"] = datetime.now().isoformat()
+        filepath = self.memory_dir / "semantic" / f"{category}.json"
+        self._save_json(filepath, data)
+
+    def add_writing_pattern(self, pattern_id: str, pattern: dict):
+        """Add or update a writing pattern."""
+        data = self.load_semantic("writing_patterns")
+        patterns = data.get("patterns", {})
+
+        if pattern_id in patterns:
+            # Update confidence based on usage
+            existing = patterns[pattern_id]
+            existing["applications"] = existing.get("applications", 0) + 1
+            existing["confidence"] = min(1.0, existing.get("confidence", 0.5) + 0.05)
+            existing["last_used"] = datetime.now().isoformat()
+        else:
+            pattern.setdefault("confidence", 0.5)
+            pattern.setdefault("applications", 1)
+            pattern.setdefault("created", datetime.now().isoformat())
+            pattern.setdefault("last_used", datetime.now().isoformat())
+            patterns[pattern_id] = pattern
+
+        data["patterns"] = patterns
+        self.save_semantic("writing_patterns", data)
+
+    def add_mistake(self, mistake: dict):
+        """Record a mistake to avoid in future."""
+        data = self.load_semantic("mistakes")
+        patterns = data.get("patterns", {})
+        mistakes_list = data.get("common_errors", [])
+
+        mistake_id = mistake.get("id", f"err-{datetime.now().strftime('%Y%m%d-%H%M%S')}")
+        mistake["recorded_at"] = datetime.now().isoformat()
+        mistake.setdefault("severity", "medium")
+        mistake.setdefault("occurrences", 1)
+
+        if mistake_id in patterns:
+            patterns[mistake_id]["occurrences"] = patterns[mistake_id].get("occurrences", 0) + 1
+            patterns[mistake_id]["last_seen"] = datetime.now().isoformat()
+        else:
+            patterns[mistake_id] = mistake
+
+        data["patterns"] = patterns
+        data["common_errors"] = mistakes_list
+        self.save_semantic("mistakes", data)
+
+    def add_niche_knowledge(self, domain: str, knowledge: dict):
+        """Add acquired domain knowledge from research."""
+        data = self.load_semantic("niche_knowledge")
+        domains = data.get("domains", {})
+
+        if domain not in domains:
+            domains[domain] = {
+                "topics": {},
+                "sources": [],
+                "acquired_at": datetime.now().isoformat()
+            }
+
+        for key, value in knowledge.items():
+            if key == "topics":
+                domains[domain]["topics"].update(value)
+            elif key == "sources":
+                domains[domain]["sources"].extend(value)
+            else:
+                domains[domain][key] = value
+
+        data["domains"] = domains
+        self.save_semantic("niche_knowledge", data)
+
+    def get_writing_context(self) -> dict:
+        """Get accumulated writing context for content generation."""
+        patterns = self.load_semantic("writing_patterns")
+        mistakes = self.load_semantic("mistakes")
+        seo = self.load_semantic("seo_patterns")
+        knowledge = self.load_semantic("niche_knowledge")
+
+        # Get top patterns by confidence
+        top_patterns = sorted(
+            patterns.get("patterns", {}).values(),
+            key=lambda p: p.get("confidence", 0) * p.get("applications", 1),
+            reverse=True
+        )[:10]
+
+        # Get recent mistakes to avoid
+        recent_mistakes = sorted(
+            mistakes.get("patterns", {}).values(),
+            key=lambda m: m.get("occurrences", 0),
+            reverse=True
+        )[:5]
+
+        return {
+            "style": patterns.get("style_preferences", {}),
+            "top_patterns": top_patterns,
+            "avoid": [m.get("description", m.get("id", "")) for m in recent_mistakes],
+            "seo_rules": seo.get("effective_titles", [])[:5],
+            "domain_knowledge": list(knowledge.get("domains", {}).keys())
+        }
+
+    # ──────────────────────────────────────────────
+    # EPISODIC MEMORY — Per-session experiences
+    # ──────────────────────────────────────────────
+
+    def record_experience(self, phase: str, outcome: str, details: dict):
+        """Record a pipeline experience."""
+        now = datetime.now()
+        year_dir = self.memory_dir / "episodic" / str(now.year)
+        year_dir.mkdir(parents=True, exist_ok=True)
+
+        episode = {
+            "id": f"ep-{now.strftime('%Y%m%d-%H%M%S')}",
+            "timestamp": now.isoformat(),
+            "phase": phase,
+            "outcome": outcome,  # success, partial, failure
+            "details": details,
+            "niche": self.config.get("niche", "unknown"),
+            "lessons": details.get("lessons", []),
+            "user_feedback": details.get("user_feedback", None)
+        }
+
+        filename = f"{now.strftime('%m-%d')}-{phase}.json"
+        filepath = year_dir / filename
+
+        # Append to existing or create new
+        if filepath.exists():
+            episodes = self._load_json(filepath)
+            if isinstance(episodes, list):
+                episodes.append(episode)
+            else:
+                episodes = [episodes, episode]
+        else:
+            episodes = [episode]
+
+        self._save_json(filepath, episodes)
+        return episode["id"]
+
+    def get_recent_experiences(self, days: int = 7, phase: str = None) -> list:
+        """Retrieve recent episodic memories."""
+        experiences = []
+        now = datetime.now()
+
+        for i in range(days):
+            day = now - timedelta(days=i)
+            year_dir = self.memory_dir / "episodic" / str(day.year)
+            if not year_dir.exists():
+                continue
+
+            for f in year_dir.glob(f"{day.strftime('%m-%d')}-*.json"):
+                data = self._load_json(f)
+                if isinstance(data, list):
+                    experiences.extend(data)
+                else:
+                    experiences.append(data)
+
+        if phase:
+            experiences = [e for e in experiences if e.get("phase") == phase]
+
+        return sorted(experiences, key=lambda x: x.get("timestamp", ""), reverse=True)
+
+    def cleanup_old_episodes(self):
+        """Remove episodic memories older than max_episodic_days."""
+        cutoff = datetime.now() - timedelta(days=self.max_episodic_days)
+        removed = 0
+
+        episodic_dir = self.memory_dir / "episodic"
+        for year_dir in episodic_dir.iterdir():
+            if not year_dir.is_dir():
+                continue
+            for f in year_dir.glob("*.json"):
+                try:
+                    data = self._load_json(f)
+                    entries = data if isinstance(data, list) else [data]
+                    if entries and entries[0].get("timestamp", ""):
+                        ts = datetime.fromisoformat(entries[0]["timestamp"])
+                        if ts < cutoff:
+                            f.unlink()
+                            removed += 1
+                except (ValueError, KeyError):
+                    pass
+
+        return removed
+
+    # ──────────────────────────────────────────────
+    # WORKING MEMORY — Current session
+    # ──────────────────────────────────────────────
+
+    def start_session(self, phase: str = None):
+        """Initialize working memory for current session."""
+        session = {
+            "session_id": f"ses-{datetime.now().strftime('%Y%m%d-%H%M%S')}",
+            "started_at": datetime.now().isoformat(),
+            "phase": phase,
+            "articles_processed": 0,
+            "errors": [],
+            "notes": []
+        }
+        self._save_json(self.memory_dir / "working" / "current_session.json", session)
+        return session["session_id"]
+
+    def update_session(self, updates: dict):
+        """Update current working memory."""
+        filepath = self.memory_dir / "working" / "current_session.json"
+        session = self._load_json(filepath) if filepath.exists() else {}
+        session.update(updates)
+        self._save_json(filepath, session)
+
+    def end_session(self, outcome: str = "complete"):
+        """Finalize session and create episodic record."""
+        filepath = self.memory_dir / "working" / "current_session.json"
+        session = self._load_json(filepath) if filepath.exists() else {}
+        session["ended_at"] = datetime.now().isoformat()
+        session["outcome"] = outcome
+
+        # Create episodic record from session
+        if session.get("phase"):
+            self.record_experience(
+                phase=session["phase"],
+                outcome=outcome,
+                details={
+                    "articles_processed": session.get("articles_processed", 0),
+                    "errors": session.get("errors", []),
+                    "notes": session.get("notes", [])
+                }
+            )
+
+        # Archive session
+        self._save_json(
+            self.memory_dir / "working" / "session_end.json", session
+        )
+
+    # ──────────────────────────────────────────────
+    # PATTERN EXTRACTION — Learn from experiences
+    # ──────────────────────────────────────────────
+
+    def extract_patterns(self):
+        """Analyze episodic memories and extract semantic patterns."""
+        experiences = self.get_recent_experiences(days=30)
+        if not experiences:
+            print("  ⚠️ No recent experiences to learn from")
+            return 0
+
+        patterns_added = 0
+
+        # Group by phase
+        phases = {}
+        for exp in experiences:
+            phase = exp.get("phase", "unknown")
+            phases.setdefault(phase, []).append(exp)
+
+        for phase, exps in phases.items():
+            successes = [e for e in exps if e.get("outcome") == "success"]
+            failures = [e for e in exps if e.get("outcome") == "failure"]
+
+            # Extract success patterns
+            if len(successes) >= 2:
+                self.add_writing_pattern(f"success-{phase}", {
+                    "description": f"{phase} phase success pattern ({len(successes)} occurrences)",
+                    "source": "episodic_analysis",
+                    "category": phase,
+                    "success_count": len(successes)
+                })
+                patterns_added += 1
+
+            # Extract failure patterns as mistakes
+            for fail in failures:
+                errors = fail.get("details", {}).get("errors", [])
+                for error in errors[:3]:
+                    self.add_mistake({
+                        "id": f"err-{phase}-{hash(str(error)) % 10000}",
+                        "description": str(error)[:200],
+                        "phase": phase,
+                        "severity": "high" if "delete" in str(error).lower() else "medium"
+                    })
+                    patterns_added += 1
+
+        print(f"  📊 Extracted {patterns_added} patterns from {len(experiences)} experiences")
+        return patterns_added
+
+    # ──────────────────────────────────────────────
+    # STATUS & UTILITIES
+    # ──────────────────────────────────────────────
+
+    def get_status(self) -> dict:
+        """Get memory system status."""
+        status = {"layers": {}}
+
+        # Semantic
+        semantic_dir = self.memory_dir / "semantic"
+        semantic_files = list(semantic_dir.glob("*.json")) if semantic_dir.exists() else []
+        total_patterns = 0
+        for f in semantic_files:
+            data = self._load_json(f)
+            total_patterns += len(data.get("patterns", {}))
+
+        status["layers"]["semantic"] = {
+            "files": len(semantic_files),
+            "total_patterns": total_patterns
+        }
+
+        # Episodic
+        episodic_dir = self.memory_dir / "episodic"
+        episodic_files = list(episodic_dir.rglob("*.json")) if episodic_dir.exists() else []
+        status["layers"]["episodic"] = {
+            "files": len(episodic_files),
+            "recent_7d": len(self.get_recent_experiences(7))
+        }
+
+        # Working
+        working_file = self.memory_dir / "working" / "current_session.json"
+        session = self._load_json(working_file) if working_file.exists() else {}
+        status["layers"]["working"] = {
+            "active_session": session.get("session_id"),
+            "phase": session.get("phase")
+        }
+
+        return status
+
+    def _load_json(self, filepath: Path) -> dict:
+        with open(filepath, "r", encoding="utf-8") as f:
+            return json.load(f)
+
+    def _save_json(self, filepath: Path, data):
+        filepath.parent.mkdir(parents=True, exist_ok=True)
+        with open(filepath, "w", encoding="utf-8") as f:
+            json.dump(data, f, ensure_ascii=False, indent=2)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Memory Engine — Self-learning system")
+    parser.add_argument("--config", required=True, help="Path to config JSON")
+    parser.add_argument("--status", action="store_true", help="Show memory status")
+    parser.add_argument("--learn", action="store_true", help="Extract patterns from experiences")
+    parser.add_argument("--cleanup", action="store_true", help="Remove old episodic memories")
+    parser.add_argument("--context", action="store_true", help="Show writing context")
+    parser.add_argument("--test", action="store_true", help="Run self-test")
+    args = parser.parse_args()
+
+    engine = MemoryEngine(args.config)
+
+    if args.status:
+        status = engine.get_status()
+        print(f"\n{'═' * 50}")
+        print(f"  🧠 Memory System Status")
+        print(f"{'═' * 50}")
+        for layer, info in status["layers"].items():
+            print(f"\n  📦 {layer.upper()}")
+            for k, v in info.items():
+                print(f"    {k}: {v}")
+        print(f"\n{'═' * 50}")
+
+    elif args.learn:
+        print("🧠 Learning from experiences...")
+        count = engine.extract_patterns()
+        print(f"  ✅ {count} patterns extracted")
+
+    elif args.cleanup:
+        removed = engine.cleanup_old_episodes()
+        print(f"🧹 Cleaned up {removed} old episodic memories")
+
+    elif args.context:
+        ctx = engine.get_writing_context()
+        print(json.dumps(ctx, ensure_ascii=False, indent=2))
+
+    elif args.test:
+        print("🧪 Testing memory system...")
+        # Test session lifecycle
+        sid = engine.start_session("test")
+        print(f"  ✅ Session started: {sid}")
+        engine.update_session({"articles_processed": 1, "notes": ["test note"]})
+        print(f"  ✅ Session updated")
+        engine.end_session("success")
+        print(f"  ✅ Session ended → episodic record created")
+
+        # Test pattern recording
+        engine.add_writing_pattern("test-pattern", {
+            "description": "Test writing pattern",
+            "category": "test"
+        })
+        print(f"  ✅ Writing pattern recorded")
+
+        engine.add_mistake({
+            "id": "test-mistake",
+            "description": "Test mistake",
+            "phase": "test"
+        })
+        print(f"  ✅ Mistake recorded")
+
+        # Test context
+        ctx = engine.get_writing_context()
+        print(f"  ✅ Writing context generated ({len(ctx)} keys)")
+
+        # Status
+        status = engine.get_status()
+        print(f"  ✅ Status: {status['layers']['semantic']['total_patterns']} patterns")
+        print(f"\n  🎉 All memory tests passed!")
+    else:
+        parser.print_help()
+
+
+if __name__ == "__main__":
+    main()