codymaster 4.1.3 → 4.3.0

package/skills/cm-content-factory/scripts/plan.py

@@ -0,0 +1,163 @@
+#!/usr/bin/env python3
+"""
+Plan Phase — Config-driven topic planning.
+
+Generates topic queue from knowledge-base using configured article types.
+Delegates to existing topic_planner.py or generates from config.
+
+Usage:
+    python3 plan.py --config content-factory.config.json
+    python3 plan.py --config content-factory.config.json --dry-run
+    python3 plan.py --config content-factory.config.json --group CVG
+"""
+
+import json
+import sys
+import os
+import subprocess
+import argparse
+from pathlib import Path
+
+
+def load_config(config_path: str) -> dict:
+    with open(config_path, "r", encoding="utf-8") as f:
+        return json.load(f)
+
+
+def run_planner(project_root: Path, config: dict, dry_run: bool, group: str = None):
+    """Run topic planning using existing project script."""
+    planner = project_root / "scripts" / "topic_planner.py"
+    if not planner.exists():
+        print("❌ scripts/topic_planner.py not found.")
+        print("   Creating basic topic plan from knowledge-base...")
+        return generate_basic_plan(project_root, config, dry_run, group)
+
+    cmd = ["python3", str(planner)]
+    if group:
+        cmd.extend(["--group", group])
+
+    print(f"  📋 Running topic planner...")
+    print(f"  Article types: {len(config['content']['article_types'])}")
+    print(f"  Output: {config['output'].get('queue_dir', 'topics-queue/')}")
+
+    if dry_run:
+        cmd.append("--dry-run")
+
+    result = subprocess.run(cmd, cwd=str(project_root))
+    return result.returncode == 0
+
+
+def generate_basic_plan(project_root: Path, config: dict, dry_run: bool, group: str = None):
+    """Generate topic plan directly from config when no planner script exists."""
+    kb_dir = project_root / config["output"].get("knowledge_dir", "knowledge-base/")
+    queue_dir = project_root / config["output"].get("queue_dir", "topics-queue/")
+    index_file = kb_dir / "index.json"
+
+    if not index_file.exists():
+        print(f"❌ Knowledge-base index not found: {index_file}")
+        print("   Run extract phase first.")
+        return False
+
+    with open(index_file, "r", encoding="utf-8") as f:
+        index = json.load(f)
+
+    article_types = config["content"]["article_types"]
+    topics = []
+
+    entries = index.get("entries", [])
+    if group:
+        entries = [e for e in entries if e.get("group_code") == group]
+
+    for entry in entries:
+        for at in article_types:
+            topic_name = entry.get("name", entry.get("disease_name", "Unknown"))
+            title = at["title_template"].replace("{topic_name}", topic_name)
+            slug = _slugify(title)
+
+            topics.append({
+                "title": title,
+                "slug": slug,
+                "article_type": at["id"],
+                "category": at["category"],
+                "seo_intent": at.get("seo_intent", "informational"),
+                "tags": at.get("tags_base", []),
+                "source_entry": entry.get("name", ""),
+                "group_code": entry.get("group_code", ""),
+                "status": "pending"
+            })
+
+    if dry_run:
+        print(f"  [DRY RUN] Would generate {len(topics)} topics")
+        for t in topics[:5]:
+            print(f"    → {t['title'][:60]}...")
+        return True
+
+    queue_dir.mkdir(parents=True, exist_ok=True)
+    from datetime import datetime
+    batch_file = queue_dir / f"batch-{datetime.now().strftime('%Y%m%d-%H%M')}.json"
+
+    batch = {
+        "generated_at": datetime.now().isoformat(),
+        "total_topics": len(topics),
+        "config_niche": config["niche"],
+        "topics": topics
+    }
+
+    with open(batch_file, "w", encoding="utf-8") as f:
+        json.dump(batch, f, ensure_ascii=False, indent=2)
+
+    print(f"  ✅ Generated {len(topics)} topics → {batch_file}")
+    return True
+
+
+def _slugify(text: str, max_len: int = 80) -> str:
+    """Convert text to URL slug."""
+    import re
+    import unicodedata
+
+    # Vietnamese character map
+    char_map = {
+        'à': 'a', 'á': 'a', 'ả': 'a', 'ã': 'a', 'ạ': 'a',
+        'ă': 'a', 'ắ': 'a', 'ằ': 'a', 'ẳ': 'a', 'ẵ': 'a', 'ặ': 'a',
+        'â': 'a', 'ấ': 'a', 'ầ': 'a', 'ẩ': 'a', 'ẫ': 'a', 'ậ': 'a',
+        'đ': 'd',
+        'è': 'e', 'é': 'e', 'ẻ': 'e', 'ẽ': 'e', 'ẹ': 'e',
+        'ê': 'e', 'ế': 'e', 'ề': 'e', 'ể': 'e', 'ễ': 'e', 'ệ': 'e',
+        'ì': 'i', 'í': 'i', 'ỉ': 'i', 'ĩ': 'i', 'ị': 'i',
+        'ò': 'o', 'ó': 'o', 'ỏ': 'o', 'õ': 'o', 'ọ': 'o',
+        'ô': 'o', 'ố': 'o', 'ồ': 'o', 'ổ': 'o', 'ỗ': 'o', 'ộ': 'o',
+        'ơ': 'o', 'ớ': 'o', 'ờ': 'o', 'ở': 'o', 'ỡ': 'o', 'ợ': 'o',
+        'ù': 'u', 'ú': 'u', 'ủ': 'u', 'ũ': 'u', 'ụ': 'u',
+        'ư': 'u', 'ứ': 'u', 'ừ': 'u', 'ử': 'u', 'ữ': 'u', 'ự': 'u',
+        'ỳ': 'y', 'ý': 'y', 'ỷ': 'y', 'ỹ': 'y', 'ỵ': 'y',
+    }
+
+    text = text.lower()
+    result = []
+    for c in text:
+        result.append(char_map.get(c, c))
+    text = ''.join(result)
+
+    text = re.sub(r'[^a-z0-9\s-]', '', text)
+    text = re.sub(r'[\s-]+', '-', text).strip('-')
+    return text[:max_len]
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Plan Phase — Topic planning")
+    parser.add_argument("--config", required=True, help="Path to config JSON")
+    parser.add_argument("--dry-run", action="store_true")
+    parser.add_argument("--group", help="Filter by group code")
+    args = parser.parse_args()
+
+    config = load_config(args.config)
+    project_root = Path(args.config).resolve().parent
+
+    print(f"📋 PLAN Phase — {len(config['content']['article_types'])} article types")
+
+    success = run_planner(project_root, config, args.dry_run, args.group)
+    sys.exit(0 if success else 1)
+
+
+if __name__ == "__main__":
+    main()

package/skills/cm-content-factory/scripts/publish.py

@@ -0,0 +1,145 @@
+#!/usr/bin/env python3
+"""
+Publish Phase — Config-driven build + deploy.
+
+Runs build command, validates content, commits, and pushes to git.
+
+Usage:
+    python3 publish.py --config content-factory.config.json --build     # Build only
+    python3 publish.py --config content-factory.config.json --push      # Git push only
+    python3 publish.py --config content-factory.config.json             # Build + push
+    python3 publish.py --config content-factory.config.json --dry-run
+"""
+
+import json
+import sys
+import subprocess
+import argparse
+from pathlib import Path
+from datetime import datetime
+
+
+def load_config(config_path: str) -> dict:
+    with open(config_path, "r", encoding="utf-8") as f:
+        return json.load(f)
+
+
+def run_build(project_root: Path, config: dict, dry_run: bool) -> bool:
+    """Run the build command from config."""
+    build_cmd = config.get("pipeline", {}).get("build_command", "")
+    if not build_cmd:
+        print("  ⚠️ No build_command in config.pipeline")
+        return True
+
+    if dry_run:
+        print(f"  [DRY RUN] Would run: {build_cmd}")
+        return True
+
+    print(f"  🔨 Building: {build_cmd}")
+    result = subprocess.run(build_cmd, shell=True, cwd=str(project_root))
+    if result.returncode == 0:
+        print("  ✅ Build complete")
+    else:
+        print("  ❌ Build failed")
+    return result.returncode == 0
+
+
+def run_git_push(project_root: Path, config: dict, dry_run: bool) -> bool:
+    """Git add, commit, and push content."""
+    content_dir = config["output"]["content_dir"]
+    branch = config.get("pipeline", {}).get("git_branch", "main")
+
+    # Check for changes
+    result = subprocess.run(
+        ["git", "status", "--porcelain", content_dir],
+        capture_output=True, text=True, cwd=str(project_root)
+    )
+    changed_lines = [l for l in result.stdout.strip().split('\n') if l.strip()]
+    num_changes = len(changed_lines)
+
+    if num_changes == 0:
+        print("  ✅ No new content to publish")
+        return True
+
+    print(f"  📄 Found {num_changes} new/modified files")
+
+    if dry_run:
+        print("  [DRY RUN] Would commit:")
+        for line in changed_lines[:10]:
+            print(f"    {line}")
+        return True
+
+    # Stage
+    subprocess.run(["git", "add", content_dir], cwd=str(project_root))
+
+    # Also stage knowledge-base and topics-queue if they exist
+    kb_dir = config["output"].get("knowledge_dir", "knowledge-base/")
+    queue_dir = config["output"].get("queue_dir", "topics-queue/")
+    for extra in [kb_dir, queue_dir]:
+        if (project_root / extra).exists():
+            subprocess.run(["git", "add", extra], cwd=str(project_root))
+
+    # Commit
+    timestamp = datetime.now().strftime("%Y-%m-%d %H:%M")
+    niche = config.get("niche", "content")
+    msg = f"auto: publish {num_changes} {niche} articles ({timestamp})"
+
+    subprocess.run(["git", "commit", "-m", msg], cwd=str(project_root))
+
+    # Push
+    push_result = subprocess.run(
+        ["git", "push", "origin", branch],
+        capture_output=True, text=True, cwd=str(project_root)
+    )
+
+    if push_result.returncode != 0:
+        # Try alternate branch names
+        alt = "master" if branch == "main" else "main"
+        push_result = subprocess.run(
+            ["git", "push", "origin", alt],
+            capture_output=True, text=True, cwd=str(project_root)
+        )
+
+    if push_result.returncode == 0:
+        print(f"  ✅ Published {num_changes} articles to {branch}")
+    else:
+        print(f"  ⚠️ Push failed: {push_result.stderr[:200]}")
+        print("     Content is committed locally. Push manually.")
+
+    return True
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Publish Phase — Build + Deploy")
+    parser.add_argument("--config", required=True, help="Path to config JSON")
+    parser.add_argument("--build", action="store_true", help="Build only")
+    parser.add_argument("--push", action="store_true", help="Git push only")
+    parser.add_argument("--dry-run", action="store_true")
+    parser.add_argument("--group", help="(unused, for pipeline compat)")
+    args = parser.parse_args()
+
+    config = load_config(args.config)
+    project_root = Path(args.config).resolve().parent
+
+    print(f"🚀 PUBLISH Phase")
+
+    build_only = args.build and not args.push
+    push_only = args.push and not args.build
+    do_both = not args.build and not args.push
+
+    success = True
+    if build_only or do_both:
+        success = run_build(project_root, config, args.dry_run) and success
+
+    if push_only or do_both:
+        if config.get("pipeline", {}).get("auto_publish", False) or push_only:
+            success = run_git_push(project_root, config, args.dry_run) and success
+        else:
+            print("  ⬜ auto_publish is disabled in config")
+            print("     Use --push to force, or set pipeline.auto_publish: true")
+
+    sys.exit(0 if success else 1)
+
+
+if __name__ == "__main__":
+    main()

package/skills/cm-content-factory/scripts/research.py

@@ -0,0 +1,337 @@
+#!/usr/bin/env python3
+"""
+Research Engine — Auto-research any topic for the content factory.
+
+Capabilities:
+  1. Web search for topic keywords
+  2. Competitor content analysis (scrape top SERP results)
+  3. Knowledge synthesis → structured JSON
+  4. Auto-save to memory/semantic/niche_knowledge.json
+
+Usage:
+    python3 research.py --config content-factory.config.json --topic "fitness recovery"
+    python3 research.py --config content-factory.config.json --topic "bấm huyệt đau lưng" --depth deep
+    python3 research.py --config content-factory.config.json --competitor https://example.com
+    python3 research.py --config content-factory.config.json --gap-scan
+"""
+
+import json
+import sys
+import re
+import subprocess
+import argparse
+from pathlib import Path
+from datetime import datetime
+
+# Import memory engine if available
+SCRIPT_DIR = Path(__file__).resolve().parent
+sys.path.insert(0, str(SCRIPT_DIR))
+try:
+    from memory import MemoryEngine
+except ImportError:
+    MemoryEngine = None
+
+
+class ResearchEngine:
+    """Auto-research engine for discovering and synthesizing knowledge."""
+
+    def __init__(self, config_path: str):
+        with open(config_path, "r", encoding="utf-8") as f:
+            self.config = json.load(f)
+
+        self.config_path = config_path
+        self.project_root = Path(config_path).resolve().parent
+        self.research_cfg = self.config.get("research", {})
+        self.memory = MemoryEngine(config_path) if MemoryEngine else None
+
+    def research_topic(self, topic: str, depth: str = "standard") -> dict:
+        """Research a topic using web search and synthesis.
+
+        Args:
+            topic: The topic to research
+            depth: "quick" (3 sources), "standard" (5), "deep" (10)
+
+        Returns:
+            Structured knowledge dict
+        """
+        max_sources = {"quick": 3, "standard": 5, "deep": 10}.get(depth, 5)
+        max_sources = min(max_sources, self.research_cfg.get("max_sources_per_topic", 10))
+
+        print(f"  🔍 Researching: {topic} (depth: {depth}, max sources: {max_sources})")
+
+        knowledge = {
+            "topic": topic,
+            "researched_at": datetime.now().isoformat(),
+            "depth": depth,
+            "sources": [],
+            "key_points": [],
+            "subtopics": [],
+            "competitor_insights": [],
+            "content_angles": [],
+            "keywords": [],
+            "questions_people_ask": []
+        }
+
+        # Step 1: Generate search queries
+        queries = self._generate_queries(topic)
+        print(f"  📋 Generated {len(queries)} search queries")
+
+        # Step 2: Search and extract
+        for query in queries[:max_sources]:
+            result = self._search_web(query)
+            if result:
+                knowledge["sources"].append(result)
+
+        # Step 3: Extract key points from sources
+        knowledge["key_points"] = self._extract_key_points(knowledge["sources"])
+
+        # Step 4: Generate content angles
+        knowledge["content_angles"] = self._suggest_content_angles(topic, knowledge["key_points"])
+
+        # Step 5: Save to memory
+        if self.memory:
+            self.memory.add_niche_knowledge(topic, {
+                "topics": {topic: knowledge},
+                "sources": [s.get("url", "") for s in knowledge["sources"]]
+            })
+            print(f"  💾 Saved to memory/semantic/niche_knowledge.json")
+
+        # Save as standalone research file
+        research_dir = self.project_root / "memory" / "research"
+        research_dir.mkdir(parents=True, exist_ok=True)
+        slug = re.sub(r'[^a-z0-9]+', '-', topic.lower())[:50]
+        research_file = research_dir / f"{slug}.json"
+        with open(research_file, "w", encoding="utf-8") as f:
+            json.dump(knowledge, f, ensure_ascii=False, indent=2)
+        print(f"  📄 Research saved: {research_file.relative_to(self.project_root)}")
+
+        return knowledge
+
+    def analyze_competitor(self, url: str) -> dict:
+        """Analyze a competitor's content structure."""
+        print(f"  🔎 Analyzing competitor: {url}")
+
+        analysis = {
+            "url": url,
+            "analyzed_at": datetime.now().isoformat(),
+            "content_structure": {},
+            "topics_covered": [],
+            "missing_from_us": [],
+            "their_strengths": [],
+            "our_advantages": []
+        }
+
+        # Try to extract page content
+        content = self._fetch_url(url)
+        if content:
+            analysis["content_structure"] = self._analyze_structure(content)
+            analysis["topics_covered"] = self._extract_topics(content)
+
+            # Compare with our content
+            our_content = self._get_our_content_inventory()
+            their_topics = set(t.lower() for t in analysis["topics_covered"])
+            our_topics = set(t.lower() for t in our_content)
+            analysis["missing_from_us"] = list(their_topics - our_topics)
+
+        return analysis
+
+    def scan_content_gaps(self) -> list:
+        """Scan for content gaps — topics we should cover but don't."""
+        print(f"  🕳️ Scanning for content gaps...")
+
+        # Get our current content inventory
+        our_topics = self._get_our_content_inventory()
+        print(f"  📄 Our inventory: {len(our_topics)} articles")
+
+        # Get competitor URLs from config
+        competitor_urls = self.research_cfg.get("competitor_urls", [])
+
+        gaps = []
+        for url in competitor_urls:
+            analysis = self.analyze_competitor(url)
+            for missing in analysis.get("missing_from_us", []):
+                gaps.append({
+                    "topic": missing,
+                    "found_at": url,
+                    "priority": "medium",
+                    "estimated_difficulty": "medium"
+                })
+
+        # Also check knowledge base for unwritten topics
+        kb_dir = self.project_root / self.config["output"].get("knowledge_dir", "knowledge-base/")
+        if kb_dir.exists():
+            for group_dir in kb_dir.iterdir():
+                if not group_dir.is_dir():
+                    continue
+                for json_file in group_dir.glob("*.json"):
+                    with open(json_file) as f:
+                        data = json.load(f)
+                    disease_name = data.get("disease_name", json_file.stem)
+                    if disease_name.lower() not in [t.lower() for t in our_topics]:
+                        gaps.append({
+                            "topic": disease_name,
+                            "found_at": "knowledge-base",
+                            "priority": "high",
+                            "estimated_difficulty": "low"
+                        })
+
+        print(f"  🕳️ Found {len(gaps)} content gaps")
+
+        # Save gap analysis
+        gap_file = self.project_root / "content-gaps.json"
+        with open(gap_file, "w", encoding="utf-8") as f:
+            json.dump({
+                "scanned_at": datetime.now().isoformat(),
+                "total_gaps": len(gaps),
+                "gaps": gaps
+            }, f, ensure_ascii=False, indent=2)
+
+        return gaps
+
+    # ──────────────────────────────────────────────
+    # Internal methods
+    # ──────────────────────────────────────────────
+
+    def _generate_queries(self, topic: str) -> list:
+        """Generate search queries from topic."""
+        lang = self.config.get("brand", {}).get("language", "vi")
+        niche = self.config.get("niche", "")
+
+        queries = [topic]
+
+        if lang == "vi":
+            queries.extend([
+                f"{topic} là gì",
+                f"cách điều trị {topic}",
+                f"{topic} triệu chứng nguyên nhân",
+                f"{topic} bấm huyệt" if "medspa" in niche else f"{topic} hướng dẫn",
+            ])
+        else:
+            queries.extend([
+                f"what is {topic}",
+                f"{topic} complete guide",
+                f"{topic} symptoms causes treatment",
+                f"best {topic} tips",
+            ])
+
+        return queries
+
+    def _search_web(self, query: str) -> dict:
+        """Search web for a query. Returns structured result."""
+        # Use curl for basic URL fetching, or delegate to browser tool
+        # This is a lightweight fallback — full research uses the AI agent's search
+        return {
+            "query": query,
+            "searched_at": datetime.now().isoformat(),
+            "method": "pending_ai_search",
+            "note": "Full search requires AI agent context. Use /research workflow."
+        }
+
+    def _fetch_url(self, url: str) -> str:
+        """Fetch URL content."""
+        try:
+            result = subprocess.run(
+                ["curl", "-s", "-L", "--max-time", "10", url],
+                capture_output=True, text=True, timeout=15
+            )
+            return result.stdout[:50000] if result.returncode == 0 else ""
+        except Exception:
+            return ""
+
+    def _analyze_structure(self, html: str) -> dict:
+        """Extract content structure from HTML."""
+        h1 = re.findall(r'<h1[^>]*>(.*?)</h1>', html, re.IGNORECASE | re.DOTALL)
+        h2 = re.findall(r'<h2[^>]*>(.*?)</h2>', html, re.IGNORECASE | re.DOTALL)
+        h3 = re.findall(r'<h3[^>]*>(.*?)</h3>', html, re.IGNORECASE | re.DOTALL)
+
+        # Clean HTML tags from headings
+        clean = lambda items: [re.sub(r'<[^>]+>', '', i).strip() for i in items]
+
+        return {
+            "h1": clean(h1),
+            "h2": clean(h2)[:20],
+            "h3": clean(h3)[:30],
+            "word_count_estimate": len(html.split()) // 3  # rough estimate
+        }
+
+    def _extract_topics(self, html: str) -> list:
+        """Extract topic names from HTML content."""
+        h2 = re.findall(r'<h2[^>]*>(.*?)</h2>', html, re.IGNORECASE | re.DOTALL)
+        return [re.sub(r'<[^>]+>', '', h).strip() for h in h2[:30]]
+
+    def _extract_key_points(self, sources: list) -> list:
+        """Extract key points from research sources."""
+        return [f"Source: {s.get('query', 'unknown')}" for s in sources if s]
+
+    def _suggest_content_angles(self, topic: str, key_points: list) -> list:
+        """Suggest content angles based on research."""
+        article_types = self.config.get("content", {}).get("article_types", [])
+        return [
+            {
+                "type": at["id"],
+                "title": at["title_template"].replace("{topic_name}", topic),
+                "angle": at.get("seo_intent", "informational")
+            }
+            for at in article_types
+        ]
+
+    def _get_our_content_inventory(self) -> list:
+        """Get list of our published content titles."""
+        content_dir = self.project_root / self.config["output"]["content_dir"]
+        titles = []
+
+        if content_dir.exists():
+            for md in content_dir.glob("*.md"):
+                with open(md, "r", encoding="utf-8") as f:
+                    content = f.read(500)
+                match = re.search(r'title:\s*["\']?(.+?)["\']?\s*$', content, re.MULTILINE)
+                if match:
+                    titles.append(match.group(1).strip())
+                else:
+                    titles.append(md.stem)
+
+        return titles
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Research Engine — Auto-research topics")
+    parser.add_argument("--config", required=True, help="Path to config JSON")
+    parser.add_argument("--topic", help="Topic to research")
+    parser.add_argument("--depth", default="standard", choices=["quick", "standard", "deep"])
+    parser.add_argument("--competitor", help="Competitor URL to analyze")
+    parser.add_argument("--gap-scan", action="store_true", help="Scan for content gaps")
+    args = parser.parse_args()
+
+    engine = ResearchEngine(args.config)
+
+    if args.topic:
+        print(f"🔬 RESEARCH Mode — Topic: {args.topic}")
+        result = engine.research_topic(args.topic, args.depth)
+        print(f"\n  📊 Results:")
+        print(f"    Sources: {len(result['sources'])}")
+        print(f"    Key points: {len(result['key_points'])}")
+        print(f"    Content angles: {len(result['content_angles'])}")
+        for angle in result["content_angles"]:
+            print(f"      → {angle['title'][:60]}...")
+
+    elif args.competitor:
+        print(f"🔎 COMPETITOR Analysis")
+        result = engine.analyze_competitor(args.competitor)
+        print(f"\n  Topics covered: {len(result.get('topics_covered', []))}")
+        print(f"  Missing from us: {len(result.get('missing_from_us', []))}")
+
+    elif args.gap_scan:
+        print(f"🕳️ CONTENT GAP Scan")
+        gaps = engine.scan_content_gaps()
+        high = [g for g in gaps if g["priority"] == "high"]
+        print(f"\n  Total gaps: {len(gaps)}")
+        print(f"  High priority: {len(high)}")
+        for g in high[:10]:
+            print(f"    → {g['topic']} (from: {g['found_at']})")
+
+    else:
+        parser.print_help()
+
+
+if __name__ == "__main__":
+    main()