@mc1global/opencode-jarvis 0.12.0 → 0.13.1

package/scripts/utilities/convert_skill_to_toon.py

@@ -0,0 +1,671 @@
+#!/usr/bin/env python3
+"""
+Skill to TOON Converter - LLM-Powered Intelligent Conversion.
+
+Uses Anthropic Haiku API for context-aware conversion:
+1. Analyze skill context (type, complexity, sections)
+2. Convert sections using context-informed prompts
+3. Validate semantic preservation
+4. Assemble optimized TOON skill
+
+Token Strategy:
+- Phase 1: Send metadata only for context (Haiku analyzes)
+- Phase 2: Convert sections one-by-one using context
+- Phase 3: Validate final result
+
+Usage:
+    python convert_skill_to_toon.py --skill rag-manager --dry-run
+    python convert_skill_to_toon.py --skill rag-manager --apply
+    python convert_skill_to_toon.py --batch high-usage --dry-run
+"""
+
+import os
+import sys
+import json
+import yaml
+import re
+from pathlib import Path
+from typing import Dict, List, Any, Optional
+from dataclasses import dataclass
+
+# Add project root
+sys.path.insert(0, str(Path(__file__).parent.parent.parent))
+
+# Anthropic client
+try:
+    from anthropic import Anthropic
+    ANTHROPIC_AVAILABLE = True
+except ImportError:
+    ANTHROPIC_AVAILABLE = False
+    print("ERROR: anthropic package not installed")
+    print("Run: pip install anthropic")
+    sys.exit(1)
+
+
+@dataclass
+class SkillContext:
+    """Context extracted from skill analysis."""
+    skill_name: str
+    skill_type: str  # file_manager, search_tool, data_processor, etc.
+    complexity: str  # simple, medium, complex
+    has_filters: bool
+    has_multiple_commands: bool
+    sections: List[str]
+    primary_command: Optional[str]
+    token_savings_claim: Optional[str]
+    estimated_tokens_original: int
+
+
+@dataclass
+class ConversionResult:
+    """Result of section conversion."""
+    section_name: str
+    original_content: str
+    converted_content: str
+    tokens_before: int
+    tokens_after: int
+    validation_passed: bool
+
+
+class SkillConverter:
+    """LLM-powered skill converter using context-aware prompts."""
+
+    def __init__(self, api_key: str, dry_run: bool = False):
+        """Initialize converter with Anthropic API."""
+        self.client = Anthropic(api_key=api_key)
+        self.model = "claude-3-5-haiku-20241022"
+        self.dry_run = dry_run
+
+    def _call_haiku(self, prompt: str, max_tokens: int = 2000) -> str:
+        """Call Haiku API with prompt."""
+        try:
+            message = self.client.messages.create(
+                model=self.model,
+                max_tokens=max_tokens,
+                temperature=0,  # Deterministic
+                messages=[
+                    {
+                        "role": "user",
+                        "content": prompt
+                    }
+                ]
+            )
+            return message.content[0].text
+
+        except Exception as e:
+            raise RuntimeError(f"Haiku API call failed: {e}")
+
+    def extract_sections(self, skill_content: str) -> Dict[str, str]:
+        """
+        Extract sections from skill markdown.
+
+        Returns:
+            Dict mapping section names to content
+        """
+        sections = {}
+
+        # Extract frontmatter
+        frontmatter_match = re.search(r'^---\n(.*?)\n---', skill_content, re.DOTALL)
+        if frontmatter_match:
+            sections['frontmatter'] = frontmatter_match.group(0)
+
+        # Extract main sections by headers
+        # ## When to Use, ## Quick Start, ## Commands, etc.
+        section_pattern = r'##\s+([^\n]+)\n(.*?)(?=\n##|\Z)'
+        for match in re.finditer(section_pattern, skill_content, re.DOTALL):
+            section_name = match.group(1).strip()
+            section_content = match.group(2).strip()
+            sections[section_name] = section_content
+
+        # Extract code blocks
+        code_blocks = re.findall(r'```(?:bash|python|toon)?\n(.*?)```', skill_content, re.DOTALL)
+        if code_blocks:
+            sections['code_examples'] = code_blocks
+
+        return sections
+
+    def analyze_skill_context(self, skill_path: Path) -> SkillContext:
+        """
+        Phase 1: Analyze skill to extract context.
+
+        Uses Haiku to understand skill type, complexity, and structure.
+        This context informs conversion prompts.
+        """
+        print(f" Analyzing skill context: {skill_path.name}")
+
+        # Read skill
+        skill_content = skill_path.read_text()
+
+        # Extract frontmatter for quick analysis
+        frontmatter_match = re.search(r'^---\n(.*?)\n---', skill_content, re.DOTALL)
+        frontmatter = yaml.safe_load(frontmatter_match.group(1)) if frontmatter_match else {}
+
+        # Get first 500 chars for preview
+        preview = skill_content[:500]
+
+        # Count sections
+        sections = self.extract_sections(skill_content)
+
+        prompt = f"""Analyze this skill file and extract context for optimization.
+
+SKILL FRONTMATTER:
+{json.dumps(frontmatter, indent=2)}
+
+SKILL PREVIEW (first 500 chars):
+{preview}
+
+SECTIONS FOUND:
+{list(sections.keys())}
+
+TOTAL LINES: {len(skill_content.splitlines())}
+
+Provide analysis in this EXACT JSON format:
+{{
+  "skill_name": "name from frontmatter",
+  "skill_type": "file_manager|search_tool|data_processor|config_manager|orchestrator|other",
+  "complexity": "simple|medium|complex",
+  "has_filters": true|false,
+  "has_multiple_commands": true|false,
+  "sections_identified": ["list", "of", "sections"],
+  "primary_command": "main command syntax or null",
+  "token_savings_claim": "percentage if mentioned or null",
+  "estimated_tokens_original": number
+}}
+
+Be concise. Output ONLY the JSON, nothing else."""
+
+        response = self._call_haiku(prompt, max_tokens=500)
+
+        # Parse JSON response
+        try:
+            # Extract JSON from response (in case Haiku adds text)
+            json_match = re.search(r'\{.*\}', response, re.DOTALL)
+            if json_match:
+                context_data = json.loads(json_match.group(0))
+            else:
+                context_data = json.loads(response)
+
+            context = SkillContext(
+                skill_name=context_data["skill_name"],
+                skill_type=context_data["skill_type"],
+                complexity=context_data["complexity"],
+                has_filters=context_data["has_filters"],
+                has_multiple_commands=context_data["has_multiple_commands"],
+                sections=context_data["sections_identified"],
+                primary_command=context_data.get("primary_command"),
+                token_savings_claim=context_data.get("token_savings_claim"),
+                estimated_tokens_original=context_data["estimated_tokens_original"]
+            )
+
+            print(f"ok:context extracted:")
+            print(f"   Type: {context.skill_type}")
+            print(f"   Complexity: {context.complexity}")
+            print(f"   Commands: {'multiple' if context.has_multiple_commands else 'single'}")
+            print(f"   Filters: {'yes' if context.has_filters else 'no'}")
+            print(f"   Sections: {len(context.sections)}")
+            print(f"   Estimated tokens: {context.estimated_tokens_original}")
+
+            return context
+
+        except json.JSONDecodeError as e:
+            raise ValueError(f"Failed to parse Haiku response as JSON: {e}\nResponse: {response}")
+
+    def convert_section(
+        self,
+        section_name: str,
+        section_content: str,
+        context: SkillContext
+    ) -> ConversionResult:
+        """
+        Phase 2: Convert individual section using context-informed prompt.
+
+        Args:
+            section_name: Name of section (e.g., "When to Use", "Commands")
+            section_content: Section content to convert
+            context: Skill context from analysis phase
+
+        Returns:
+            ConversionResult with converted content and metrics
+        """
+        print(f"   Converting section: {section_name}")
+
+        # Create context-aware prompt
+        prompt = f"""Convert this skill section to TOON-optimized format for LLM consumption.
+
+SKILL CONTEXT:
+- Type: {context.skill_type}
+- Complexity: {context.complexity}
+- Has filters: {context.has_filters}
+- Multiple commands: {context.has_multiple_commands}
+
+SECTION NAME: {section_name}
+
+SECTION CONTENT:
+{section_content}
+
+CRITICAL CONVERSION RULES:
+1. Output MUST be valid TOON format (no markdown, no prose)
+2. Extract ALL commands from section (not just examples)
+3. Keep command syntax EXACTLY as-is (character-perfect)
+4. Use pipe-separated: item1|item2|item3 (NO bullet lists)
+5. Remove ALL explanations (LLM doesn't need them)
+6. Keep maximum 3 examples (most representative)
+7. NO decorative emojis, NO whitespace padding
+8. Output format depends on section type (see below)
+
+SECTION-SPECIFIC FORMATS (FOLLOW EXACTLY):
+
+IF section is "When to Use" OR contains criteria:
+when: scenario1|scenario2|scenario3
+avoid: scenario1→alternative|scenario2→alternative
+
+IF section contains commands OR code blocks with "python scripts/":
+commands[N]{{cmd,syntax,output}}:
+  cmd1,"full command syntax here",expected_output_format
+  cmd2,"full command syntax here",expected_output_format
+(Include ALL commands found, not just 1-2 examples)
+
+IF section is "Examples" OR "Common Patterns":
+nlp_examples[3]:
+  natural language description of task 1
+  natural language description of task 2
+  natural language description of task 3
+(Maximum 3, express as NLP queries for command_intelligence.py, NOT full command syntax)
+Example: "search for authentication code" instead of "python scripts/rag/rag_cli.py search 'authentication'"
+
+IF section is "Troubleshooting" OR "Common Issues":
+troubleshoot[N]:
+  issue1: solution_command_or_action
+  issue2: solution_command_or_action
+
+IF section is "Features" OR describes benefits:
+(Skip entirely - LLM doesn't need marketing)
+
+IF section is "Configuration" OR "Setup":
+config: key_setting_only (one line max)
+
+CRITICAL: Output ONLY the TOON format for THIS section.
+- NO section headers
+- NO markdown formatting
+- NO explanatory text
+- JUST the TOON structure
+
+If section doesn't fit any format above, output:
+note: {{ultra_compact_summary_one_line}}
+"""
+
+        response = self._call_haiku(prompt, max_tokens=1000)
+
+        # Estimate tokens
+        tokens_before = len(section_content) // 4
+        tokens_after = len(response) // 4
+
+        result = ConversionResult(
+            section_name=section_name,
+            original_content=section_content,
+            converted_content=response.strip(),
+            tokens_before=tokens_before,
+            tokens_after=tokens_after,
+            validation_passed=True  # TODO: Add validation
+        )
+
+        print(f"      Before: {tokens_before} tokens")
+        print(f"      After: {tokens_after} tokens")
+        print(f"      Economy: {((tokens_before - tokens_after) / tokens_before * 100):.1f}%")
+
+        return result
+
+    def assemble_toon_skill(
+        self,
+        frontmatter: str,
+        converted_sections: List[ConversionResult],
+        skill_name: str
+    ) -> str:
+        """
+        Phase 3: Assemble converted sections into final TOON skill.
+        """
+        print(f" Assembling TOON skill: {skill_name}")
+
+        # Start with frontmatter (preserved exactly)
+        parts = [frontmatter, "", f"# {skill_name}", ""]
+
+        # Add usage pattern (command intelligence integration)
+        parts.append("usage_pattern: command_intelligence.py query \"{nlp_description}\" | cli_executor.py")
+        parts.append("")
+
+        # Add converted sections in logical order
+        section_order = [
+            "usage",
+            "when",
+            "avoid",
+            "commands",
+            "filters",
+            "nlp_examples",
+            "examples",
+            "savings",
+            "troubleshoot",
+            "reference"
+        ]
+
+        # Map section names to content
+        section_map = {
+            result.section_name.lower(): result.converted_content
+            for result in converted_sections
+        }
+
+        # Add sections in order
+        for section_key in section_order:
+            # Find matching section (flexible matching)
+            for sec_name, content in section_map.items():
+                if section_key in sec_name.lower() or sec_name in section_key:
+                    parts.append(content)
+                    parts.append("")
+                    break
+
+        return "\n".join(parts).strip() + "\n"
+
+    def extract_all_data(self, skill_content: str) -> Dict[str, Any]:
+        """
+        PASS 1: Extract ALL structured data from skill.
+
+        Haiku sees complete skill and extracts:
+        - All commands with exact syntax
+        - All filters/parameters
+        - NLP use cases
+        - When/avoid criteria
+        """
+        print(" PASS 1: Extracting structured data...")
+
+        prompt = f"""Extract ALL structured data from this skill for TOON conversion.
+
+COMPLETE SKILL CONTENT:
+{skill_content}
+
+Extract and output in this EXACT JSON format:
+{{
+  "skill_name": "name from frontmatter",
+  "commands": [
+    {{"name": "command_name", "syntax": "full python scripts/... command", "output": "output_format"}},
+    ...all commands found in the skill...
+  ],
+  "filters": [
+    {{"name": "filter_name", "flag": "--flag-name", "values": ["val1", "val2"]}},
+    ...all filters if present...
+  ],
+  "when_criteria": ["scenario1", "scenario2", ...all scenarios when to use...],
+  "avoid_criteria": ["scenario1→alternative", "scenario2→alternative", ...all scenarios to avoid...],
+  "nlp_examples": ["nlp description 1", "nlp description 2", "nlp description 3"],
+  "troubleshoot": [{{"issue": "issue_description", "solution": "solution_command"}}],
+  "savings_claim": "percentage if mentioned",
+  "primary_usage": "most common command pattern"
+}}
+
+CRITICAL:
+- Extract ALL commands (index, search, update, delete, etc.) - don't miss any
+- Keep command syntax EXACTLY as written
+- Convert regular examples to NLP descriptions (e.g., "search for authentication" not full command)
+- Include all filters found (workspace, domain, type, etc.)
+
+Output ONLY valid JSON, nothing else."""
+
+        response = self._call_haiku(prompt, max_tokens=2000)
+
+        # Parse JSON
+        try:
+            json_match = re.search(r'\{.*\}', response, re.DOTALL)
+            if json_match:
+                extracted_data = json.loads(json_match.group(0))
+            else:
+                extracted_data = json.loads(response)
+
+            print(f"ok:extracted:")
+            print(f"   Commands: {len(extracted_data.get('commands', []))}")
+            print(f"   Filters: {len(extracted_data.get('filters', []))}")
+            print(f"   When criteria: {len(extracted_data.get('when_criteria', []))}")
+            print(f"   NLP examples: {len(extracted_data.get('nlp_examples', []))}")
+
+            return extracted_data
+
+        except json.JSONDecodeError as e:
+            raise ValueError(f"Failed to parse extraction JSON: {e}\nResponse: {response}")
+
+    def format_toon_skill(self, frontmatter: str, extracted_data: Dict[str, Any]) -> str:
+        """
+        PASS 2: Format TOON skill from extracted data.
+
+        Haiku generates TOON skill following example format.
+        """
+        print(" PASS 2: Formatting TOON skill...")
+
+        prompt = f"""Generate NLP-ONLY TOON skill file using this extracted data.
+
+FRONTMATTER (preserve exactly):
+{frontmatter}
+
+EXTRACTED DATA:
+{json.dumps(extracted_data, indent=2)}
+
+OUTPUT FORMAT - NLP-FIRST APPROACH (follow exactly):
+
+---
+[frontmatter - preserve exactly as provided]
+---
+
+# [skill_name]
+
+usage: command_intelligence.py query "{{nlp_query}}" | cli_executor.py
+
+when: criteria1|criteria2|criteria3
+avoid: scenario1→alternative|scenario2→alternative
+
+nlp_examples[N]:
+  nlp query 1
+  nlp query 2
+  nlp query 3
+  [... use extracted nlp_examples - convert any command examples to NLP queries]
+
+filters_available: filter1|filter2|filter3  # if skill has filters
+
+savings: X% vs alternative
+
+troubleshoot_nlp[K]:
+  issue_description: "nlp query to solve it"
+
+CRITICAL RULES - NLP-ONLY APPROACH:
+1. NO "commands" section with Python syntax (agents use command_intelligence instead)
+2. NLP examples are natural language queries like "search for authentication code"
+3. Convert any command examples to NLP: "python scripts/..." → "search for X" or "index Y workspace"
+4. Troubleshooting solutions as NLP queries too
+5. Ultra-compact: when/avoid on ONE line each
+6. Filters just list available, no syntax details
+
+PHILOSOPHY: Agents describe what they want (NLP), command_intelligence generates syntax.
+
+Output ONLY the complete NLP-optimized TOON skill. No explanations."""
+
+        response = self._call_haiku(prompt, max_tokens=1500)
+
+        return response.strip()
+
+    def convert_skill(self, skill_path: Path) -> Dict[str, Any]:
+        """
+        2-PASS conversion approach.
+
+        PASS 1: Extract all structured data (Haiku sees complete skill)
+        PASS 2: Format TOON (Haiku fills template with extracted data)
+
+        Returns:
+            Conversion report with metrics
+        """
+        print(f"\n{'='*60}")
+        print(f" Converting: {skill_path.name}")
+        print(f"{'='*60}\n")
+
+        # Read original
+        original_content = skill_path.read_text()
+        original_lines = len(original_content.splitlines())
+        original_tokens = len(original_content) // 4
+
+        # Extract frontmatter
+        frontmatter_match = re.search(r'^---\n(.*?)\n---', original_content, re.DOTALL)
+        frontmatter = frontmatter_match.group(0) if frontmatter_match else ''
+
+        # PASS 1: Extract structured data
+        extracted_data = self.extract_all_data(original_content)
+
+        # PASS 2: Format TOON
+        final_skill = self.format_toon_skill(frontmatter, extracted_data)
+
+        final_lines = len(final_skill.splitlines())
+        final_tokens = len(final_skill) // 4
+
+        # Report
+        report = {
+            "skill_name": extracted_data.get("skill_name", skill_path.stem),
+            "original_lines": original_lines,
+            "final_lines": final_lines,
+            "line_reduction": f"{((original_lines - final_lines) / original_lines * 100):.1f}%",
+            "original_tokens": original_tokens,
+            "final_tokens": final_tokens,
+            "token_reduction": f"{((original_tokens - final_tokens) / original_tokens * 100):.1f}%",
+            "final_content": final_skill
+        }
+
+        print(f"\n{'='*60}")
+        print(f" Conversion Report")
+        print(f"{'='*60}")
+        print(f"Lines: {original_lines} → {final_lines} ({report['line_reduction']} reduction)")
+        print(f"Tokens: {original_tokens} → {final_tokens} ({report['token_reduction']} reduction)")
+
+        return report
+
+
+# ============================================================================
+# MAIN
+# ============================================================================
+
+def main():
+    import argparse
+
+    parser = argparse.ArgumentParser(description="Convert skills to TOON format using Haiku")
+    parser.add_argument("--skill", help="Skill name to convert")
+    parser.add_argument("--batch", choices=["high-usage", "medium-usage", "all"], help="Batch conversion")
+    parser.add_argument("--dry-run", action="store_true", help="Preview only, don't save")
+    parser.add_argument("--apply", action="store_true", help="Apply conversion and save")
+    parser.add_argument("--api-key", help="Anthropic API key (or use ANTHROPIC_API_KEY env)")
+
+    args = parser.parse_args()
+
+    # Get API key
+    api_key = args.api_key or os.getenv("ANTHROPIC_API_KEY")
+    if not api_key:
+        print("ERROR: Anthropic API key required")
+        print("Set ANTHROPIC_API_KEY env var or use --api-key")
+        sys.exit(1)
+
+    # Initialize converter
+    converter = SkillConverter(api_key=api_key, dry_run=args.dry_run)
+
+    # Determine skills to convert
+    if args.skill:
+        skills_to_convert = [args.skill]
+    elif args.batch == "high-usage":
+        skills_to_convert = [
+            "rag-manager",
+            "code-block-manager",
+            "markdown-manager",
+            "yaml-manager",
+            "context-memory-guide"
+        ]
+    elif args.batch == "medium-usage":
+        skills_to_convert = [
+            "json-manager",
+            "file-manager",
+            "git-repo-manager",
+            "bash-optimizer",
+            "llm-manager"
+        ]
+    elif args.batch == "all":
+        # Get all skills
+        skills_dir = Path(".opencode/skills")
+        skills_to_convert = [
+            d.name for d in skills_dir.iterdir()
+            if d.is_dir() and (d / "SKILL.md").exists() and d.name != "_templates"
+        ]
+    else:
+        print("ERROR: Specify --skill NAME or --batch TYPE")
+        sys.exit(1)
+
+    print(f"\n🎯 Converting {len(skills_to_convert)} skills")
+    print(f"Mode: {'DRY-RUN (preview only)' if args.dry_run else 'APPLY (will save changes)'}")
+    print("")
+
+    # Convert each skill
+    reports = []
+
+    for skill_name in skills_to_convert:
+        skill_path = Path(f".opencode/skills/{skill_name}/SKILL.md")
+
+        if not skill_path.exists():
+            print(f"warning:skill_not_found: {skill_path}")
+            continue
+
+        try:
+            # Backup original
+            backup_path = skill_path.with_suffix(".md.backup")
+            if args.apply and not args.dry_run:
+                skill_path.rename(backup_path)
+                print(f" Backup created: {backup_path}")
+
+            # Convert
+            report = converter.convert_skill(skill_path if args.dry_run else backup_path)
+            reports.append(report)
+
+            # Preview
+            if args.dry_run:
+                print("\n" + "="*60)
+                print("PREVIEW (first 30 lines):")
+                print("="*60)
+                preview_lines = report["final_content"].splitlines()[:30]
+                print("\n".join(preview_lines))
+                if len(report["final_content"].splitlines()) > 30:
+                    print(f"\n... ({len(report['final_content'].splitlines()) - 30} more lines)")
+                print("="*60)
+
+            # Save if applying
+            if args.apply and not args.dry_run:
+                skill_path.write_text(report["final_content"])
+                print(f"ok:saved: {skill_path}")
+
+        except Exception as e:
+            print(f"error:conversion_failed for {skill_name}: {e}")
+
+            # Restore backup if failed
+            if args.apply and not args.dry_run and backup_path.exists():
+                backup_path.rename(skill_path)
+                print(f" Restored from backup")
+
+    # Summary
+    if reports:
+        print("\n" + "="*60)
+        print(" CONVERSION SUMMARY")
+        print("="*60)
+
+        total_tokens_before = sum(r["original_tokens"] for r in reports)
+        total_tokens_after = sum(r["final_tokens"] for r in reports)
+        total_economy = ((total_tokens_before - total_tokens_after) / total_tokens_before * 100)
+
+        print(f"Skills converted: {len(reports)}")
+        print(f"Total tokens before: {total_tokens_before:,}")
+        print(f"Total tokens after: {total_tokens_after:,}")
+        print(f"Total economy: {total_economy:.1f}%")
+        print(f"Tokens saved: {total_tokens_before - total_tokens_after:,}")
+
+        if args.dry_run:
+            print("\ninfo:dry_run_mode:use_--apply_to_execute")
+        else:
+            print("\nok:conversion_applied:backups:*.md.backup")
+
+
+if __name__ == "__main__":
+    main()