@openhands/extensions 0.0.1-alpha → 0.2.0

package/scripts/sync_extensions.py

@@ -0,0 +1,494 @@
+#!/usr/bin/env python3
+"""Keep the OpenHands extensions registry in sync.
+
+Four sync tasks, runnable individually or all at once:
+
+  1. **commands**  — generate Claude Code ``commands/<trigger>.md`` files from
+     SKILL.md slash-triggers.
+  2. **catalog**   — regenerate the auto-generated catalog section in README.md.
+  3. **coverage**  — warn when a skill/plugin directory is not listed in any
+     marketplace, or a marketplace entry points to a missing directory.
+  4. **symlinks**  — enforce ``.plugin/`` as the canonical manifest directory
+     with vendor symlinks (``.claude-plugin``, ``.codex-plugin``).
+
+Usage:
+    python scripts/sync_extensions.py                   # run all, write changes
+    python scripts/sync_extensions.py --check           # CI mode — exit 1 if anything is out of sync
+    python scripts/sync_extensions.py commands           # only sync command files
+    python scripts/sync_extensions.py catalog            # only regenerate README catalog
+    python scripts/sync_extensions.py coverage           # only check marketplace coverage
+    python scripts/sync_extensions.py symlinks           # only check/fix vendor symlinks
+    python scripts/sync_extensions.py commands catalog   # combine sub-commands
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+
+import yaml  # requires: pip install pyyaml (also in pyproject.toml [test] group)
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+README_PATH = REPO_ROOT / "README.md"
+SKILL_DIRS = [REPO_ROOT / "skills", REPO_ROOT / "plugins"]
+MARKETPLACES_DIR = REPO_ROOT / "marketplaces"
+# Max description length in the catalog table.  120 chars fits GitHub's
+# diff viewer and rendered Markdown tables without horizontal scroll.
+MAX_DESC_LEN = 120
+
+# Sentinel markers in README.md
+CATALOG_BEGIN = "<!-- BEGIN AUTO-GENERATED CATALOG -->"
+CATALOG_END = "<!-- END AUTO-GENERATED CATALOG -->"
+
+# Marker for auto-generated Claude Code command files (YAML comment inside frontmatter)
+CMD_MARKER = "# auto-generated by sync_extensions.py"
+# Also recognize the legacy HTML-comment headers from older versions of the script
+_LEGACY_CMD_MARKERS = [
+    "<!-- AUTO-GENERATED by scripts/sync_extensions.py",
+    "<!-- AUTO-GENERATED by scripts/sync_claude_commands.py",
+]
+
+
+# ── Frontmatter helpers ──────────────────────────────────────────────
+
+def parse_frontmatter(text: str) -> dict[str, str | list[str]]:
+    """Extract name, description, and triggers from YAML frontmatter.
+
+    Uses PyYAML for robust parsing.  Returns an empty dict when the
+    frontmatter is missing or malformed.
+    """
+    m = re.match(r"^---\n(.*?)\n---", text, re.DOTALL)
+    if not m:
+        return {}
+    block = m.group(1)
+    try:
+        data = yaml.safe_load(block)
+    except yaml.YAMLError as exc:
+        print(f"[warning] malformed YAML frontmatter: {exc}", file=sys.stderr)
+        return {}
+    if not isinstance(data, dict):
+        return {}
+    result: dict[str, str | list[str]] = {}
+    for key in ("name", "description"):
+        if key in data and data[key] is not None:
+            result[key] = str(data[key]).strip()
+    triggers = data.get("triggers")
+    if isinstance(triggers, list):
+        result["triggers"] = [str(t).strip() for t in triggers if t is not None]
+    else:
+        result["triggers"] = []
+    return result
+
+
+def slash_triggers(meta: dict) -> list[str]:
+    return [t for t in meta.get("triggers", []) if isinstance(t, str) and t.startswith("/")]
+
+
+# ── Marketplace loading ──────────────────────────────────────────────
+
+def load_marketplaces() -> list[dict]:
+    """Return list of parsed marketplace dicts, sorted by filename."""
+    mps: list[dict] = []
+    for mp_file in sorted(MARKETPLACES_DIR.glob("*.json")):
+        try:
+            with open(mp_file) as f:
+                data = json.load(f)
+        except (json.JSONDecodeError, OSError) as exc:
+            print(f"[warning] skipping {mp_file.name}: {exc}", file=sys.stderr)
+            continue
+        data["_file"] = mp_file
+        mps.append(data)
+    return mps
+
+
+# ── 1. Claude Code command sync ──────────────────────────────────────
+
+def build_command_content(description: str) -> str:
+    lines = [
+        "---",
+        CMD_MARKER,
+        f"description: {description}",
+        "---",
+        "",
+        "Read and follow the complete instructions in the SKILL.md file located in this skill's directory.",
+        "",
+        "$ARGUMENTS",
+        "",
+    ]
+    return "\n".join(lines)
+
+
+@dataclass
+class CommandSpec:
+    """A Claude Code command file that should exist for a slash trigger."""
+    path: Path
+    trigger: str
+    description: str
+
+
+def collect_needed_commands() -> list[CommandSpec]:
+    """Return a CommandSpec for every slash trigger found in SKILL.md files."""
+    needed: list[CommandSpec] = []
+    for base in SKILL_DIRS:
+        if not base.is_dir():
+            continue
+        for skill_dir in sorted(base.iterdir()):
+            skill_md = skill_dir / "SKILL.md"
+            if not skill_md.is_file():
+                continue
+            meta = parse_frontmatter(skill_md.read_text())
+            desc = str(meta.get("description", ""))
+            for trigger in slash_triggers(meta):
+                cmd_name = trigger.lstrip("/")
+                cmd_path = skill_dir / "commands" / f"{cmd_name}.md"
+                needed.append(CommandSpec(path=cmd_path, trigger=trigger, description=desc))
+    return needed
+
+
+def sync_commands(*, check: bool) -> list[str]:
+    """Returns list of problem descriptions (empty = all good)."""
+    problems: list[str] = []
+    for spec in collect_needed_commands():
+        expected = build_command_content(spec.description)
+        cmd_path = spec.path
+        if cmd_path.is_file():
+            existing = cmd_path.read_text()
+            if existing == expected:
+                continue
+            is_auto = CMD_MARKER in existing or any(m in existing for m in _LEGACY_CMD_MARKERS)
+            if not is_auto:
+                rel = cmd_path.relative_to(REPO_ROOT)
+                msg = (
+                    f"{rel} exists but has no auto-generated marker "
+                    f"— it won't be updated.  Add the marker or delete "
+                    f"the file to let sync manage it."
+                )
+                if check:
+                    problems.append(f"manually-edited: {rel}")
+                else:
+                    print(f"[warning] {msg}", file=sys.stderr)
+                continue
+            problems.append(f"stale: {cmd_path.relative_to(REPO_ROOT)}")
+        else:
+            problems.append(f"missing: {cmd_path.relative_to(REPO_ROOT)}")
+
+        if not check:
+            cmd_path.parent.mkdir(parents=True, exist_ok=True)
+            cmd_path.write_text(expected)
+
+    return problems
+
+
+# ── 2. README catalog generation ─────────────────────────────────────
+
+def _entry_type(source: str) -> str:
+    """Classify a marketplace entry as ``"skill"`` or ``"plugin"``.
+
+    Checks the source path prefix first, then falls back to examining
+    the resolved directory contents (SKILL.md → skill, hooks/ or
+    scripts/ → plugin).  Returns ``"skill"`` as the default when the
+    type cannot be determined, since most entries are skills.
+    """
+    if source.startswith("./skills/") or source.startswith("../skills/"):
+        return "skill"
+    if source.startswith("./plugins/") or source.startswith("../plugins/"):
+        return "plugin"
+    resolved = REPO_ROOT / source.lstrip("./")
+    if resolved.is_dir():
+        if "skills" in resolved.parts:
+            return "skill"
+        if "plugins" in resolved.parts:
+            return "plugin"
+        # Infer from directory contents
+        if (resolved / "hooks").is_dir() or (resolved / "scripts").is_dir():
+            return "plugin"
+    return "skill"
+
+
+_slash_cmd_cache: dict[str, list[str]] = {}
+
+
+def _get_slash_commands(source: str) -> list[str]:
+    """Read SKILL.md from source path to extract slash triggers (cached)."""
+    if source in _slash_cmd_cache:
+        return _slash_cmd_cache[source]
+    skill_md = REPO_ROOT / source.lstrip("./") / "SKILL.md"
+    if not skill_md.is_file():
+        _slash_cmd_cache[source] = []
+        return []
+    meta = parse_frontmatter(skill_md.read_text())
+    result = slash_triggers(meta)
+    _slash_cmd_cache[source] = result
+    return result
+
+
+def _format_marketplace_section(mp: dict) -> list[str]:
+    """Format a single marketplace as Markdown lines."""
+    mp_name = mp.get("name", mp["_file"].stem)
+    mp_desc = mp.get("metadata", {}).get("description", "")
+    plugins = mp.get("plugins", [])
+    skill_count = sum(1 for p in plugins if _entry_type(p.get("source", "")) == "skill")
+    plugin_count = sum(1 for p in plugins if _entry_type(p.get("source", "")) == "plugin")
+
+    lines: list[str] = []
+    lines.append(f"### {mp_name}")
+    lines.append("")
+    if mp_desc:
+        lines.append(mp_desc)
+        lines.append("")
+    lines.append(f"**{len(plugins)} extensions** ({skill_count} skills, {plugin_count} plugins)")
+    lines.append("")
+    lines.append("| Name | Type | Description | Commands |")
+    lines.append("|------|------|-------------|----------|")
+    for p in sorted(plugins, key=lambda x: x["name"]):
+        name = p["name"]
+        etype = _entry_type(p.get("source", ""))
+        desc = (p.get("description") or "").replace("|", "\\|")
+        if len(desc) > MAX_DESC_LEN:
+            desc = desc[:MAX_DESC_LEN - 3] + "..."
+        cmds = _get_slash_commands(p.get("source", ""))
+        cmds_str = ", ".join(f"`{c}`" for c in cmds) if cmds else "—"
+        lines.append(f"| {name} | {etype} | {desc} | {cmds_str} |")
+
+    lines.append("")
+    return lines
+
+
+def generate_catalog() -> str:
+    """Build the Markdown catalog section."""
+    mps = load_marketplaces()
+    total_skills = 0
+    total_plugins = 0
+
+    for mp in mps:
+        for p in mp.get("plugins", []):
+            t = _entry_type(p.get("source", ""))
+            if t == "skill":
+                total_skills += 1
+            elif t == "plugin":
+                total_plugins += 1
+
+    lines: list[str] = [""]
+    total = sum(len(mp.get("plugins", [])) for mp in mps)
+    lines.append(
+        f"This repository contains **{len(mps)} marketplace(s)** "
+        f"with **{total} extensions** ({total_skills} skills, {total_plugins} plugins)."
+    )
+    lines.append("")
+
+    for mp in mps:
+        lines.extend(_format_marketplace_section(mp))
+    return "\n".join(lines)
+
+
+def sync_catalog(*, check: bool) -> list[str]:
+    """Returns list of problem descriptions (empty = all good)."""
+    if not README_PATH.is_file():
+        return ["README.md not found"]
+
+    readme = README_PATH.read_text()
+    if CATALOG_BEGIN not in readme or CATALOG_END not in readme:
+        return [
+            f"README.md missing catalog markers ({CATALOG_BEGIN} / {CATALOG_END}). "
+            "Add them where the catalog should appear."
+        ]
+
+    before = readme[: readme.index(CATALOG_BEGIN) + len(CATALOG_BEGIN)]
+    after = readme[readme.index(CATALOG_END) :]
+    existing_catalog = readme[len(before) : readme.index(CATALOG_END)]
+
+    new_catalog = generate_catalog()
+
+    if existing_catalog == new_catalog:
+        return []
+
+    if check:
+        return ["README.md catalog section is out of date"]
+
+    README_PATH.write_text(before + new_catalog + after)
+    return []
+
+
+# ── 3. Marketplace coverage ─────────────────────────────────────────
+
+def sync_coverage(*, check: bool) -> list[str]:
+    """Check that every directory is in a marketplace and vice versa."""
+    # Collect all dirs with SKILL.md or .plugin/plugin.json (or vendor symlinks like .claude-plugin)
+    all_dirs: set[str] = set()
+    for base in SKILL_DIRS:
+        if not base.is_dir():
+            continue
+        for d in base.iterdir():
+            if not d.is_dir() or d.name.startswith("."):
+                continue
+            has_skill = (d / "SKILL.md").exists()
+            has_plugin = (
+                (d / ".claude-plugin" / "plugin.json").exists()
+                or (d / ".plugin" / "plugin.json").exists()
+            )
+            if has_skill or has_plugin:
+                all_dirs.add(f"./{base.name}/{d.name}")
+
+    # Collect all marketplace sources
+    all_sources: set[str] = set()
+    for mp in load_marketplaces():
+        for p in mp.get("plugins", []):
+            all_sources.add(p.get("source", ""))
+
+    # Collect skill dirs that are symlink targets inside a plugin's skills/
+    # subdirectory — these are covered by the parent plugin entry.
+    symlink_targets: set[str] = set()
+    plugins_dir = REPO_ROOT / "plugins"
+    if plugins_dir.is_dir():
+        for plugin_dir in plugins_dir.iterdir():
+            skills_sub = plugin_dir / "skills"
+            if not skills_sub.is_dir():
+                continue
+            for entry in skills_sub.iterdir():
+                if entry.is_symlink():
+                    resolved = entry.resolve()
+                    try:
+                        rel = f"./{resolved.relative_to(REPO_ROOT)}"
+                        symlink_targets.add(rel)
+                    except ValueError:
+                        pass
+
+    problems: list[str] = []
+
+    missing_from_mp = sorted(all_dirs - all_sources - symlink_targets)
+    for d in missing_from_mp:
+        problems.append(f"not in any marketplace: {d}")
+
+    ghost_entries = sorted(all_sources - all_dirs)
+    for s in ghost_entries:
+        p = REPO_ROOT / s.lstrip("./")
+        if not p.is_dir():
+            problems.append(f"marketplace entry points to missing directory: {s}")
+        else:
+            problems.append(f"marketplace entry has no SKILL.md or plugin.json: {s}")
+
+    return problems
+
+
+# ── 4. Vendor symlinks ────────────────────────────────────────────────
+
+VENDOR_SYMLINKS = [".claude-plugin", ".codex-plugin"]  # add new vendors here
+
+
+def _check_vendor_symlinks(directory: Path, check: bool) -> list[str]:
+    """Check/fix vendor symlinks for a single directory with .plugin/."""
+    problems: list[str] = []
+    canon = directory / ".plugin"
+    if not canon.is_dir():
+        return problems
+    for vendor in VENDOR_SYMLINKS:
+        link = directory / vendor
+        if link.is_symlink():
+            target = link.resolve()
+            if target == canon.resolve():
+                continue
+            problems.append(f"wrong target: {link.relative_to(REPO_ROOT)} → {link.readlink()}")
+        elif link.exists():
+            problems.append(f"not a symlink: {link.relative_to(REPO_ROOT)}")
+            continue
+        else:
+            problems.append(f"missing: {link.relative_to(REPO_ROOT)}")
+        if not check:
+            link.unlink(missing_ok=True)
+            link.symlink_to(".plugin")
+    return problems
+
+
+def sync_symlinks(*, check: bool) -> list[str]:
+    """Ensure every directory with .plugin/ also has vendor symlinks.
+
+    Scans both plugins/ and skills/ directories.  Skills that ship a
+    ``.plugin/`` manifest (e.g. those with ``commands/``) need vendor
+    symlinks so that Codex and Claude Code can discover them.
+    """
+    problems: list[str] = []
+    for base in SKILL_DIRS:
+        if not base.is_dir():
+            continue
+        for entry_dir in sorted(base.iterdir()):
+            if not entry_dir.is_dir() or entry_dir.name.startswith("."):
+                continue
+            problems.extend(_check_vendor_symlinks(entry_dir, check))
+    return problems
+
+
+# ── Main ─────────────────────────────────────────────────────────────
+
+ALL_SYNCS = {
+    "commands": sync_commands,
+    "catalog": sync_catalog,
+    "coverage": sync_coverage,
+    "symlinks": sync_symlinks,
+}
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description="Keep the OpenHands extensions registry in sync.",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog=__doc__,
+    )
+    valid_tasks = list(ALL_SYNCS.keys())
+    parser.add_argument(
+        "tasks",
+        nargs="*",
+        default=[],
+        metavar="TASK",
+        help=f"Which sync tasks to run: {', '.join(valid_tasks)} (default: all).",
+    )
+    parser.add_argument(
+        "--check",
+        action="store_true",
+        help="Check mode: exit 1 if anything is out of sync (for CI).",
+    )
+    args = parser.parse_args()
+    for t in args.tasks:
+        if t not in ALL_SYNCS:
+            parser.error(f"unknown task: {t!r} (choose from {', '.join(valid_tasks)})")
+    tasks = args.tasks or list(ALL_SYNCS.keys())
+
+    all_problems: dict[str, list[str]] = {}
+    for task in tasks:
+        fn = ALL_SYNCS[task]
+        problems = fn(check=args.check)
+        if problems:
+            all_problems[task] = problems
+
+    if all_problems:
+        for task, problems in all_problems.items():
+            severity = "warning" if task == "coverage" else "error"
+            print(f"\n[{severity}] {task}:")
+            for p in problems:
+                print(f"  {p}")
+        if args.check:
+            has_errors = any(k != "coverage" for k in all_problems)
+            has_warnings = "coverage" in all_problems
+            if has_errors:
+                print(f"\nRun `python scripts/sync_extensions.py` to fix.")
+                return 1
+            if has_warnings:
+                # Coverage warnings don't fail CI, just warn
+                print(f"\n⚠️  Coverage warnings above are non-blocking.")
+                return 0
+        else:
+            print("\nSync complete.")
+        return 0
+
+    if args.check:
+        print("All extensions in sync. ✓")
+    else:
+        print("Everything already up to date.")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())