@openhands/extensions 0.1.0 → 0.3.0

package/tests/test_sdk_loading.py

@@ -0,0 +1,150 @@
+"""Test that marketplaces, skills, and plugins can be loaded by the SDK."""
+
+from pathlib import Path
+
+import pytest
+from openhands.sdk.skills import Skill
+from openhands.sdk.marketplace import Marketplace, MarketplacePluginEntry
+
+
+def get_repo_root():
+    """Get the path to the repository root."""
+    return Path(__file__).parent.parent
+
+
+def get_all_skill_directories():
+    """Get all skill directory names that have a SKILL.md file."""
+    skills_dir = get_repo_root() / "skills"
+    return [
+        d for d in skills_dir.iterdir()
+        if d.is_dir() and not d.name.startswith('.') and (d / "SKILL.md").exists()
+    ]
+
+
+class TestDefaultMarketplace:
+    """Test that the default marketplace can be loaded by the SDK."""
+
+    def test_marketplace_loads_with_sdk(self):
+        """Verify the default marketplace can be loaded using SDK's Marketplace model."""
+        marketplace_path = get_repo_root() / "marketplaces" / "openhands-extensions.json"
+        
+        # Load using SDK's pydantic model
+        import json
+        with open(marketplace_path) as f:
+            data = json.load(f)
+        
+        marketplace = Marketplace.model_validate({**data, "path": str(get_repo_root())})
+        
+        assert marketplace.name == "openhands-extensions"
+        assert marketplace.owner is not None
+        assert marketplace.owner.name == "OpenHands"
+        assert len(marketplace.plugins) > 0
+
+    def test_all_plugin_entries_valid(self):
+        """Verify all plugin entries can be validated as MarketplacePluginEntry."""
+        marketplace_path = get_repo_root() / "marketplaces" / "openhands-extensions.json"
+        
+        import json
+        with open(marketplace_path) as f:
+            data = json.load(f)
+        
+        errors = []
+        for plugin_data in data.get("plugins", []):
+            try:
+                entry = MarketplacePluginEntry.model_validate(plugin_data)
+                assert entry.name, f"Plugin missing name"
+                assert entry.source, f"Plugin {entry.name} missing source"
+                assert entry.description, f"Plugin {entry.name} missing description"
+            except Exception as e:
+                errors.append(f"{plugin_data.get('name', 'unknown')}: {e}")
+        
+        assert len(errors) == 0, f"Plugin validation errors:\n" + "\n".join(errors)
+
+    def test_marketplace_source_paths_exist(self):
+        """Verify all source paths in the marketplace resolve to real directories."""
+        import json
+
+        marketplace_path = get_repo_root() / "marketplaces" / "openhands-extensions.json"
+        with open(marketplace_path) as f:
+            data = json.load(f)
+
+        root = get_repo_root()
+        missing = []
+        for entry in data["plugins"]:
+            src = entry["source"]
+            resolved = root / src
+            if not resolved.exists():
+                missing.append(f"{entry['name']}: {src} -> {resolved}")
+        assert len(missing) == 0, (
+            f"Source paths that don't exist on disk:\n" + "\n".join(missing)
+        )
+
+    def test_marketplace_includes_all_skills(self):
+        """Verify every skill directory is referenced in at least one marketplace."""
+        import json
+
+        marketplaces_dir = get_repo_root() / "marketplaces"
+        marketplace_files = sorted(marketplaces_dir.glob("*.json"))
+        assert len(marketplace_files) > 0, "No marketplace JSON files found"
+
+        # Collect sources across all marketplaces
+        all_sources = set()
+        for mp_path in marketplace_files:
+            with open(mp_path) as f:
+                data = json.load(f)
+            marketplace = Marketplace.model_validate({**data, "path": str(get_repo_root())})
+            for plugin in marketplace.plugins:
+                source = plugin.source if isinstance(plugin.source, str) else ""
+                # Normalize: strip leading "./" and resolve parent refs to get the leaf name
+                name = Path(source).name
+                all_sources.add(name)
+
+        # Get all skill directories
+        skill_dirs = {d.name for d in get_all_skill_directories()}
+
+        missing = skill_dirs - all_sources
+        assert len(missing) == 0, f"Skills not in any marketplace: {sorted(missing)}"
+
+
+class TestSkillsLoadWithSDK:
+    """Test that all skills can be loaded by the SDK."""
+
+    def test_all_skills_load_with_sdk(self):
+        """Verify all SKILL.md files can be loaded using SDK's Skill.load()."""
+        skill_dirs = get_all_skill_directories()
+        
+        errors = []
+        for skill_dir in skill_dirs:
+            skill_path = skill_dir / "SKILL.md"
+            try:
+                skill = Skill.load(skill_path, skill_base_dir=skill_dir)
+                assert skill.name, f"Skill {skill_dir.name} has no name"
+                assert skill.content, f"Skill {skill_dir.name} has no content"
+            except Exception as e:
+                errors.append(f"{skill_dir.name}: {e}")
+        
+        assert len(errors) == 0, (
+            f"Failed to load {len(errors)} skills:\n" + "\n".join(errors)
+        )
+
+    def test_skills_have_valid_metadata(self):
+        """Verify all skills have valid metadata (name, description)."""
+        skill_dirs = get_all_skill_directories()
+        
+        missing_metadata = []
+        for skill_dir in skill_dirs:
+            skill_path = skill_dir / "SKILL.md"
+            try:
+                skill = Skill.load(skill_path, skill_base_dir=skill_dir)
+                if not skill.description:
+                    missing_metadata.append(f"{skill_dir.name}: missing description")
+            except Exception:
+                pass  # Already caught in test_all_skills_load_with_sdk
+        
+        assert len(missing_metadata) == 0, (
+            f"Skills with missing metadata:\n" + "\n".join(missing_metadata)
+        )
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])

package/tests/test_skill_plugin_loading.py

@@ -0,0 +1,149 @@
+"""Test that skills listed in marketplaces can be loaded as Codex/Claude plugins.
+
+Every marketplace entry that references a ``skills/`` directory needs a
+``.plugin/plugin.json`` manifest and vendor symlinks (``.codex-plugin``,
+``.claude-plugin``) so that Codex and Claude Code can discover and load them.
+
+Regression test for: https://github.com/OpenHands/extensions/issues/201
+"""
+
+import json
+from pathlib import Path
+
+import pytest
+
+
+REPO_ROOT = Path(__file__).parent.parent
+SKILLS_DIR = REPO_ROOT / "skills"
+PLUGINS_DIR = REPO_ROOT / "plugins"
+MARKETPLACES_DIR = REPO_ROOT / "marketplaces"
+
+REQUIRED_MANIFEST_FIELDS = ["name", "description", "author", "version"]
+VENDOR_SYMLINKS = [".claude-plugin", ".codex-plugin"]
+
+
+def _all_dirs_with_plugin_manifest():
+    """Yield directories (plugins/ and skills/) that have .plugin/plugin.json."""
+    for base in [PLUGINS_DIR, SKILLS_DIR]:
+        if not base.is_dir():
+            continue
+        for entry_dir in sorted(base.iterdir()):
+            if not entry_dir.is_dir() or entry_dir.name.startswith("."):
+                continue
+            manifest = entry_dir / ".plugin" / "plugin.json"
+            if manifest.exists():
+                yield entry_dir
+
+
+def _marketplace_skill_entries():
+    """Yield (skill_name, source_path) for every marketplace entry under skills/."""
+    for mp_file in sorted(MARKETPLACES_DIR.glob("*.json")):
+        with open(mp_file) as f:
+            data = json.load(f)
+        for p in data.get("plugins", []):
+            src = p["source"]
+            if src.startswith("./skills/"):
+                yield p["name"], REPO_ROOT / src.lstrip("./")
+
+
+class TestAllMarketplaceSkillsHaveManifests:
+    """Every skill listed in a marketplace must have a plugin manifest for Codex."""
+
+    def test_all_marketplace_skills_have_plugin_json(self):
+        """Every marketplace skill entry must have .plugin/plugin.json."""
+        missing = []
+        for name, path in _marketplace_skill_entries():
+            manifest = path / ".plugin" / "plugin.json"
+            if not manifest.exists():
+                missing.append(name)
+        assert not missing, (
+            f"Skills missing .plugin/plugin.json (Codex can't load them): "
+            f"{', '.join(missing)}"
+        )
+
+    def test_all_marketplace_skills_have_vendor_symlinks(self):
+        """Every marketplace skill with a manifest must have vendor symlinks."""
+        problems = []
+        for name, path in _marketplace_skill_entries():
+            if not (path / ".plugin" / "plugin.json").exists():
+                continue
+            for vendor in VENDOR_SYMLINKS:
+                link = path / vendor
+                if not link.is_symlink():
+                    problems.append(f"{name}/{vendor}")
+        assert not problems, (
+            f"Missing vendor symlinks: {', '.join(problems)}"
+        )
+
+    def test_all_manifests_have_required_fields(self):
+        """Every skill manifest must have name, description, author, version."""
+        problems = []
+        for name, path in _marketplace_skill_entries():
+            manifest = path / ".plugin" / "plugin.json"
+            if not manifest.exists():
+                continue
+            data = json.loads(manifest.read_text())
+            for field in REQUIRED_MANIFEST_FIELDS:
+                if field not in data:
+                    problems.append(f"{name}: missing '{field}'")
+            author = data.get("author")
+            if not isinstance(author, dict):
+                problems.append(f"{name}: 'author' must be an object")
+            elif "name" not in author:
+                problems.append(f"{name}: author missing 'name'")
+        assert not problems, (
+            f"Manifest validation errors:\n" + "\n".join(problems)
+        )
+
+    def test_manifest_name_matches_directory(self):
+        """The 'name' in plugin.json must match the directory name."""
+        problems = []
+        for name, path in _marketplace_skill_entries():
+            manifest = path / ".plugin" / "plugin.json"
+            if not manifest.exists():
+                continue
+            data = json.loads(manifest.read_text())
+            if data.get("name") != path.name:
+                problems.append(f"{name}: manifest name '{data.get('name')}' != dir '{path.name}'")
+        assert not problems, (
+            f"Name mismatches:\n" + "\n".join(problems)
+        )
+
+
+class TestIteratePluginLoading:
+    """Verify the iterate skill can be loaded as a plugin (issue #201)."""
+
+    def test_iterate_loads_as_sdk_plugin(self):
+        """The iterate skill must load via Plugin.load() without error."""
+        from openhands.sdk.plugin.plugin import Plugin
+
+        plugin = Plugin.load(SKILLS_DIR / "iterate")
+        assert plugin.name == "iterate"
+        assert plugin.version == "1.0.0"
+        assert len(plugin.commands) > 0
+        command_names = {c.name for c in plugin.commands}
+        assert "iterate" in command_names
+        assert "babysit" in command_names
+        assert "verify" in command_names
+
+
+class TestVendorSymlinksForManifests:
+    """Every directory with .plugin/ must have vendor symlinks."""
+
+    @pytest.fixture(
+        params=list(_all_dirs_with_plugin_manifest()),
+        ids=lambda d: f"{d.parent.name}/{d.name}",
+    )
+    def dir_with_manifest(self, request):
+        return request.param
+
+    def test_has_vendor_symlinks(self, dir_with_manifest):
+        """Directories with .plugin/ must have .claude-plugin and .codex-plugin symlinks."""
+        for vendor in VENDOR_SYMLINKS:
+            link = dir_with_manifest / vendor
+            assert link.is_symlink(), (
+                f"{link.relative_to(REPO_ROOT)} must be a symlink to .plugin"
+            )
+            assert link.resolve() == (dir_with_manifest / ".plugin").resolve(), (
+                f"{link.relative_to(REPO_ROOT)} must point to .plugin"
+            )

package/tests/test_skills_catalog.py

@@ -0,0 +1,336 @@
+"""Tests for the skills catalog codegen (scripts/build-skills-catalog.mjs).
+
+Covers:
+- parseFrontmatter edge cases (hyphenated keys, multiline, list items, etc.)
+- End-to-end buildCatalog against temp fixtures
+- Generated skills/index.js validity and structure
+- Determinism: re-running the script produces identical output
+"""
+
+import json
+import subprocess
+import textwrap
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+SCRIPT = ROOT / "scripts" / "build-skills-catalog.mjs"
+SKILLS_INDEX = ROOT / "skills" / "index.js"
+
+
+def run_node(script: str, *, cwd: str | Path = ROOT, check: bool = True) -> subprocess.CompletedProcess:
+    return subprocess.run(
+        ["node", "--input-type=module", "-e", script],
+        cwd=str(cwd),
+        capture_output=True,
+        text=True,
+        check=check,
+    )
+
+
+# ---------------------------------------------------------------------------
+# parseFrontmatter unit tests (via Node subprocess)
+# ---------------------------------------------------------------------------
+
+class TestParseFrontmatter:
+    """Unit tests for parseFrontmatter, exercised via Node import."""
+
+    def _parse(self, frontmatter_body: str) -> dict:
+        """Run parseFrontmatter on *frontmatter_body* and return the result."""
+        escaped = json.dumps(frontmatter_body)
+        script = textwrap.dedent(f"""\
+            import {{ parseFrontmatter }} from './scripts/build-skills-catalog.mjs';
+            const result = parseFrontmatter({escaped});
+            process.stdout.write(JSON.stringify(result));
+        """)
+        result = run_node(script)
+        return json.loads(result.stdout)
+
+    def test_simple_key_value(self):
+        fm = self._parse("name: my-skill\ndescription: A test skill")
+        assert fm["name"] == "my-skill"
+        assert fm["description"] == "A test skill"
+
+    def test_hyphenated_keys(self):
+        fm = self._parse("name: my-skill\ncompatibility: works-everywhere")
+        assert fm["compatibility"] == "works-everywhere"
+
+    def test_hyphenated_key_not_silently_dropped(self):
+        """Keys like event-key: should be parsed, not ignored."""
+        # We verify indirectly: if a hyphenated key precedes a list,
+        # the list items must be collected under that key.
+        script = textwrap.dedent("""\
+            import { parseFrontmatter } from './scripts/build-skills-catalog.mjs';
+            // parseFrontmatter returns a fixed shape, so test the internal
+            // parsing by checking a known hyphenated output key.
+            const fm = parseFrontmatter("license: MIT");
+            // Also verify a raw parse captures hyphenated keys internally
+            // by re-implementing a quick check:
+            const raw = "event-key: hello";
+            const match = raw.match(/^[\\w-]+:\\s*(.*)/);
+            if (!match) { console.error("Regex failed for hyphenated key"); process.exit(1); }
+            process.stdout.write("ok");
+        """)
+        result = run_node(script)
+        assert result.stdout == "ok"
+
+    def test_list_items(self):
+        fm = self._parse("name: test\ntriggers:\n- github\n- git\n- pull request")
+        assert fm["triggers"] == ["github", "git", "pull request"]
+
+    def test_empty_triggers_when_scalar(self):
+        fm = self._parse("name: test\ntriggers: not-a-list")
+        assert fm["triggers"] == []
+
+    def test_multiline_description_folded(self):
+        fm = self._parse("name: test\ndescription: >\n  This is a long\n  description text")
+        assert "This is a long" in fm["description"]
+        assert "description text" in fm["description"]
+
+    def test_multiline_description_literal(self):
+        fm = self._parse("name: test\ndescription: |\n  Line one\n  Line two")
+        assert "Line one" in fm["description"]
+        assert "Line two" in fm["description"]
+
+    def test_missing_name_returns_empty(self):
+        fm = self._parse("description: no name here")
+        assert fm["name"] == ""
+
+    def test_missing_description_returns_empty(self):
+        fm = self._parse("name: test")
+        assert fm["description"] == ""
+        assert fm["triggers"] == []
+
+    def test_optional_fields_omitted_when_absent(self):
+        fm = self._parse("name: test\ndescription: desc")
+        assert "license" not in fm
+        assert "compatibility" not in fm
+
+    def test_optional_fields_present_when_set(self):
+        fm = self._parse("name: test\nlicense: MIT\ncompatibility: all platforms")
+        assert fm["license"] == "MIT"
+        assert fm["compatibility"] == "all platforms"
+
+    def test_empty_input(self):
+        fm = self._parse("")
+        assert fm["name"] == ""
+        assert fm["description"] == ""
+        assert fm["triggers"] == []
+
+
+# ---------------------------------------------------------------------------
+# End-to-end codegen tests with temp fixtures (using buildCatalog)
+# ---------------------------------------------------------------------------
+
+class TestCodegenEndToEnd:
+    """Run buildCatalog against temp SKILL.md fixtures."""
+
+    def _run_codegen(self, tmp_path: Path, skills: dict[str, str]) -> list[dict]:
+        """Create temp skill dirs, run buildCatalog, return parsed catalog."""
+        skills_dir = tmp_path / "skills"
+        skills_dir.mkdir()
+
+        for name, content in skills.items():
+            skill_dir = skills_dir / name
+            skill_dir.mkdir()
+            (skill_dir / "SKILL.md").write_text(content)
+
+        script = textwrap.dedent(f"""\
+            import {{ buildCatalog }} from './scripts/build-skills-catalog.mjs';
+            const entries = buildCatalog({json.dumps(str(skills_dir))});
+            process.stdout.write(JSON.stringify(entries));
+        """)
+        result = run_node(script)
+        return json.loads(result.stdout)
+
+    def test_basic_skill(self, tmp_path):
+        entries = self._run_codegen(tmp_path, {
+            "my-skill": "---\nname: my-skill\ndescription: A skill\ntriggers:\n- test\n---\nBody content here"
+        })
+        assert len(entries) == 1
+        assert entries[0]["name"] == "my-skill"
+        assert entries[0]["description"] == "A skill"
+        assert entries[0]["triggers"] == ["test"]
+        assert entries[0]["content"] == "Body content here"
+
+    def test_name_falls_back_to_directory(self, tmp_path):
+        entries = self._run_codegen(tmp_path, {
+            "fallback-dir": "---\ndescription: no name field\n---\nBody"
+        })
+        assert entries[0]["name"] == "fallback-dir"
+
+    def test_missing_frontmatter_skipped(self, tmp_path):
+        entries = self._run_codegen(tmp_path, {
+            "good-skill": "---\nname: good\ndescription: yes\n---\nBody",
+            "bad-skill": "No frontmatter delimiters at all",
+        })
+        assert len(entries) == 1
+        assert entries[0]["name"] == "good"
+
+    def test_missing_frontmatter_warns(self, tmp_path):
+        """SKILL.md without frontmatter should produce a stderr warning."""
+        skills_dir = tmp_path / "skills"
+        skills_dir.mkdir()
+        (skills_dir / "bad").mkdir()
+        (skills_dir / "bad" / "SKILL.md").write_text("No frontmatter here")
+
+        script = textwrap.dedent(f"""\
+            import {{ buildCatalog }} from './scripts/build-skills-catalog.mjs';
+            buildCatalog({json.dumps(str(skills_dir))});
+        """)
+        result = run_node(script)
+        assert "Warning" in result.stderr
+        assert "missing frontmatter" in result.stderr
+
+    def test_body_with_triple_dashes_preserved(self, tmp_path):
+        content = "---\nname: test\ndescription: d\n---\nBefore\n---\nAfter"
+        entries = self._run_codegen(tmp_path, {"test-skill": content})
+        assert "---" in entries[0]["content"]
+        assert "Before" in entries[0]["content"]
+        assert "After" in entries[0]["content"]
+
+    def test_generated_file_is_valid_js(self, tmp_path):
+        """Write a catalog to a file and verify it's importable as JS."""
+        skills_dir = tmp_path / "skills"
+        skills_dir.mkdir()
+        for name in ("alpha", "beta"):
+            (skills_dir / name).mkdir()
+            (skills_dir / name / "SKILL.md").write_text(
+                f"---\nname: {name}\ndescription: {name} desc\n---\nBody {name}"
+            )
+
+        output = tmp_path / "output.js"
+        script = textwrap.dedent(f"""\
+            import {{ writeFileSync }} from "node:fs";
+            import {{ buildCatalog }} from './scripts/build-skills-catalog.mjs';
+            const entries = buildCatalog({json.dumps(str(skills_dir))});
+            const src = "export const SKILLS_CATALOG = " + JSON.stringify(entries) + ";\\nexport default SKILLS_CATALOG;\\n";
+            writeFileSync({json.dumps(str(output))}, src);
+        """)
+        run_node(script)
+
+        verify = textwrap.dedent(f"""\
+            const mod = await import({json.dumps(str(output))});
+            if (!Array.isArray(mod.SKILLS_CATALOG)) process.exit(1);
+            if (mod.SKILLS_CATALOG.length !== 2) process.exit(1);
+            if (mod.default !== mod.SKILLS_CATALOG) process.exit(1);
+        """)
+        run_node(verify)
+
+    def test_entries_sorted_by_directory_name(self, tmp_path):
+        entries = self._run_codegen(tmp_path, {
+            "zebra": "---\nname: zebra\ndescription: z\n---\nZ",
+            "alpha": "---\nname: alpha\ndescription: a\n---\nA",
+            "middle": "---\nname: middle\ndescription: m\n---\nM",
+        })
+        names = [e["name"] for e in entries]
+        assert names == ["alpha", "middle", "zebra"]
+
+    def test_optional_fields_included_when_present(self, tmp_path):
+        entries = self._run_codegen(tmp_path, {
+            "lic": "---\nname: lic\ndescription: d\nlicense: MIT\ncompatibility: all\n---\nBody"
+        })
+        assert entries[0]["license"] == "MIT"
+        assert entries[0]["compatibility"] == "all"
+
+    def test_optional_fields_omitted_when_absent(self, tmp_path):
+        entries = self._run_codegen(tmp_path, {
+            "bare": "---\nname: bare\ndescription: d\n---\nBody"
+        })
+        assert "license" not in entries[0]
+        assert "compatibility" not in entries[0]
+
+    def test_directories_without_skill_md_ignored(self, tmp_path):
+        skills_dir = tmp_path / "skills"
+        skills_dir.mkdir()
+        (skills_dir / "has-skill").mkdir()
+        (skills_dir / "has-skill" / "SKILL.md").write_text("---\nname: ok\ndescription: d\n---\nBody")
+        (skills_dir / "no-skill").mkdir()  # no SKILL.md
+
+        script = textwrap.dedent(f"""\
+            import {{ buildCatalog }} from './scripts/build-skills-catalog.mjs';
+            const entries = buildCatalog({json.dumps(str(skills_dir))});
+            process.stdout.write(JSON.stringify(entries));
+        """)
+        result = run_node(script)
+        entries = json.loads(result.stdout)
+        assert len(entries) == 1
+        assert entries[0]["name"] == "ok"
+
+
+# ---------------------------------------------------------------------------
+# Generated skills/index.js validation (real repo data)
+# ---------------------------------------------------------------------------
+
+class TestGeneratedSkillsIndex:
+    """Validate the checked-in skills/index.js against the live SKILL.md files."""
+
+    def test_index_js_is_valid_and_exports_array(self):
+        """The generated skills/index.js is valid JS with a non-empty array export."""
+        script = textwrap.dedent("""\
+            import { SKILLS_CATALOG } from './skills/index.js';
+            if (!Array.isArray(SKILLS_CATALOG)) process.exit(1);
+            if (SKILLS_CATALOG.length === 0) process.exit(1);
+            import SKILLS from './skills/index.js';
+            if (SKILLS !== SKILLS_CATALOG) process.exit(1);
+        """)
+        run_node(script)
+
+    def test_every_entry_has_required_fields(self):
+        """Each catalog entry must have name, description, triggers, and content."""
+        script = textwrap.dedent("""\
+            import { SKILLS_CATALOG } from './skills/index.js';
+            for (const entry of SKILLS_CATALOG) {
+              if (typeof entry.name !== 'string' || !entry.name) {
+                console.error('Missing name:', JSON.stringify(entry).slice(0, 100));
+                process.exit(1);
+              }
+              if (typeof entry.description !== 'string') {
+                console.error('Missing description for:', entry.name);
+                process.exit(1);
+              }
+              if (!Array.isArray(entry.triggers)) {
+                console.error('Missing triggers for:', entry.name);
+                process.exit(1);
+              }
+              if (typeof entry.content !== 'string') {
+                console.error('Missing content for:', entry.name);
+                process.exit(1);
+              }
+            }
+        """)
+        run_node(script)
+
+    def test_no_duplicate_names(self):
+        """Skill names should be unique in the catalog."""
+        script = textwrap.dedent("""\
+            import { SKILLS_CATALOG } from './skills/index.js';
+            const names = SKILLS_CATALOG.map(e => e.name);
+            const dupes = names.filter((n, i) => names.indexOf(n) !== i);
+            if (dupes.length > 0) {
+              console.error('Duplicate names:', dupes);
+              process.exit(1);
+            }
+        """)
+        run_node(script)
+
+    def test_catalog_is_sorted(self):
+        """Entries should be sorted alphabetically by name."""
+        script = textwrap.dedent("""\
+            import { SKILLS_CATALOG } from './skills/index.js';
+            const names = SKILLS_CATALOG.map(e => e.name);
+            const sorted = [...names].sort();
+            for (let i = 0; i < names.length; i++) {
+              if (names[i] !== sorted[i]) {
+                console.error('Not sorted: ' + names[i] + ' should be ' + sorted[i]);
+                process.exit(1);
+              }
+            }
+        """)
+        run_node(script)
+
+    def test_index_is_up_to_date(self):
+        """Re-running the codegen script should produce identical output."""
+        before = SKILLS_INDEX.read_text()
+        subprocess.run(["node", str(SCRIPT)], cwd=str(ROOT), check=True, capture_output=True)
+        after = SKILLS_INDEX.read_text()
+        assert before == after, "skills/index.js is out of date — run: node scripts/build-skills-catalog.mjs"