@openhands/extensions 0.1.0 → 0.3.0

package/tests/test_catalogs.py

@@ -0,0 +1,109 @@
+import json
+import re
+import subprocess
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def load_catalog_entries(relative_path: str):
+    entries = []
+    for entry_path in sorted((ROOT / relative_path).glob("*.json")):
+        entry = json.loads(entry_path.read_text())
+        assert entry["id"] == entry_path.stem
+        entries.append(entry)
+    return entries
+
+
+def test_catalog_ids_are_unique_and_automations_reference_existing_integrations():
+    integrations = load_catalog_entries("integrations/catalog")
+    automations = load_catalog_entries("automations/catalog")
+
+    integration_ids = [entry["id"] for entry in integrations]
+    automation_ids = [entry["id"] for entry in automations]
+
+    assert len(integration_ids) == len(set(integration_ids))
+    assert len(automation_ids) == len(set(automation_ids))
+
+    known_integration_ids = set(integration_ids)
+    for automation in automations:
+        assert automation["requiredIntegrationIds"]
+        missing_ids = (
+            set(automation["requiredIntegrationIds"]) - known_integration_ids
+        )
+        assert missing_ids == set()
+
+
+def test_catalog_entries_have_required_fields():
+    for entry in load_catalog_entries("integrations/catalog"):
+        assert entry["id"]
+        assert entry["name"]
+        assert entry["description"]
+        assert entry["kind"] in {"mcp", "http"}
+        assert entry["iconBg"]
+        assert entry["connectionOptions"]
+        assert entry["defaultConnectionOptionId"]
+        for option in entry["connectionOptions"]:
+            assert option["id"]
+            assert option["provider"] in {"mcp", "http"}
+            assert option["auth"]["strategy"] in {
+                "none",
+                "api_key",
+                "bearer",
+                "basic",
+                "oauth2",
+            }
+            if option["provider"] == "mcp":
+                assert option["transport"]["kind"] in {"stdio", "shttp", "sse"}
+                if option["transport"]["kind"] == "stdio":
+                    assert option["transport"]["serverName"]
+                    assert option["transport"]["command"]
+                    assert isinstance(option["transport"]["args"], list)
+                else:
+                    assert option["transport"]["url"].startswith("https://")
+
+    for entry in load_catalog_entries("automations/catalog"):
+        assert entry["id"]
+        assert entry["name"]
+        assert entry["prompt"]
+        assert entry["exampleImplementation"]
+        assert isinstance(entry["popularityRank"], int)
+        assert isinstance(entry["estimatedSetupMinutes"], int)
+
+
+def test_credential_fields_have_helper_text_and_link():
+    """All password fields must have helperText plus a link (either a helperLink field or a
+    markdown link embedded in helperText) so users know how to get credentials."""
+    markdown_link_re = re.compile(r"\[.+?\]\(https://[^)]+\)")
+
+    for entry in load_catalog_entries("integrations/catalog"):
+        for option in entry["connectionOptions"]:
+            transport = option.get("transport", {})
+            for field_group in ("envFields", "argFields"):
+                for field in transport.get(field_group, []):
+                    if field.get("type") == "password":
+                        field_key = field.get("key", "<unknown>")
+                        assert "helperText" in field, (
+                            f"{entry['id']}: password field '{field_key}' is missing helperText"
+                        )
+                        assert field["helperText"], (
+                            f"{entry['id']}: password field '{field_key}' has empty helperText"
+                        )
+                        has_helper_link = field.get("helperLink", "").startswith("https://")
+                        has_inline_link = bool(markdown_link_re.search(field["helperText"]))
+                        assert has_helper_link or has_inline_link, (
+                            f"{entry['id']}: password field '{field_key}' must have a helperLink "
+                            f"or a markdown link in helperText"
+                        )
+
+
+def test_node_package_exports_catalogs():
+    script = """
+      import { INTEGRATION_CATALOG, AUTOMATION_CATALOG } from './index.js';
+      if (!Array.isArray(INTEGRATION_CATALOG) || INTEGRATION_CATALOG.length === 0) process.exit(1);
+      if (!Array.isArray(AUTOMATION_CATALOG) || AUTOMATION_CATALOG.length === 0) process.exit(1);
+      if (!INTEGRATION_CATALOG.some((entry) => entry.id === 'github')) process.exit(1);
+      if (!AUTOMATION_CATALOG.some((entry) => entry.id === 'github-pr-reviewer')) process.exit(1);
+    """
+    subprocess.run(["node", "--input-type=module", "-e", script], cwd=ROOT, check=True)

package/tests/test_code_review_risk_evaluation.py

@@ -0,0 +1,94 @@
+"""Test that risk/safety evaluation is integrated into the unified code review skill."""
+
+from pathlib import Path
+
+
+def get_repo_root():
+    return Path(__file__).parent.parent
+
+
+def get_code_review_skill():
+    return (get_repo_root() / "skills" / "code-review" / "SKILL.md").read_text()
+
+
+def get_risk_evaluation_reference():
+    return (
+        get_repo_root()
+        / "skills"
+        / "code-review"
+        / "references"
+        / "risk-evaluation.md"
+    ).read_text()
+
+
+class TestRiskEvaluationReference:
+    """Verify the risk-evaluation.md reference file is complete."""
+
+    def test_reference_file_exists(self):
+        path = (
+            get_repo_root()
+            / "skills"
+            / "code-review"
+            / "references"
+            / "risk-evaluation.md"
+        )
+        assert path.exists()
+
+    def test_has_all_risk_levels(self):
+        content = get_risk_evaluation_reference()
+        assert "Low Risk" in content
+        assert "Medium Risk" in content
+        assert "High Risk" in content
+
+    def test_has_risk_factors(self):
+        content = get_risk_evaluation_reference()
+        factors = [
+            "Pattern conformance",
+            "Security sensitivity",
+            "Infrastructure dependencies",
+            "Blast radius",
+            "Core system impact",
+        ]
+        for factor in factors:
+            assert factor in content, f"Missing risk factor: {factor}"
+
+    def test_has_high_risk_escalation_guidance(self):
+        content = get_risk_evaluation_reference()
+        assert "not auto-merg" in content.lower()
+        assert "human" in content.lower()
+
+    def test_has_repo_specific_risk_rules(self):
+        content = get_risk_evaluation_reference()
+        assert "AGENTS.md" in content
+        assert "repo-specific" in content.lower() or "Repo-specific" in content
+
+    def test_has_risk_assessment_output_format(self):
+        content = get_risk_evaluation_reference()
+        assert "Risk Assessment" in content
+
+
+class TestCodeReviewSkillReferencesRisk:
+    """Verify the unified code-review skill references the risk evaluation."""
+
+    def test_has_risk_evaluation_scenario(self):
+        content = get_code_review_skill()
+        assert "Risk and Safety Evaluation" in content
+
+    def test_references_risk_evaluation_file(self):
+        content = get_code_review_skill()
+        assert "references/risk-evaluation.md" in content
+
+    def test_risk_section_appears_after_dependency_section(self):
+        content = get_code_review_skill()
+        dep_pos = content.index("8. **Dependency Changes**")
+        risk_pos = content.index("9. **Risk and Safety Evaluation**")
+        assert risk_pos > dep_pos
+
+    def test_always_include_risk_instruction(self):
+        content = get_code_review_skill()
+        assert "Always include the" in content
+        assert "Risk and Safety Evaluation" in content
+
+    def test_has_risk_assessment_in_output_format(self):
+        content = get_code_review_skill()
+        assert "RISK ASSESSMENT" in content

package/tests/test_github_pr_review_skill.py

@@ -0,0 +1,114 @@
+"""Tests for the github-pr-review skill content.
+
+These tests guard the guidance the agent reads when it builds a GitHub review
+with inline suggestions. The skill must teach the agent how GitHub
+``suggestion`` blocks actually behave (range replacement), so accepted
+suggestions don't silently duplicate or delete lines, and so the prose
+description always matches the resulting change.
+
+See: https://github.com/OpenHands/extensions/issues/292
+"""
+
+from pathlib import Path
+
+import pytest
+
+SKILL_PATH = (
+    Path(__file__).parent.parent / "skills" / "github-pr-review" / "SKILL.md"
+)
+
+
+@pytest.fixture(scope="module")
+def skill_text() -> str:
+    return SKILL_PATH.read_text(encoding="utf-8")
+
+
+def test_skill_file_exists():
+    assert SKILL_PATH.is_file(), f"missing skill file: {SKILL_PATH}"
+
+
+def test_skill_does_not_claim_suggestion_must_match_range_length(skill_text: str):
+    """Regression: the old "same number of lines as the range" rule was wrong.
+
+    GitHub replaces the targeted range with the suggestion body regardless of
+    how many lines the body contains. Telling the agent the counts must match
+    pushed it to either delete lines (when shrinking a block) or duplicate
+    lines (when expanding one), depending on which side it adjusted to satisfy
+    the bogus rule. The skill must no longer state that constraint.
+    """
+    forbidden_phrases = [
+        "suggestion must have the same number of lines as the range",
+        "must have the same number of lines as the range",
+    ]
+    for phrase in forbidden_phrases:
+        assert phrase not in skill_text, (
+            f"SKILL.md still contains the misleading rule {phrase!r}; "
+            "GitHub suggestions replace the range with any number of lines."
+        )
+
+
+def test_skill_explains_replace_semantics(skill_text: str):
+    """The skill must say in plain language that suggestions REPLACE the range."""
+    assert "How Suggestions Actually Work" in skill_text
+    # Mention replacement explicitly so the agent does not treat the suggestion
+    # body as "extra lines added on top of the original".
+    lowered = skill_text.lower()
+    assert "replace" in lowered
+    assert "any number of lines" in lowered
+
+
+def test_skill_covers_add_remove_change_patterns(skill_text: str):
+    """The skill should give explicit recipes for the three intents that
+    most commonly produced broken suggestions: changing, adding, and
+    deleting lines. Without these, the agent invents wrong patterns.
+    """
+    for keyword in ("Change line", "Add", "Delete"):
+        assert keyword in skill_text, (
+            f"SKILL.md is missing guidance for intent {keyword!r}; "
+            "see issue #292."
+        )
+
+
+def test_skill_warns_about_duplicated_and_deleted_lines(skill_text: str):
+    """The skill must call out the two failure modes from issue #292:
+    accepted suggestions that duplicate or remove lines."""
+    lowered = skill_text.lower()
+    assert "duplicat" in lowered, (
+        "SKILL.md should warn about suggestions that duplicate lines."
+    )
+    # "disappear"/"disappearing" is unique to the failure-mode description in
+    # the "Common Mistakes" section. The generic terms "delete" and "remove"
+    # also appear in the recipe table for intentional deletions, so checking
+    # for those would let the failure-mode guidance be removed silently.
+    assert "disappear" in lowered, (
+        "SKILL.md should warn about suggestions that silently drop lines "
+        "(see 'Disappearing lines' under Common Mistakes)."
+    )
+
+
+def test_skill_requires_description_to_match_suggestion(skill_text: str):
+    """The skill must explicitly require the prose description to match the
+    actual code change in the suggestion block (the third bug from #292)."""
+    lowered = skill_text.lower()
+    assert any(phrase in lowered for phrase in ("description does not match", "prose description", "description must match"))
+    # The text should connect the description to the resulting code change.
+    assert (
+        "match" in lowered
+    ), "SKILL.md should say the description must match the change."
+
+
+def test_skill_has_pre_post_verification_step(skill_text: str):
+    """Before posting, the agent must verify the suggestion against the file."""
+    assert "Mandatory Verification" in skill_text or "Verification" in skill_text
+    # Should point the agent at the real file content, not just the diff.
+    assert "sed -n" in skill_text or "read the actual file" in skill_text.lower()
+
+
+def test_skill_offers_prose_fallback(skill_text: str):
+    """When unsure, the agent should drop the suggestion block and use prose
+    instead — a correct comment beats a broken one-click fix."""
+    lowered = skill_text.lower()
+    assert "prose" in lowered, (
+        "SKILL.md should tell the agent to fall back to a prose comment "
+        "when the suggestion cannot be verified."
+    )

package/tests/test_issue_duplicate_checker.py

@@ -0,0 +1,240 @@
+import importlib.util
+import io
+import urllib.error
+from pathlib import Path
+
+import pytest
+
+
+ROOT = Path(__file__).resolve().parents[1]
+PLUGIN_DIR = ROOT / "plugins" / "issue-duplicate-checker" / "scripts"
+
+
+def load_script(name: str):
+    path = PLUGIN_DIR / f"{name}.py"
+    spec = importlib.util.spec_from_file_location(name, path)
+    module = importlib.util.module_from_spec(spec)
+    assert spec.loader is not None
+    spec.loader.exec_module(module)
+    return module
+
+
+def test_action_shell_blocks_do_not_interpolate_expressions_directly():
+    action_path = ROOT / "plugins" / "issue-duplicate-checker" / "action.yml"
+    lines = action_path.read_text().splitlines()
+    in_block = False
+    block_indent = 0
+    for line_number, line in enumerate(lines, start=1):
+        stripped = line.lstrip()
+        indent = len(line) - len(stripped)
+        if in_block and stripped and indent <= block_indent:
+            in_block = False
+        if stripped.startswith(("run:", "script:")):
+            block_scalar = stripped.split(":", 1)[1].strip()
+            if block_scalar.startswith(("|", ">")):
+                in_block = True
+                block_indent = indent
+            continue
+        assert not (in_block and "${{" in line), (
+            f"Move GitHub expression on line {line_number} into env before using it"
+        )
+
+
+def test_normalize_result_preserves_model_should_comment_false():
+    script = load_script("issue_duplicate_check_openhands")
+
+    result = script.normalize_result(
+        {
+            "should_comment": False,
+            "is_duplicate": True,
+            "auto_close_candidate": True,
+            "classification": "duplicate",
+            "confidence": "high",
+            "canonical_issue_number": 123,
+            "candidate_issues": [{"number": 123}],
+        }
+    )
+
+    assert result["should_comment"] is False
+    assert result["auto_close_candidate"] is False
+
+
+def test_github_headers_require_token(monkeypatch):
+    script = load_script("issue_duplicate_check_openhands")
+    monkeypatch.delenv("GITHUB_TOKEN", raising=False)
+
+    with pytest.raises(
+        RuntimeError,
+        match="GITHUB_TOKEN environment variable is required",
+    ):
+        script.github_headers()
+
+    monkeypatch.setenv("GITHUB_TOKEN", "token")
+    headers = script.github_headers()
+    assert headers["Authorization"] == "Bearer token"
+
+
+def test_issue_check_request_json_raises_structured_http_error(monkeypatch):
+    script = load_script("issue_duplicate_check_openhands")
+
+    def fake_urlopen(*args, **kwargs):
+        raise urllib.error.HTTPError(
+            url="https://api.example.test/example",
+            code=403,
+            msg="Forbidden",
+            hdrs=None,
+            fp=io.BytesIO(b'{"message":"rate limited"}'),
+        )
+
+    monkeypatch.setattr(script.urllib.request, "urlopen", fake_urlopen)
+
+    with pytest.raises(script.HTTPError) as exc_info:
+        script.request_json("https://api.example.test", "/example")
+
+    assert exc_info.value.status_code == 403
+    assert exc_info.value.url == "https://api.example.test/example"
+
+
+def test_parse_agent_json_extracts_fenced_json():
+    script = load_script("issue_duplicate_check_openhands")
+
+    result = script.parse_agent_json(
+        'Here is the result:\n```json\n'
+        '{"classification":"duplicate","confidence":"high"}\n```'
+    )
+
+    assert result == {"classification": "duplicate", "confidence": "high"}
+
+
+def test_normalize_result_guards_auto_close_and_infers_canonical_issue():
+    script = load_script("issue_duplicate_check_openhands")
+
+    overlapping = script.normalize_result(
+        {
+            "should_comment": True,
+            "is_duplicate": False,
+            "auto_close_candidate": True,
+            "classification": "overlapping-scope",
+            "confidence": "high",
+            "candidate_issues": [{"number": 10}],
+        }
+    )
+    assert overlapping["should_comment"] is True
+    assert overlapping["auto_close_candidate"] is False
+
+    low_confidence = script.normalize_result(
+        {
+            "should_comment": True,
+            "is_duplicate": True,
+            "auto_close_candidate": True,
+            "classification": "duplicate",
+            "confidence": "low",
+            "candidate_issues": [{"number": 11}],
+        }
+    )
+    assert low_confidence["should_comment"] is False
+    assert low_confidence["auto_close_candidate"] is False
+
+    inferred = script.normalize_result(
+        {
+            "should_comment": True,
+            "is_duplicate": True,
+            "auto_close_candidate": True,
+            "classification": "duplicate",
+            "confidence": "high",
+            "canonical_issue_number": None,
+            "candidate_issues": [{"number": "12"}],
+        }
+    )
+    assert inferred["auto_close_candidate"] is True
+    assert inferred["canonical_issue_number"] == 12
+
+
+def test_build_prompt_includes_required_schema_keys():
+    script = load_script("issue_duplicate_check_openhands")
+
+    prompt = script.build_prompt(
+        "OpenHands/extensions",
+        {
+            "number": 123,
+            "title": "Duplicate bug",
+            "body": "Looks related",
+            "html_url": "https://github.com/OpenHands/extensions/issues/123",
+        },
+    )
+
+    for key in [
+        "should_comment",
+        "is_duplicate",
+        "auto_close_candidate",
+        "classification",
+        "confidence",
+        "canonical_issue_number",
+        "candidate_issues",
+    ]:
+        assert key in prompt
+
+
+def test_find_latest_auto_close_comment_uses_newest_auto_close_marker():
+    script = load_script("auto_close_duplicate_issues")
+
+    latest, canonical = script.find_latest_auto_close_comment(
+        [
+            {
+                "id": 1,
+                "created_at": "2026-01-01T00:00:00Z",
+                "body": "<!-- openhands-duplicate-check canonical=10 auto-close=true -->",
+            },
+            {
+                "id": 2,
+                "created_at": "2026-01-03T00:00:00Z",
+                "body": "<!-- openhands-duplicate-check canonical=20 auto-close=false -->",
+            },
+            {
+                "id": 3,
+                "created_at": None,
+                "body": "<!-- openhands-duplicate-check canonical=30 auto-close=true -->",
+            },
+            {
+                "id": 4,
+                "created_at": "2026-01-02T00:00:00Z",
+                "body": "<!-- openhands-duplicate-check canonical=40 auto-close=true -->",
+            },
+        ]
+    )
+
+    assert latest["id"] == 4
+    assert canonical == 40
+
+
+def test_request_json_raises_structured_http_error(monkeypatch):
+    script = load_script("auto_close_duplicate_issues")
+    monkeypatch.setenv("GITHUB_TOKEN", "token")
+
+    def fake_urlopen(*args, **kwargs):
+        raise urllib.error.HTTPError(
+            url="https://api.github.com/example",
+            code=404,
+            msg="Not Found",
+            hdrs=None,
+            fp=io.BytesIO(b'{"message":"missing"}'),
+        )
+
+    monkeypatch.setattr(script.urllib.request, "urlopen", fake_urlopen)
+
+    with pytest.raises(script.HTTPError) as exc_info:
+        script.request_json("/example")
+
+    assert exc_info.value.status_code == 404
+    assert exc_info.value.path == "/example"
+
+
+def test_fetch_issue_returns_none_on_404(monkeypatch):
+    script = load_script("auto_close_duplicate_issues")
+
+    def fake_request_json(path):
+        raise script.HTTPError("GET", path, 404, "missing")
+
+    monkeypatch.setattr(script, "request_json", fake_request_json)
+
+    assert script.fetch_issue("OpenHands/extensions", 123) is None