@openhands/extensions 0.0.1-alpha → 0.2.0

package/tests/test_pr_review_diff_payload.py

@@ -0,0 +1,202 @@
+"""Tests for the per-file diff payload introduced to replace the global byte-
+slice truncation in agent_script.py.
+
+The legacy `truncate_text()` cut the raw diff at byte 100,000, which silently
+dropped files whose patches lived past that point (issue #233). These tests
+exercise the replacement: a manifest that lists every file plus per-file
+budgeted patches that mark abbreviation/omission explicitly.
+"""
+
+from __future__ import annotations
+
+# Re-use the module loader from the review-context test file so we don't
+# duplicate ~130 lines of openhands-SDK stubbing.
+from test_pr_review_review_context import _load_agent_script_module
+
+
+def _file(
+    filename: str,
+    *,
+    status: str = "modified",
+    additions: int = 0,
+    deletions: int = 0,
+    patch: str | None = "",
+    previous_filename: str | None = None,
+) -> dict:
+    return {
+        "filename": filename,
+        "status": status,
+        "additions": additions,
+        "deletions": deletions,
+        "patch": patch,
+        "previous_filename": previous_filename,
+    }
+
+
+# --- manifest -----------------------------------------------------------
+
+
+def test_manifest_lists_every_file():
+    module = _load_agent_script_module()
+    files = [
+        _file("a.py", status="added", additions=10),
+        _file("b.py", status="modified", additions=2, deletions=3),
+        _file("c.py", status="removed", deletions=15),
+    ]
+    manifest = module.format_files_manifest(files)
+    assert "`a.py`" in manifest
+    assert "`b.py`" in manifest
+    assert "`c.py`" in manifest
+    assert "[added]" in manifest
+    assert "[modified]" in manifest
+    assert "[removed]" in manifest
+    assert "+10" in manifest
+    assert "+2/-3" in manifest
+    assert "-15" in manifest
+
+
+def test_manifest_includes_totals_in_header():
+    module = _load_agent_script_module()
+    files = [
+        _file("a.py", additions=10, deletions=2),
+        _file("b.py", additions=5, deletions=1),
+    ]
+    manifest = module.format_files_manifest(files)
+    assert "2 files" in manifest
+    assert "+15" in manifest
+    assert "-3" in manifest
+
+
+def test_manifest_handles_renames():
+    module = _load_agent_script_module()
+    files = [
+        _file(
+            "new.py",
+            status="renamed",
+            additions=1,
+            deletions=1,
+            previous_filename="old.py",
+        ),
+    ]
+    manifest = module.format_files_manifest(files)
+    assert "`new.py`" in manifest
+    assert "renamed from old.py" in manifest
+
+
+def test_manifest_flags_binary_files():
+    module = _load_agent_script_module()
+    files = [
+        _file("logo.png", status="modified", additions=1, deletions=1, patch=""),
+    ]
+    manifest = module.format_files_manifest(files)
+    assert "`logo.png`" in manifest
+    assert "binary or unavailable" in manifest
+
+
+# --- patch formatting ---------------------------------------------------
+
+
+def test_patches_include_header_for_every_file():
+    """Every file gets a diff header even when only a stub fits."""
+    module = _load_agent_script_module()
+    files = [
+        _file("a.py", status="added", additions=1, patch="@@ -0,0 +1 @@\n+x\n"),
+        _file("b.py", status="added", additions=1, patch="@@ -0,0 +1 @@\n+y\n"),
+    ]
+    out = module.format_patches(files)
+    assert "diff --git a/a.py b/a.py" in out
+    assert "diff --git a/b.py b/b.py" in out
+
+
+def test_patches_below_budget_passthrough():
+    module = _load_agent_script_module()
+    patch = "@@ -1,2 +1,2 @@\n-foo\n+bar\n"
+    files = [_file("a.py", patch=patch)]
+    out = module.format_patches(files, max_total=10000, max_per_file=10000)
+    assert patch in out
+    assert "[patch abbreviated" not in out
+
+
+def test_per_file_cap_abbreviates_huge_single_file():
+    """A single large patch is abbreviated, not dropped."""
+    module = _load_agent_script_module()
+    big_patch = "@@ -0,0 +1,1000 @@\n" + "".join(
+        f"+line {i}\n" for i in range(1000)
+    )
+    files = [_file("big.py", status="added", patch=big_patch)]
+    out = module.format_patches(files, max_total=100000, max_per_file=500)
+    assert "diff --git a/big.py b/big.py" in out
+    # The marker is the contract — the agent knows the patch was cut.
+    assert "[patch abbreviated" in out
+    # Should reference the actual file path so the agent knows where to look.
+    assert "`big.py`" in out
+    # Must not contain the full patch text.
+    assert len(out) < len(big_patch)
+
+
+def test_total_budget_omits_late_files_with_marker_not_silence():
+    """When total budget is exhausted, later files become header-only stubs
+    with an explicit `[patch omitted]` marker — never silently dropped."""
+    module = _load_agent_script_module()
+    fat_patch = "@@ -0,0 +1,100 @@\n" + "".join(f"+x{i}\n" for i in range(100))
+    files = [
+        _file("first.py", status="added", patch=fat_patch),
+        _file("second.py", status="added", patch=fat_patch),
+        _file("third.py", status="added", patch=fat_patch),
+    ]
+    # Budget that comfortably fits one full patch but not three.
+    out = module.format_patches(files, max_total=800, max_per_file=10000)
+
+    # All three files appear in the patch block — at minimum as headers.
+    assert "diff --git a/first.py b/first.py" in out
+    assert "diff --git a/second.py b/second.py" in out
+    assert "diff --git a/third.py b/third.py" in out
+    # At least one of the later files is marked omitted.
+    assert "[patch omitted" in out
+
+
+def test_no_patch_field_is_annotated_not_silent():
+    """A file with no patch text (binary, rename, etc) gets a clear note."""
+    module = _load_agent_script_module()
+    files = [_file("logo.png", additions=1, deletions=1, patch="")]
+    out = module.format_patches(files)
+    assert "diff --git a/logo.png b/logo.png" in out
+    assert "no patch available" in out
+
+
+def test_smoking_gun_pr_14401_shape():
+    """Regression for issue #233 — the specific failure mode that triggered
+    the redesign. Earlier files have huge patches; the implementation file
+    of interest sits past the byte-100K mark of the original raw diff. With
+    a per-file budget the late file's *patch* may still be abbreviated, but
+    its *presence* must always be visible in both manifest and patch block."""
+    module = _load_agent_script_module()
+
+    # Two heavy files dominate the byte budget.
+    bulk_patch = "@@ -1 +1,2000 @@\n" + "".join(
+        f"+filler {i}\n" for i in range(2000)
+    )
+    # The implementation file the bot kept missing.
+    target = (
+        "@@ -0,0 +1,55 @@\n"
+        + "".join(f"+line {i}\n" for i in range(55))
+    )
+    files = [
+        _file("enterprise/poetry.lock", patch=bulk_patch),
+        _file("openhands/long_module.py", patch=bulk_patch),
+        _file(
+            "frontend/src/utils/shell-tokenize.ts",
+            status="added",
+            additions=55,
+            patch=target,
+        ),
+    ]
+
+    manifest = module.format_files_manifest(files)
+    patches = module.format_patches(files, max_total=2000, max_per_file=600)
+
+    # Manifest always names the target file — this is the property that
+    # prevents the bot from claiming the file is missing.
+    assert "`frontend/src/utils/shell-tokenize.ts`" in manifest
+    # Patch block lists it too, even when its content is abbreviated/omitted.
+    assert "diff --git a/frontend/src/utils/shell-tokenize.ts" in patches

package/tests/test_pr_review_feedback.py

@@ -0,0 +1,263 @@
+"""Tests for PR review feedback collection."""
+
+import importlib.util
+import sys
+import types
+from pathlib import Path
+
+import yaml
+
+
+_ROOT = Path(__file__).parent.parent
+_PR_REVIEW_SCRIPTS = _ROOT / "plugins" / "pr-review" / "scripts"
+
+
+def _load_prompt_module():
+    path = _PR_REVIEW_SCRIPTS / "prompt.py"
+    spec = importlib.util.spec_from_file_location("pr_review_prompt_feedback", path)
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[spec.name] = module
+    spec.loader.exec_module(module)
+    return module
+
+
+
+def _load_eval_module():
+    """Load evaluate_review.py with Laminar stubbed."""
+    lmnr_mod = types.ModuleType("lmnr")
+
+    class _FakeLaminar:
+        @staticmethod
+        def initialize():
+            return None
+
+        @staticmethod
+        def get_trace_id():
+            return None
+
+        @staticmethod
+        def get_laminar_span_context():
+            return None
+
+        @staticmethod
+        def set_trace_metadata(metadata):
+            return None
+
+        @staticmethod
+        def set_span_output(output):
+            return None
+
+        @staticmethod
+        def flush():
+            return None
+
+        @staticmethod
+        def start_as_current_span(**kwargs):
+            import contextlib
+
+            return contextlib.nullcontext()
+
+    class _FakeClient:
+        class evaluators:
+            @staticmethod
+            def score(**kwargs):
+                return None
+
+        class tags:
+            @staticmethod
+            def tag(trace_id, tags):
+                return None
+
+    lmnr_mod.Laminar = _FakeLaminar
+    lmnr_mod.LaminarClient = _FakeClient
+
+    saved = sys.modules.get("lmnr")
+    sys.modules["lmnr"] = lmnr_mod
+    try:
+        path = _PR_REVIEW_SCRIPTS / "evaluate_review.py"
+        spec = importlib.util.spec_from_file_location("pr_review_evaluate", path)
+        module = importlib.util.module_from_spec(spec)
+        sys.modules[spec.name] = module
+        spec.loader.exec_module(module)
+        return module
+    finally:
+        if saved is None:
+            sys.modules.pop("lmnr", None)
+        else:
+            sys.modules["lmnr"] = saved
+
+
+
+def _format_prompt(*, collect_feedback: bool, review_run_url: str = "") -> str:
+    module = _load_prompt_module()
+    return module.format_prompt(
+        skill_trigger="/codereview",
+        title="Add review feedback footer",
+        body="## Summary\nCapture review reactions without extra PR spam.",
+        repo_name="OpenHands/extensions",
+        base_branch="main",
+        head_branch="feature/feedback-footer",
+        pr_number="249",
+        commit_id="abc123",
+        diff="diff --git a/file b/file",
+        review_context="",
+        require_evidence=False,
+        collect_feedback=collect_feedback,
+        review_run_url=review_run_url,
+        use_sub_agents=False,
+    )
+
+
+
+def test_action_collect_feedback_defaults_to_true_and_uses_agent_prompt():
+    action_yml = _ROOT / "plugins" / "pr-review" / "action.yml"
+    with open(action_yml) as f:
+        action = yaml.safe_load(f)
+
+    collect_feedback = action["inputs"]["collect-feedback"]
+    assert collect_feedback["default"] == "true"
+
+    run_step = next(
+        step for step in action["runs"]["steps"] if step["name"] == "Run PR review"
+    )
+    assert run_step["env"]["COLLECT_FEEDBACK"] == "${{ inputs.collect-feedback }}"
+    assert run_step["env"]["REVIEW_RUN_URL"] == (
+        "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+    )
+    assert all(
+        step["name"] != "Post PR review feedback prompt"
+        for step in action["runs"]["steps"]
+    )
+
+
+
+def test_prompt_omits_feedback_footer_by_default():
+    prompt = _format_prompt(collect_feedback=False)
+
+    assert "## Review Feedback Footer" not in prompt
+    assert "React with 👍 or 👎 to this review" not in prompt
+    assert "<!-- openhands-pr-review-feedback -->" not in prompt
+
+
+
+def test_prompt_includes_feedback_footer_when_enabled():
+    prompt = _format_prompt(
+        collect_feedback=True,
+        review_run_url="https://github.com/OpenHands/extensions/actions/runs/123",
+    )
+
+    assert "## Review Feedback Footer" in prompt
+    assert "main review body, not in a separate issue comment" in prompt
+    assert "React with 👍 or 👎 to this review" in prompt
+    assert "https://github.com/OpenHands/extensions/actions/runs/123" in prompt
+    assert "<!-- openhands-pr-review-feedback -->" in prompt
+
+
+
+def test_extract_review_feedback_counts_thumbs_reactions_from_reviews():
+    module = _load_eval_module()
+
+    reviews = [
+        {
+            "id": 101,
+            "user": {"login": "openhands-agent"},
+            "body": "Automated review\n<!-- openhands-pr-review-feedback -->",
+            "submitted_at": "2026-05-19T12:00:00Z",
+            "reactions": {"+1": 3, "-1": 1, "total_count": 4},
+        },
+        {
+            "id": 102,
+            "user": {"login": "openhands-agent"},
+            "body": "Regular review comment",
+            "reactions": {"+1": 100, "-1": 100},
+        },
+        {
+            "id": 103,
+            "user": {"login": "human-dev"},
+            "body": "<!-- openhands-pr-review-feedback -->",
+            "reactions": {"+1": 5, "-1": 0},
+        },
+    ]
+
+    assert module.extract_review_feedback([], reviews) == [
+        {
+            "comment_id": 101,
+            "created_at": "2026-05-19T12:00:00Z",
+            "thumbs_up": 3,
+            "thumbs_down": 1,
+            "total": 4,
+        }
+    ]
+
+
+
+def test_extract_review_feedback_keeps_legacy_issue_comment_support():
+    module = _load_eval_module()
+
+    result = module.extract_review_feedback(
+        [
+            {
+                "id": 201,
+                "user": {"login": "all-hands-bot"},
+                "body": "<!-- openhands-pr-review-feedback -->",
+                "created_at": "2026-05-19T12:00:00Z",
+                "reactions": {"+1": 2, "-1": 0},
+            }
+        ]
+    )
+
+    assert result == [
+        {
+            "comment_id": 201,
+            "created_at": "2026-05-19T12:00:00Z",
+            "thumbs_up": 2,
+            "thumbs_down": 0,
+            "total": 2,
+        }
+    ]
+
+
+
+def test_extract_review_feedback_handles_missing_reactions():
+    module = _load_eval_module()
+
+    result = module.extract_review_feedback(
+        [],
+        [
+            {
+                "id": 301,
+                "user": {"login": "all-hands-bot"},
+                "body": "<!-- openhands-pr-review-feedback -->",
+            }
+        ],
+    )
+
+    assert result == [
+        {
+            "comment_id": 301,
+            "created_at": None,
+            "thumbs_up": 0,
+            "thumbs_down": 0,
+            "total": 0,
+        }
+    ]
+
+
+
+def test_extract_review_feedback_accepts_github_actions_bot():
+    module = _load_eval_module()
+
+    result = module.extract_review_feedback(
+        [],
+        [
+            {
+                "id": 401,
+                "user": {"login": "github-actions[bot]"},
+                "body": "<!-- openhands-pr-review-feedback -->",
+                "reactions": {"+1": 1, "-1": 2},
+            }
+        ],
+    )
+
+    assert result[0]["thumbs_up"] == 1
+    assert result[0]["thumbs_down"] == 2

package/tests/test_pr_review_prompt.py

@@ -0,0 +1,152 @@
+import importlib.util
+import sys
+from pathlib import Path
+
+import yaml
+
+
+def _load_prompt_module():
+    script_path = (
+        Path(__file__).parent.parent
+        / "plugins"
+        / "pr-review"
+        / "scripts"
+        / "prompt.py"
+    )
+    spec = importlib.util.spec_from_file_location("pr_review_prompt", script_path)
+    module = importlib.util.module_from_spec(spec)
+    sys.modules["pr_review_prompt"] = module
+    spec.loader.exec_module(module)
+    return module
+
+
+def _format_prompt(
+    *, require_evidence: bool, use_sub_agents: bool = False
+) -> str:
+    module = _load_prompt_module()
+    return module.format_prompt(
+        skill_trigger="/codereview",
+        title="Add evidence enforcement",
+        body="## Summary\nAdds stricter review guidance.",
+        repo_name="OpenHands/extensions",
+        base_branch="main",
+        head_branch="feature/evidence",
+        pr_number="102",
+        commit_id="abc123",
+        diff="diff --git a/file b/file",
+        review_context="",
+        require_evidence=require_evidence,
+        use_sub_agents=use_sub_agents,
+    )
+
+
+def test_prompt_with_roasted_trigger():
+    """Verify the backward-compatibility trigger alias works."""
+    module = _load_prompt_module()
+    prompt = module.format_prompt(
+        skill_trigger="/codereview-roasted",
+        title="Test PR",
+        body="Test body",
+        repo_name="owner/repo",
+        base_branch="main",
+        head_branch="feature",
+        pr_number="1",
+        commit_id="abc123",
+        diff="test diff",
+        review_context="",
+        require_evidence=False,
+    )
+    assert "/codereview-roasted" in prompt
+
+
+def test_format_prompt_omits_evidence_requirements_by_default():
+    prompt = _format_prompt(require_evidence=False)
+
+    assert "## PR Description Evidence Requirement" not in prompt
+    assert "real code path end-to-end" not in prompt
+    assert "https://app.all-hands.dev/conversations/{conversation_id}" not in prompt
+
+
+def test_format_prompt_includes_evidence_requirements_when_enabled():
+    prompt = _format_prompt(require_evidence=True)
+
+    assert "## PR Description Evidence Requirement" in prompt
+    assert "`Evidence` section" in prompt
+    assert "screenshot or video" in prompt
+    assert "real code path end-to-end" in prompt
+    assert "unit test output" in prompt
+    assert "https://app.all-hands.dev/conversations/{conversation_id}" in prompt
+
+
+# --- Sub-agent delegation prompt tests ---
+
+
+def test_format_prompt_uses_standard_prompt_by_default():
+    prompt = _format_prompt(require_evidence=False, use_sub_agents=False)
+
+    # Standard prompt should NOT mention delegation or sub-agents
+    assert "review coordinator" not in prompt
+    assert "TaskToolSet" not in prompt
+    assert "file_reviewer" not in prompt
+    # Standard prompt should contain the normal review instruction
+    assert "Analyze the changes and post your review" in prompt
+
+
+def test_format_prompt_appends_delegation_suffix_when_enabled():
+    prompt = _format_prompt(require_evidence=False, use_sub_agents=True)
+
+    # Should still include the base prompt content
+    assert "Add evidence enforcement" in prompt
+    assert "OpenHands/extensions" in prompt
+    assert "abc123" in prompt
+    assert "diff --git a/file b/file" in prompt
+    assert "Analyze the changes and post your review" in prompt
+    # Delegation suffix appended
+    assert "Sub-agent Delegation" in prompt
+    assert "file_reviewer" in prompt
+    assert "task" in prompt.lower()
+
+
+def test_delegation_suffix_with_evidence():
+    prompt = _format_prompt(require_evidence=True, use_sub_agents=True)
+
+    assert "Sub-agent Delegation" in prompt
+    assert "## PR Description Evidence Requirement" in prompt
+
+
+def test_file_reviewer_skill_content():
+    module = _load_prompt_module()
+    content = module.FILE_REVIEWER_SKILL
+
+    assert "file-level code reviewer" in content
+    # Unified review style
+    assert "pragmatic" in content
+    # JSON schema documented
+    assert "path" in content
+    assert "line" in content
+    assert "severity" in content
+    assert "body" in content
+    assert "critical" in content
+    # Tool access documented
+    assert "terminal" in content
+    assert "file_editor" in content
+    # Sub-agent returns results via finish tool
+    assert "finish" in content
+
+
+# --- action.yml default guard ---
+
+
+def test_use_sub_agents_defaults_to_false():
+    """Guard: use-sub-agents must default to 'false' until cost issues are resolved (#208)."""
+    action_yml = (
+        Path(__file__).parent.parent / "plugins" / "pr-review" / "action.yml"
+    )
+    with open(action_yml) as f:
+        action = yaml.safe_load(f)
+
+    default = action["inputs"]["use-sub-agents"]["default"]
+    assert default == "false", (
+        f"use-sub-agents default should be 'false' (got {default!r}). "
+        "See https://github.com/OpenHands/extensions/issues/208"
+    )