@openhands/extensions 0.22.0 → 0.23.0

package/tests/test_github_reviewer_delivery.py

@@ -1,11 +1,30 @@
 """Contract tests for delegated GitHub PR review."""
 
+import json
 from pathlib import Path
 from unittest.mock import Mock
 
+import pytest
+
 from github_automation_helpers import worker
 
 
+def _reviews(
+    verdict="✅ APPROVED",
+    submitted_at="2026-01-02T00:00:00Z",
+    sha="head-2",
+    login="all-hands-bot",
+):
+    return [
+        {
+            "body": f"Review body\n\n{verdict}",
+            "commit_id": sha,
+            "submitted_at": submitted_at,
+            "user": {"login": login},
+        }
+    ]
+
+
 def _reviewer(tmp_path, monkeypatch):
     module = worker("github-pr-reviewer", tmp_path, monkeypatch)
     run = object.__new__(module.PullRequestReviewer)
@@ -13,10 +32,33 @@ def _reviewer(tmp_path, monkeypatch):
     run.repository = "owner/repo"
     run.token = "token"
     run.token_name = "FACTORY_GITHUB_REVIEWER_TOKEN"
+    run.github_login = "all-hands-bot"
     run.dispatcher = Mock()
+    monkeypatch.delenv("AUTOMATION_EVENT_PAYLOAD", raising=False)
+    monkeypatch.setattr(
+        module.workflow,
+        "_latest_trigger_label_event",
+        lambda *args: {"id": 7, "created_at": "2026-01-01T00:00:00Z"},
+    )
     return module, run
 
 
+def _event(monkeypatch, *, action="review_requested", login="all-hands-bot"):
+    payload = {
+        "action": action,
+        "repository": {"full_name": "owner/repo"},
+        "pull_request": {"number": 2},
+    }
+    if action == "review_requested":
+        payload["requested_reviewer"] = {"login": login}
+    else:
+        payload["review"] = {"user": {"login": login}}
+    monkeypatch.setenv(
+        "AUTOMATION_EVENT_PAYLOAD",
+        json.dumps({"automation_id": "automation", "event": {"payload": payload}}),
+    )
+
+
 def test_reviewer_submits_each_labeled_exact_head(tmp_path, monkeypatch):
     module, run = _reviewer(tmp_path, monkeypatch)
     prs = [
@@ -45,12 +87,168 @@ def test_reviewer_submits_each_labeled_exact_head(tmp_path, monkeypatch):
     assert submit.call_args.kwargs["delivery"] == "7:head-2"
     prompt = submit.call_args.kwargs["prompt"]
     assert "publish the review directly to GitHub" in prompt
-    assert "software-factory/review" in prompt
-    assert "software-factory/tests" in prompt
+    assert "Do not create commit statuses or Checks" in prompt
     assert "FACTORY_GITHUB_REVIEWER_TOKEN" in prompt
     assert "gh auth setup-git" in prompt
     assert "GIT_TERMINAL_PROMPT=0" in prompt
     assert "Never paste JSON artifacts" in prompt
+    assert "stop immediately" in prompt
+
+
+def test_reviewer_submits_requested_exact_head(tmp_path, monkeypatch):
+    _module, run = _reviewer(tmp_path, monkeypatch)
+    _event(monkeypatch)
+    pr = {"number": 2, "head": {"sha": "head-2"}, "labels": []}
+    request = {
+        "id": 42,
+        "event": "review_requested",
+        "created_at": "2026-01-01T00:00:00Z",
+        "requested_reviewer": {"login": "all-hands-bot"},
+    }
+    run.gh = Mock(side_effect=[{"id": 99}, pr])
+    run.gh_pages = lambda path: [request] if path.endswith("/events") else []
+    run.dispatcher.deliver.return_value = {
+        "disposition": "created",
+        "conversation_id": "conversation",
+    }
+
+    run.run()
+
+    call = run.dispatcher.deliver.call_args.kwargs
+    assert call["subject"] == "99:pr:2"
+    assert call["delivery"] == "42:head-2"
+    assert "latest review request for `all-hands-bot` event 42" in call["prompt"]
+    assert "new head requires another reviewer request" in call["prompt"]
+
+
+def test_reviewer_ignores_request_for_another_reviewer(tmp_path, monkeypatch):
+    _module, run = _reviewer(tmp_path, monkeypatch)
+    _event(monkeypatch, login="human")
+    run.gh = Mock(return_value={"id": 99})
+
+    run.run()
+
+    run.dispatcher.deliver.assert_not_called()
+
+
+def test_reviewer_event_hands_positive_review_to_maintainer(tmp_path, monkeypatch):
+    module, run = _reviewer(tmp_path, monkeypatch)
+    _event(monkeypatch, action="submitted")
+    run.config["maintainers"] = "neubig, VascoSch92"
+    pr = {"number": 2, "head": {"sha": "head-2"}, "labels": []}
+    request = {
+        "id": 42,
+        "event": "review_requested",
+        "created_at": "2026-01-01T00:00:00Z",
+        "requested_reviewer": {"login": "all-hands-bot"},
+    }
+    run.gh = Mock(side_effect=[{"id": 99}, pr, pr])
+    run.gh_pages = lambda path: [request] if path.endswith("/events") else _reviews()
+    handoff = Mock(return_value="VascoSch92")
+    monkeypatch.setattr(module, "request_maintainer_review", handoff)
+
+    run.run()
+
+    handoff.assert_called_once_with(run, pr, ["neubig", "VascoSch92"])
+    assert all(call.args[0] != "DELETE" for call in run.gh.call_args_list)
+    run.dispatcher.deliver.assert_not_called()
+
+
+def test_reviewer_submitted_non_decisive_review_does_not_dispatch(
+    tmp_path, monkeypatch
+):
+    module, run = _reviewer(tmp_path, monkeypatch)
+    _event(monkeypatch, action="submitted")
+    run.config["maintainers"] = "neubig"
+    pr = {"number": 2, "head": {"sha": "head-2"}, "labels": []}
+    request = {
+        "id": 42,
+        "event": "review_requested",
+        "created_at": "2026-01-01T00:00:00Z",
+        "requested_reviewer": {"login": "all-hands-bot"},
+    }
+    run.gh = Mock(side_effect=[{"id": 99}, pr])
+    run.gh_pages = lambda path: (
+        [request] if path.endswith("/events") else _reviews(verdict="LGTM")
+    )
+    handoff = Mock()
+    monkeypatch.setattr(module, "request_maintainer_review", handoff)
+
+    run.run()
+
+    run.dispatcher.deliver.assert_not_called()
+    handoff.assert_not_called()
+
+
+def test_reviewer_submitted_review_on_superseded_head_does_not_dispatch(
+    tmp_path, monkeypatch
+):
+    module, run = _reviewer(tmp_path, monkeypatch)
+    _event(monkeypatch, action="submitted")
+    run.config["maintainers"] = "neubig"
+    pr = {"number": 2, "head": {"sha": "head-3"}, "labels": []}
+    request = {
+        "id": 42,
+        "event": "review_requested",
+        "created_at": "2026-01-01T00:00:00Z",
+        "requested_reviewer": {"login": "all-hands-bot"},
+    }
+    run.gh = Mock(side_effect=[{"id": 99}, pr])
+    run.gh_pages = lambda path: (
+        [request] if path.endswith("/events") else _reviews(sha="head-2")
+    )
+    handoff = Mock()
+    monkeypatch.setattr(module, "request_maintainer_review", handoff)
+
+    run.run()
+
+    run.dispatcher.deliver.assert_not_called()
+    handoff.assert_not_called()
+
+
+def test_reviewer_redelivers_when_review_predates_latest_label(tmp_path, monkeypatch):
+    _module, run = _reviewer(tmp_path, monkeypatch)
+    pr = {
+        "number": 2,
+        "head": {"sha": "head-2"},
+        "labels": [{"name": "openhands-review"}],
+    }
+    run.gh_pages = lambda path: (
+        [pr]
+        if path.startswith("/pulls?")
+        else _reviews(submitted_at="2025-12-31T00:00:00Z")
+    )
+    run.gh = Mock(side_effect=[{"id": 99}, pr])
+    run.dispatcher.deliver.return_value = {
+        "disposition": "continued",
+        "conversation_id": "conversation",
+    }
+
+    run.run()
+
+    run.dispatcher.deliver.assert_called_once()
+    assert run.dispatcher.deliver.call_args.kwargs["delivery"] == "7:head-2"
+
+
+def test_reviewer_does_not_trust_another_reviewers_verdict(tmp_path, monkeypatch):
+    _module, run = _reviewer(tmp_path, monkeypatch)
+    pr = {
+        "number": 2,
+        "head": {"sha": "head-2"},
+        "labels": [{"name": "openhands-review"}],
+    }
+    run.gh_pages = lambda path: (
+        [pr] if path.startswith("/pulls?") else _reviews(login="other-reviewer")
+    )
+    run.gh = Mock(side_effect=[{"id": 99}, pr])
+    run.dispatcher.deliver.return_value = {
+        "disposition": "continued",
+        "conversation_id": "conversation",
+    }
+
+    run.run()
+
+    run.dispatcher.deliver.assert_called_once()
 
 
 def test_reviewer_ignores_unlabeled_prs(tmp_path, monkeypatch):
@@ -65,7 +263,9 @@ def test_reviewer_ignores_unlabeled_prs(tmp_path, monkeypatch):
     submit.assert_not_called()
 
 
-def test_reviewer_continues_after_one_submission_fails(tmp_path, monkeypatch):
+def test_reviewer_continues_after_one_submission_fails_then_reports_run_failure(
+    tmp_path, monkeypatch
+):
     module, run = _reviewer(tmp_path, monkeypatch)
     prs = [
         {
@@ -90,7 +290,8 @@ def test_reviewer_continues_after_one_submission_fails(tmp_path, monkeypatch):
     )
     run.dispatcher.deliver = submit
 
-    run.run()
+    with pytest.raises(RuntimeError, match="Reviewer scan failed for PRs: #1"):
+        run.run()
 
     assert submit.call_count == 2
 
@@ -108,3 +309,118 @@ def test_reviewer_scanner_has_no_conversation_or_runtime_code():
         "AUTOMATION_CONVERSATION_ID",
     ):
         assert forbidden not in source
+
+
+def test_reviewer_hands_positive_exact_head_to_maintainer(tmp_path, monkeypatch):
+    module, run = _reviewer(tmp_path, monkeypatch)
+    run.config["maintainers"] = "neubig, VascoSch92"
+    pr = {
+        "number": 2,
+        "head": {"sha": "head-2"},
+        "labels": [{"name": "openhands-review"}],
+    }
+    run.gh_pages = lambda path: [pr] if path.startswith("/pulls?") else _reviews()
+    run.gh = Mock(side_effect=[{"id": 99}, pr, pr, {}])
+    handoff = Mock(return_value="VascoSch92")
+    monkeypatch.setattr(module, "request_maintainer_review", handoff)
+
+    run.run()
+
+    handoff.assert_called_once_with(run, pr, ["neubig", "VascoSch92"])
+    assert run.gh.call_args_list[-1].args == (
+        "DELETE",
+        "/issues/2/labels/openhands-review",
+    )
+    run.dispatcher.deliver.assert_not_called()
+
+
+def test_reviewer_retries_failed_handoff_without_clearing_label(tmp_path, monkeypatch):
+    module, run = _reviewer(tmp_path, monkeypatch)
+    run.config["maintainers"] = "neubig"
+    pr = {
+        "number": 2,
+        "head": {"sha": "head-2"},
+        "labels": [{"name": "openhands-review"}],
+    }
+    run.gh_pages = lambda path: [pr] if path.startswith("/pulls?") else _reviews()
+    run.gh = Mock(side_effect=[{"id": 99}, pr])
+    monkeypatch.setattr(
+        module,
+        "request_maintainer_review",
+        Mock(side_effect=RuntimeError("temporary GitHub failure")),
+    )
+
+    with pytest.raises(RuntimeError, match="Reviewer scan failed for PRs: #2"):
+        run.run()
+
+    assert all(call.args[0] != "DELETE" for call in run.gh.call_args_list)
+    run.dispatcher.deliver.assert_not_called()
+
+
+def test_reviewer_reports_permanent_handoff_error_once(tmp_path, monkeypatch):
+    module, run = _reviewer(tmp_path, monkeypatch)
+    run.config["maintainers"] = "author"
+    pr = {
+        "number": 2,
+        "head": {"sha": "head-2"},
+        "labels": [{"name": "openhands-review"}],
+    }
+    run.gh_pages = lambda path: [pr] if path.startswith("/pulls?") else _reviews()
+    run.gh = Mock(side_effect=[{"id": 99}, pr, {}, pr, {}])
+    monkeypatch.setattr(
+        module,
+        "request_maintainer_review",
+        Mock(
+            side_effect=module.HandoffConfigurationError(
+                "No eligible maintainer remains"
+            )
+        ),
+    )
+
+    run.run()
+
+    assert "configuration check" in run.gh.call_args_list[2].args[2]["body"]
+    assert run.gh.call_args_list[-1].args == (
+        "DELETE",
+        "/issues/2/labels/openhands-review",
+    )
+
+
+def test_reviewer_keeps_label_when_head_moves_before_cleanup(tmp_path, monkeypatch):
+    _module, run = _reviewer(tmp_path, monkeypatch)
+    pr = {
+        "number": 2,
+        "head": {"sha": "head-2"},
+        "labels": [{"name": "openhands-review"}],
+    }
+    moved = {**pr, "head": {"sha": "head-3"}}
+    run.gh_pages = lambda path: (
+        [pr] if path.startswith("/pulls?") else _reviews(verdict="🔄 CHANGES REQUESTED")
+    )
+    run.gh = Mock(side_effect=[{"id": 99}, pr, moved])
+
+    run.run()
+
+    assert all(call.args[0] != "DELETE" for call in run.gh.call_args_list)
+    run.dispatcher.deliver.assert_not_called()
+
+
+def test_reviewer_does_not_handoff_failed_review(tmp_path, monkeypatch):
+    module, run = _reviewer(tmp_path, monkeypatch)
+    run.config["maintainers"] = "neubig"
+    pr = {
+        "number": 2,
+        "head": {"sha": "head-2"},
+        "labels": [{"name": "openhands-review"}],
+    }
+    run.gh_pages = lambda path: (
+        [pr] if path.startswith("/pulls?") else _reviews(verdict="🔄 CHANGES REQUESTED")
+    )
+    run.gh = Mock(side_effect=[{"id": 99}, pr, pr, {}])
+    handoff = Mock()
+    monkeypatch.setattr(module, "request_maintainer_review", handoff)
+
+    run.run()
+
+    handoff.assert_not_called()
+    run.dispatcher.deliver.assert_not_called()

package/tests/test_github_triage_delivery.py

@@ -41,6 +41,29 @@ def test_triage_submits_each_changed_issue_as_agent_work(tmp_path, monkeypatch):
     ]
     assert "Do not implement code" in submit.call_args_list[0].kwargs["prompt"]
     assert "GITHUB_PERSONAL_ACCESS_TOKEN" in submit.call_args_list[0].kwargs["prompt"]
+    assert (
+        "The following comments and acceptance criteria were added by the "
+        "OpenHands AI agent." in submit.call_args_list[0].kwargs["prompt"]
+    )
+    assert (
+        "closest existing or adjacent implementation"
+        in submit.call_args_list[0].kwargs["prompt"]
+    )
+    assert "Add `ready-for-dev` only after" in submit.call_args_list[0].kwargs["prompt"]
+    assert (
+        "Do not invent acceptance criteria around an arbitrary choice"
+        in submit.call_args_list[0].kwargs["prompt"]
+    )
+    assert (
+        "ask only the focused follow-up questions needed to resolve it"
+        in submit.call_args_list[0].kwargs["prompt"]
+    )
+    prompt = submit.call_args_list[0].kwargs["prompt"]
+    assert "<!-- openhands-ai-triage:start -->" in prompt
+    assert "<!-- openhands-ai-triage:end -->" in prompt
+    assert "Do not post a triage comment" in prompt
+    assert "DECISION NEEDED" in prompt
+    assert "Leave the human-owned issue body unchanged" in prompt
 
 
 def test_triage_continues_after_one_submission_fails(tmp_path, monkeypatch):
@@ -65,7 +88,44 @@ def test_triage_continues_after_one_submission_fails(tmp_path, monkeypatch):
     ]
 
 
-def test_triage_skips_ready_blocked_and_unchanged_issues(tmp_path, monkeypatch):
+def test_triage_event_submits_only_the_named_issue(tmp_path, monkeypatch):
+    issues = [
+        {"number": 1, "title": "First", "body": "A", "labels": []},
+        {"number": 2, "title": "Second", "body": "B", "labels": []},
+    ]
+    _module, run = _triage(tmp_path, monkeypatch, issues)
+    monkeypatch.setenv(
+        "AUTOMATION_EVENT_PAYLOAD",
+        '{"event":{"payload":{"repository":{"full_name":"owner/repo"},'
+        '"issue":{"number":2}}}}',
+    )
+    run.dispatcher.deliver.return_value = {
+        "disposition": "created",
+        "conversation_id": "second",
+    }
+
+    run.run()
+
+    assert run.dispatcher.deliver.call_args.kwargs["subject"] == "9876:issue:2"
+
+
+def test_triage_event_skips_other_repositories(tmp_path, monkeypatch):
+    issues = [{"number": 1, "title": "First", "body": "A", "labels": []}]
+    _module, run = _triage(tmp_path, monkeypatch, issues)
+    monkeypatch.setenv(
+        "AUTOMATION_EVENT_PAYLOAD",
+        '{"event":{"payload":{"repository":{"full_name":"other/repo"},'
+        '"issue":{"number":1}}}}',
+    )
+
+    run.run()
+
+    run.dispatcher.deliver.assert_not_called()
+
+
+def test_triage_checks_ready_issue_but_skips_blocked_and_unchanged_issues(
+    tmp_path, monkeypatch
+):
     issues = [
         {
             "number": 1,
@@ -78,13 +138,147 @@ def test_triage_skips_ready_blocked_and_unchanged_issues(tmp_path, monkeypatch):
     ]
     module, run = _triage(tmp_path, monkeypatch, issues)
     run.dependencies_complete = lambda issue: issue["number"] != 2
-    marker = module.hashlib.sha256(
-        module.json.dumps(["Unchanged", "", []], sort_keys=True).encode()
+    unchanged_marker = module.hashlib.sha256(
+        module.json.dumps(
+            [module.TRIAGE_FORMAT_VERSION, "Unchanged", "", []], sort_keys=True
+        ).encode()
     ).hexdigest()
-    run.gh_pages = lambda path: [{"id": 10, "body": f"<!-- triage-source:{marker} -->"}]
-    submit = Mock()
+    run.gh_pages = lambda path: (
+        [
+            {
+                "id": 10,
+                "body": f"<!-- triage-source:{unchanged_marker} -->",
+            }
+        ]
+        if path == "/issues/3/comments"
+        else []
+    )
+    submit = Mock(return_value={"disposition": "created", "conversation_id": "ready"})
     run.dispatcher.deliver = submit
 
     run.run()
 
-    submit.assert_not_called()
+    assert [call.kwargs["subject"] for call in submit.call_args_list] == [
+        "9876:issue:1"
+    ]
+
+
+def test_triage_passes_human_discussion_and_stale_bot_comment(tmp_path, monkeypatch):
+    issue = {
+        "number": 7,
+        "title": "Improve behavior",
+        "body": "Original request",
+        "labels": [{"name": "enhancement"}],
+    }
+    _module, run = _triage(tmp_path, monkeypatch, [issue])
+    run.gh_pages = lambda path: [
+        {
+            "id": 101,
+            "user": {"login": "maintainer"},
+            "created_at": "2026-09-18T00:00:00Z",
+            "updated_at": "2026-09-18T00:00:00Z",
+            "body": "Keep compatibility with saved configurations.",
+        },
+        {
+            "id": 102,
+            "user": {"login": "all-hands-bot"},
+            "created_at": "2026-09-18T00:01:00Z",
+            "updated_at": "2026-09-18T00:01:00Z",
+            "body": "Old triage\n<!-- triage-source:old -->",
+        },
+    ]
+    run.dispatcher.deliver.return_value = {
+        "disposition": "resumed",
+        "conversation_id": "triage",
+    }
+
+    run.run()
+
+    prompt = run.dispatcher.deliver.call_args.kwargs["prompt"]
+    assert '"author": "maintainer"' in prompt
+    assert "Keep compatibility with saved configurations." in prompt
+    assert '"id": 102' in prompt
+    assert "Old triage" not in prompt
+    assert "Delete prior comments listed as automated triage comments" in prompt
+
+
+def test_triage_ignores_managed_body_when_computing_delivery(tmp_path, monkeypatch):
+    original = "Human request\n\n### Desired Behavior\nKeep the API stable."
+    issue = {
+        "number": 7,
+        "title": "Improve behavior",
+        "body": original,
+        "labels": [{"name": "enhancement"}],
+    }
+    module, run = _triage(tmp_path, monkeypatch, [issue])
+    digest = module.hashlib.sha256(
+        module.json.dumps(
+            [module.TRIAGE_FORMAT_VERSION, "Improve behavior", original, []],
+            sort_keys=True,
+        ).encode()
+    ).hexdigest()
+    issue["body"] += (
+        "\n\n<!-- openhands-ai-triage:start -->\n"
+        "---\n## OpenHands AI triage\n"
+        "### Acceptance Criteria\n- [ ] Preserve behavior\n"
+        f"<!-- triage-source:{digest} -->\n"
+        "<!-- openhands-ai-triage:end -->"
+    )
+
+    run.run()
+
+    run.dispatcher.deliver.assert_not_called()
+    assert module.author_body(issue["body"]) == original
+
+
+def test_triage_reprocesses_legacy_comment_digest(tmp_path, monkeypatch):
+    issue = {"number": 7, "title": "Improve behavior", "body": "Request", "labels": []}
+    module, run = _triage(tmp_path, monkeypatch, [issue])
+    legacy_digest = module.hashlib.sha256(
+        module.json.dumps(["Improve behavior", "Request", []], sort_keys=True).encode()
+    ).hexdigest()
+    run.gh_pages = lambda path: [
+        {"id": 10, "body": f"<!-- triage-source:{legacy_digest} -->"}
+    ]
+    run.dispatcher.deliver.return_value = {
+        "disposition": "created",
+        "conversation_id": "triage",
+    }
+
+    run.run()
+
+    run.dispatcher.deliver.assert_called_once()
+
+
+def test_triage_passes_only_human_owned_body_to_agent(tmp_path, monkeypatch):
+    issue = {
+        "number": 8,
+        "title": "Clarify behavior",
+        "body": (
+            "Original description\n\n"
+            "<!-- openhands-ai-triage:start -->\n"
+            "Old generated criteria\n"
+            "<!-- triage-source:old -->\n"
+            "<!-- openhands-ai-triage:end -->"
+        ),
+        "labels": [{"name": "enhancement"}],
+    }
+    _module, run = _triage(tmp_path, monkeypatch, [issue])
+    run.dispatcher.deliver.return_value = {
+        "disposition": "resumed",
+        "conversation_id": "triage",
+    }
+
+    run.run()
+
+    prompt = run.dispatcher.deliver.call_args.kwargs["prompt"]
+    assert '"author_body": "Original description"' in prompt
+    assert '"managed_triage_section_present": true' in prompt
+    assert "Old generated criteria" not in prompt
+
+
+def test_incomplete_marker_pair_remains_human_owned(tmp_path, monkeypatch):
+    module, _run = _triage(tmp_path, monkeypatch, [])
+    body = "Human text\n<!-- openhands-ai-triage:start -->\nUnclosed text"
+
+    assert module.author_body(body) == body