@openhands/extensions 0.24.0 → 0.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +30 -0
- package/.codex-plugin/marketplace.json +30 -0
- package/.github/pull_request_template.md +17 -2
- package/.github/scripts/check_pr_description.py +64 -5
- package/.github/scripts/markdown_sections.py +95 -0
- package/.github/workflows/build-vscode-app.yml +30 -0
- package/.github/workflows/check-extensions.yml +3 -0
- package/.github/workflows/tests.yml +26 -0
- package/.plugin/marketplace.json +30 -0
- package/.release-please-manifest.json +1 -1
- package/AGENTS.md +3 -2
- package/README.md +10 -3
- package/apps/README.md +20 -0
- package/apps/catalog.json +10 -0
- package/apps/index.d.ts +11 -0
- package/apps/index.js +5 -0
- package/apps/vscode/README.md +13 -0
- package/apps/vscode/canvas-extension.json +19 -0
- package/apps/vscode/extension.js +114 -0
- package/apps/vscode/scripts/package.mjs +84 -0
- package/automations/bundle-index.js +18 -14
- package/automations/catalog/github-agents-md-maintainer/manifest.json +1 -1
- package/automations/catalog/github-pr-reviewer/manifest.json +19 -8
- package/automations/catalog/github-stale-ci-pr-closer/manifest.json +71 -0
- package/automations/catalog/gitlab-issue-to-mr/manifest.json +1 -1
- package/automations/catalog/news-digest/manifest.json +2 -2
- package/automations/catalog-index.js +2 -0
- package/index.d.ts +2 -0
- package/index.js +1 -0
- package/marketplaces/openhands-extensions.json +30 -0
- package/package.json +9 -1
- package/plugins/openhands/skills/openhands-api/README.md +1 -1
- package/plugins/openhands/skills/openhands-api/SKILL.md +3 -3
- package/plugins/openhands/skills/openhands-api/scripts/openhands_api.py +13 -0
- package/plugins/openhands/skills/openhands-automation/SKILL.md +66 -4
- package/plugins/openhands/skills/openhands-automation/references/custom-automation.md +21 -7
- package/plugins/openhands/skills/openhands-sdk/SKILL.md +1 -0
- package/plugins/pr-review/skills/github-pr-review/SKILL.md +12 -2
- package/plugins/qa-changes/skills/github-pr-review/SKILL.md +12 -2
- package/plugins/runtime-api-configs/.claude-plugin/plugin.json +20 -0
- package/plugins/runtime-api-configs/.codex-plugin/plugin.json +20 -0
- package/plugins/runtime-api-configs/.plugin/plugin.json +20 -0
- package/plugins/runtime-api-configs/README.md +59 -0
- package/plugins/runtime-api-configs/SKILL.md +240 -0
- package/plugins/runtime-api-configs/scripts/warm_runtime_configs.py +331 -0
- package/plugins/runtime-api-configs/tests/test_warm_runtime_configs.py +338 -0
- package/plugins/vulnerability-remediation/skills/github/scripts/agent_conversation.py +223 -43
- package/plugins/vulnerability-remediation/skills/github/scripts/github_client.py +183 -15
- package/pyproject.toml +1 -1
- package/python/openhands_extensions/_version.py +1 -1
- package/scripts/check_plugin_resources.py +116 -0
- package/scripts/sync_extensions.py +32 -5
- package/skills/canvas-extension-api/SKILL.md +34 -3
- package/skills/canvas-extension-api/references/backend-safety.md +14 -0
- package/skills/canvas-extension-api/references/connections.md +24 -0
- package/skills/canvas-extension-api/references/sidecar-pattern.md +49 -10
- package/skills/canvas-extension-api/references/v1-contract.md +53 -3
- package/skills/canvas-extension-api/scripts/validate-extension.mjs +52 -0
- package/skills/canvas-extension-api/scripts/validate-extension.test.mjs +192 -1
- package/skills/github/scripts/agent_conversation.py +223 -43
- package/skills/github/scripts/github_client.py +183 -15
- package/skills/github-agents-md-maintainer/README.md +7 -0
- package/skills/github-agents-md-maintainer/SKILL.md +13 -6
- package/skills/github-agents-md-maintainer/scripts/github_client.py +183 -15
- package/skills/github-agents-md-maintainer/scripts/main.py +199 -29
- package/skills/github-delivery-watchdog/scripts/github_client.py +183 -15
- package/skills/github-issue-to-pr/scripts/agent_conversation.py +223 -43
- package/skills/github-issue-to-pr/scripts/github_client.py +183 -15
- package/skills/github-issue-to-pr/scripts/worker.py +10 -1
- package/skills/github-issue-triage/scripts/agent_conversation.py +223 -43
- package/skills/github-issue-triage/scripts/github_client.py +183 -15
- package/skills/github-pr-review/SKILL.md +12 -2
- package/skills/github-pr-reviewer/README.md +36 -0
- package/skills/github-pr-reviewer/SKILL.md +147 -4
- package/skills/github-pr-reviewer/scripts/agent_conversation.py +223 -43
- package/skills/github-pr-reviewer/scripts/github_client.py +183 -15
- package/skills/github-pr-reviewer/scripts/main.py +17 -1
- package/skills/github-pr-reviewer/scripts/worker.py +842 -57
- package/skills/github-repo-monitor/scripts/github_client.py +183 -15
- package/skills/github-repo-monitor/scripts/main.py +32 -1
- package/skills/github-stale-ci-pr-closer/.claude-plugin/plugin.json +9 -0
- package/skills/github-stale-ci-pr-closer/.codex-plugin/plugin.json +9 -0
- package/skills/github-stale-ci-pr-closer/.plugin/plugin.json +9 -0
- package/skills/github-stale-ci-pr-closer/README.md +9 -0
- package/skills/github-stale-ci-pr-closer/SKILL.md +24 -0
- package/skills/github-stale-ci-pr-closer/commands/github-stale-ci-pr-closer.md +8 -0
- package/skills/github-stale-ci-pr-closer/scripts/worker.py +508 -0
- package/skills/github-stale-ci-pr-closer/tests/test_worker.py +353 -0
- package/skills/gitlab-issue-to-mr/README.md +12 -0
- package/skills/gitlab-issue-to-mr/SKILL.md +13 -6
- package/skills/gitlab-issue-to-mr/scripts/main.py +217 -42
- package/skills/incident-retrospective/SKILL.md +5 -5
- package/skills/index.js +34 -16
- package/skills/jira-issue-to-pr/README.md +12 -4
- package/skills/jira-issue-to-pr/SKILL.md +129 -18
- package/skills/jira-issue-to-pr/references/setup.md +8 -0
- package/skills/jira-issue-to-pr/scripts/main.py +181 -54
- package/skills/jira-issue-to-pr/scripts/setup_webhook.py +290 -0
- package/skills/linear-triage/SKILL.md +5 -5
- package/skills/news-digest/README.md +6 -0
- package/skills/news-digest/SKILL.md +19 -9
- package/skills/news-digest/commands/news-digest-setup.md +1 -1
- package/skills/news-digest/scripts/main.py +133 -15
- package/skills/openhands-api/README.md +1 -1
- package/skills/openhands-api/SKILL.md +3 -3
- package/skills/openhands-api/scripts/openhands_api.py +13 -0
- package/skills/openhands-automation/SKILL.md +66 -4
- package/skills/openhands-automation/references/custom-automation.md +21 -7
- package/skills/openhands-sdk/SKILL.md +1 -0
- package/skills/research-brief/SKILL.md +5 -5
- package/skills/slack-standup-digest/SKILL.md +5 -5
- package/skills/ticket-to-code-change/SKILL.md +6 -3
- package/skills/use-jev/.claude-plugin/plugin.json +8 -0
- package/skills/use-jev/.codex-plugin/plugin.json +8 -0
- package/skills/use-jev/.plugin/plugin.json +8 -0
- package/skills/use-jev/README.md +66 -0
- package/skills/use-jev/SKILL.md +66 -0
- package/tests/fixtures/automations/github-pr-reviewer.json +39 -24
- package/tests/test_agent_conversation_dispatch.py +212 -0
- package/tests/test_apps_catalog.py +26 -0
- package/tests/test_automation_setup.py +44 -0
- package/tests/test_check_pr_description.py +89 -6
- package/tests/test_github_automation_foundation.py +158 -0
- package/tests/test_github_repo_monitor.py +107 -0
- package/tests/test_github_reviewer_delivery.py +2247 -6
- package/tests/test_github_stale_ci_pr_closer.py +56 -0
- package/tests/test_jira_issue_to_pr.py +312 -0
- package/tests/test_jira_setup_webhook.py +309 -0
- package/tests/test_openhands_api_python.py +37 -0
- package/tests/test_skill_plugin_loading.py +25 -15
- package/tests/test_sync_extensions.py +102 -0
- package/tests/test_vscode_app.py +33 -0
- package/tests/vscode-app.test.mjs +43 -0
|
@@ -37,6 +37,15 @@ def _reviewer(tmp_path, monkeypatch):
|
|
|
37
37
|
run.token_name = "FACTORY_GITHUB_REVIEWER_TOKEN"
|
|
38
38
|
run.github_login = "all-hands-bot"
|
|
39
39
|
run.dispatcher = Mock()
|
|
40
|
+
# Default to a head with no reported check runs or workflow runs, which the
|
|
41
|
+
# gate reads as green. Gate-specific tests override these.
|
|
42
|
+
run.check_runs = lambda sha: []
|
|
43
|
+
run.workflow_runs = lambda sha: []
|
|
44
|
+
run.statuses = lambda sha: {}
|
|
45
|
+
# Default to no required-check signal so the gate uses its conservative
|
|
46
|
+
# all-run fallback, matching the pre-required-check behavior. Tests that
|
|
47
|
+
# exercise required-only classification override this.
|
|
48
|
+
run.required_check_contexts = lambda number: []
|
|
40
49
|
monkeypatch.delenv("AUTOMATION_EVENT_PAYLOAD", raising=False)
|
|
41
50
|
monkeypatch.setattr(
|
|
42
51
|
module.workflow,
|
|
@@ -46,6 +55,65 @@ def _reviewer(tmp_path, monkeypatch):
|
|
|
46
55
|
return module, run
|
|
47
56
|
|
|
48
57
|
|
|
58
|
+
def _checks(*runs, sha="head-2"):
|
|
59
|
+
"""Check runs as the API reports them: name, status, and conclusion."""
|
|
60
|
+
return [{"name": name, "status": status, "conclusion": conclusion, "head_sha": sha}
|
|
61
|
+
for name, status, conclusion in runs]
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _run(
|
|
65
|
+
name,
|
|
66
|
+
status,
|
|
67
|
+
conclusion,
|
|
68
|
+
*,
|
|
69
|
+
app="github-actions",
|
|
70
|
+
started_at="",
|
|
71
|
+
run_id=0,
|
|
72
|
+
sha="head-2",
|
|
73
|
+
suite_id=0,
|
|
74
|
+
):
|
|
75
|
+
"""One check run with the app identity and ordering fields the API sends."""
|
|
76
|
+
return {
|
|
77
|
+
"name": name,
|
|
78
|
+
"status": status,
|
|
79
|
+
"conclusion": conclusion,
|
|
80
|
+
"head_sha": sha,
|
|
81
|
+
"app": {"slug": app},
|
|
82
|
+
"started_at": started_at,
|
|
83
|
+
"id": run_id,
|
|
84
|
+
"check_suite": {"id": suite_id},
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _workflow_run(
|
|
89
|
+
name,
|
|
90
|
+
status,
|
|
91
|
+
conclusion,
|
|
92
|
+
*,
|
|
93
|
+
run_id=0,
|
|
94
|
+
suite_id=0,
|
|
95
|
+
workflow_id=0,
|
|
96
|
+
sha="head-2",
|
|
97
|
+
started_at="",
|
|
98
|
+
):
|
|
99
|
+
"""One Actions workflow run, as the runs endpoint reports it.
|
|
100
|
+
|
|
101
|
+
`check_suite_id` links the run to its check runs; a suite whose workflows
|
|
102
|
+
failed before any job reported has no check runs under it.
|
|
103
|
+
"""
|
|
104
|
+
return {
|
|
105
|
+
"name": name,
|
|
106
|
+
"status": status,
|
|
107
|
+
"conclusion": conclusion,
|
|
108
|
+
"head_sha": sha,
|
|
109
|
+
"id": run_id,
|
|
110
|
+
"check_suite_id": suite_id,
|
|
111
|
+
"workflow_id": workflow_id,
|
|
112
|
+
"run_started_at": started_at,
|
|
113
|
+
"created_at": started_at,
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
|
|
49
117
|
def _event(monkeypatch, *, action="review_requested", login="all-hands-bot"):
|
|
50
118
|
payload = {
|
|
51
119
|
"action": action,
|
|
@@ -70,7 +138,9 @@ def test_reviewer_submits_each_labeled_exact_head(tmp_path, monkeypatch):
|
|
|
70
138
|
"head": {"sha": "head-2"},
|
|
71
139
|
"labels": [{"name": "openhands-review"}],
|
|
72
140
|
},
|
|
73
|
-
|
|
141
|
+
# A draft is not eligible for the unrequested scan, so it is skipped
|
|
142
|
+
# without a full read and only the labeled head is submitted.
|
|
143
|
+
{"number": 1, "head": {"sha": "head-1"}, "labels": [], "draft": True},
|
|
74
144
|
]
|
|
75
145
|
run.gh_pages = lambda path: prs
|
|
76
146
|
run.gh = Mock(side_effect=[{"id": 99}, prs[0]])
|
|
@@ -96,12 +166,48 @@ def test_reviewer_submits_each_labeled_exact_head(tmp_path, monkeypatch):
|
|
|
96
166
|
assert "GIT_TERMINAL_PROMPT=0" in prompt
|
|
97
167
|
assert "Never paste JSON artifacts" in prompt
|
|
98
168
|
assert "stop immediately" in prompt
|
|
169
|
+
# The prompt carries the dedup rule as well as the worker enforcing it: the
|
|
170
|
+
# agent is the one that reads GitHub before publishing, so it must be told
|
|
171
|
+
# not to publish a second review of a head it already reviewed.
|
|
172
|
+
assert "do NOT publish a second review" in prompt
|
|
173
|
+
assert "currently underway for this head" in prompt
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def test_reviewer_prompt_forbids_republishing_an_unchanged_reviewed_head(
|
|
177
|
+
tmp_path, monkeypatch
|
|
178
|
+
):
|
|
179
|
+
"""The reviewer prompt names the head and the no-clarification condition."""
|
|
180
|
+
module, run = _reviewer(tmp_path, monkeypatch)
|
|
181
|
+
pr = {
|
|
182
|
+
"number": 2,
|
|
183
|
+
"head": {"sha": "head-2"},
|
|
184
|
+
"labels": [{"name": "openhands-review"}],
|
|
185
|
+
}
|
|
186
|
+
run.gh_pages = lambda path: [pr]
|
|
187
|
+
run.gh = Mock(side_effect=[{"id": 99}, pr])
|
|
188
|
+
monkeypatch.setattr(
|
|
189
|
+
module.workflow,
|
|
190
|
+
"_latest_trigger_label_event",
|
|
191
|
+
lambda *args: {"id": 7, "created_at": "now"},
|
|
192
|
+
)
|
|
193
|
+
submit = Mock(
|
|
194
|
+
return_value={"disposition": "created", "conversation_id": "conversation"}
|
|
195
|
+
)
|
|
196
|
+
run.dispatcher.deliver = submit
|
|
197
|
+
|
|
198
|
+
run.run()
|
|
199
|
+
|
|
200
|
+
prompt = submit.call_args.kwargs["prompt"]
|
|
201
|
+
assert "already published a review on `head-2`" in prompt
|
|
202
|
+
assert "no one else has commented since that review" in prompt
|
|
203
|
+
assert "Review the head again only when the head has moved" in prompt
|
|
99
204
|
|
|
100
205
|
|
|
101
|
-
|
|
206
|
+
@pytest.mark.parametrize("draft", [False, True])
|
|
207
|
+
def test_reviewer_submits_requested_exact_head(tmp_path, monkeypatch, draft):
|
|
102
208
|
_module, run = _reviewer(tmp_path, monkeypatch)
|
|
103
209
|
_event(monkeypatch)
|
|
104
|
-
pr = {"number": 2, "head": {"sha": "head-2"}, "labels": []}
|
|
210
|
+
pr = {"number": 2, "head": {"sha": "head-2"}, "labels": [], "draft": draft}
|
|
105
211
|
request = {
|
|
106
212
|
"id": 42,
|
|
107
213
|
"event": "review_requested",
|
|
@@ -304,7 +410,16 @@ def test_reviewer_submitted_review_on_superseded_head_does_not_dispatch(
|
|
|
304
410
|
handoff.assert_not_called()
|
|
305
411
|
|
|
306
412
|
|
|
307
|
-
def
|
|
413
|
+
def test_reviewer_does_not_redeliver_a_label_for_an_already_reviewed_head(
|
|
414
|
+
tmp_path, monkeypatch
|
|
415
|
+
):
|
|
416
|
+
"""A re-applied label on an unchanged, already-reviewed head is not new work.
|
|
417
|
+
|
|
418
|
+
The label event predates the existing review, so `_finish_completed_review`
|
|
419
|
+
would not treat the review as this event's result and the old code dispatched
|
|
420
|
+
a second review of identical code. A review on the current head with nothing
|
|
421
|
+
posted after it is already the answer, whatever trigger arrives next.
|
|
422
|
+
"""
|
|
308
423
|
_module, run = _reviewer(tmp_path, monkeypatch)
|
|
309
424
|
pr = {
|
|
310
425
|
"number": 2,
|
|
@@ -316,6 +431,42 @@ def test_reviewer_redelivers_when_review_predates_latest_label(tmp_path, monkeyp
|
|
|
316
431
|
if path.startswith("/pulls?")
|
|
317
432
|
else _reviews(submitted_at="2025-12-31T00:00:00Z")
|
|
318
433
|
)
|
|
434
|
+
run.gh = Mock(side_effect=[{"id": 99}, pr, pr])
|
|
435
|
+
run.dispatcher.deliver.return_value = {
|
|
436
|
+
"disposition": "continued",
|
|
437
|
+
"conversation_id": "conversation",
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
run.run()
|
|
441
|
+
|
|
442
|
+
run.dispatcher.deliver.assert_not_called()
|
|
443
|
+
|
|
444
|
+
|
|
445
|
+
def test_reviewer_redelivers_a_label_when_a_human_clarified_since(
|
|
446
|
+
tmp_path, monkeypatch
|
|
447
|
+
):
|
|
448
|
+
"""A comment from someone else after the review makes the head reviewable."""
|
|
449
|
+
_module, run = _reviewer(tmp_path, monkeypatch)
|
|
450
|
+
pr = {
|
|
451
|
+
"number": 2,
|
|
452
|
+
"head": {"sha": "head-2"},
|
|
453
|
+
"labels": [{"name": "openhands-review"}],
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
def pages(path):
|
|
457
|
+
if path.startswith("/pulls?"):
|
|
458
|
+
return [pr]
|
|
459
|
+
if path.endswith("/comments"):
|
|
460
|
+
return [
|
|
461
|
+
{
|
|
462
|
+
"body": "I added the missing test you asked for.",
|
|
463
|
+
"created_at": "2026-01-02T00:00:00Z",
|
|
464
|
+
"user": {"login": "neubig"},
|
|
465
|
+
}
|
|
466
|
+
]
|
|
467
|
+
return _reviews(submitted_at="2025-12-31T00:00:00Z")
|
|
468
|
+
|
|
469
|
+
run.gh_pages = pages
|
|
319
470
|
run.gh = Mock(side_effect=[{"id": 99}, pr])
|
|
320
471
|
run.dispatcher.deliver.return_value = {
|
|
321
472
|
"disposition": "continued",
|
|
@@ -326,6 +477,130 @@ def test_reviewer_redelivers_when_review_predates_latest_label(tmp_path, monkeyp
|
|
|
326
477
|
|
|
327
478
|
run.dispatcher.deliver.assert_called_once()
|
|
328
479
|
assert run.dispatcher.deliver.call_args.kwargs["delivery"] == "7:head-2"
|
|
480
|
+
assert run.dispatcher.deliver.call_args.kwargs["head"] == "head-2"
|
|
481
|
+
|
|
482
|
+
|
|
483
|
+
def test_reviewer_does_not_re_review_a_head_after_a_repeat_request(
|
|
484
|
+
tmp_path, monkeypatch
|
|
485
|
+
):
|
|
486
|
+
"""The extensions#698 sequence: repeat requests must not review twice.
|
|
487
|
+
|
|
488
|
+
A review landed on `head-2`; a second review request arrived for the same
|
|
489
|
+
head with no clarifying comment in between. The old trigger-keyed delivery
|
|
490
|
+
started another conversation and published a second review of identical
|
|
491
|
+
code (three APPROVED reviews on one head in production). The head is now
|
|
492
|
+
the guard: an unchanged head with nothing posted after the review is done.
|
|
493
|
+
"""
|
|
494
|
+
_module, run = _reviewer(tmp_path, monkeypatch)
|
|
495
|
+
pr = _requested_pr(2, "head-2")
|
|
496
|
+
run.check_runs = lambda sha: _checks(("ci", "completed", "success"), sha=sha)
|
|
497
|
+
submitted = Mock(
|
|
498
|
+
return_value={"disposition": "created", "conversation_id": "first"}
|
|
499
|
+
)
|
|
500
|
+
run.dispatcher.deliver = submitted
|
|
501
|
+
|
|
502
|
+
# First request: no review exists yet, so the review is dispatched.
|
|
503
|
+
run.gh_pages = _request_pages(pr, _review_request_event(42), [])
|
|
504
|
+
run.gh = Mock(side_effect=[{"id": 99}, pr])
|
|
505
|
+
_event(monkeypatch)
|
|
506
|
+
run.run()
|
|
507
|
+
assert submitted.call_count == 1
|
|
508
|
+
|
|
509
|
+
# Second request for the same head, and the review is now on GitHub with no
|
|
510
|
+
# comment after it: the repeat request starts nothing. The review predates
|
|
511
|
+
# this request (the #698 ordering), so `_finish_completed_review` alone would
|
|
512
|
+
# not recognize it as this request's result and the old code dispatched again.
|
|
513
|
+
reviewed = _reviews(
|
|
514
|
+
sha="head-2", submitted_at="2026-01-02T00:00:00Z"
|
|
515
|
+
)
|
|
516
|
+
later_request = {**_review_request_event(43), "created_at": "2026-01-03T00:00:00Z"}
|
|
517
|
+
|
|
518
|
+
def pages(path):
|
|
519
|
+
if path.startswith("/pulls?"):
|
|
520
|
+
return [pr]
|
|
521
|
+
if path.endswith("/events"):
|
|
522
|
+
return [later_request]
|
|
523
|
+
if path.endswith("/reviews"):
|
|
524
|
+
return reviewed
|
|
525
|
+
return []
|
|
526
|
+
|
|
527
|
+
run.gh_pages = pages
|
|
528
|
+
run.gh = Mock(side_effect=[{"id": 99}, pr, pr])
|
|
529
|
+
_event(monkeypatch)
|
|
530
|
+
run.run()
|
|
531
|
+
|
|
532
|
+
assert submitted.call_count == 1
|
|
533
|
+
|
|
534
|
+
|
|
535
|
+
def test_reviewer_rereviews_a_head_after_a_dismissed_review(
|
|
536
|
+
tmp_path, monkeypatch
|
|
537
|
+
):
|
|
538
|
+
"""A dismissed review does not stand: the head is reviewable again.
|
|
539
|
+
|
|
540
|
+
GitHub keeps a dismissed review in /pulls/{n}/reviews with its commit_id and
|
|
541
|
+
state "DISMISSED". Counting it as a standing review would suppress every
|
|
542
|
+
later review of that head forever, because no other condition clears it. A
|
|
543
|
+
maintainer who dismisses the verdict and re-requests review is asking for a
|
|
544
|
+
fresh look, so the guard must skip dismissed reviews.
|
|
545
|
+
"""
|
|
546
|
+
_module, run = _reviewer(tmp_path, monkeypatch)
|
|
547
|
+
pr = {
|
|
548
|
+
"number": 2,
|
|
549
|
+
"head": {"sha": "head-2"},
|
|
550
|
+
"labels": [{"name": "openhands-review"}],
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
def pages(path):
|
|
554
|
+
if path.startswith("/pulls?"):
|
|
555
|
+
return [pr]
|
|
556
|
+
reviews = _reviews(submitted_at="2025-12-31T00:00:00Z")
|
|
557
|
+
reviews[0]["state"] = "DISMISSED"
|
|
558
|
+
return reviews
|
|
559
|
+
|
|
560
|
+
run.gh_pages = pages
|
|
561
|
+
run.gh = Mock(side_effect=[{"id": 99}, pr])
|
|
562
|
+
run.dispatcher.deliver.return_value = {
|
|
563
|
+
"disposition": "continued",
|
|
564
|
+
"conversation_id": "conversation",
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
run.run()
|
|
568
|
+
|
|
569
|
+
run.dispatcher.deliver.assert_called_once()
|
|
570
|
+
|
|
571
|
+
|
|
572
|
+
def test_reviewer_clears_the_label_when_the_review_predates_the_trigger(
|
|
573
|
+
tmp_path, monkeypatch
|
|
574
|
+
):
|
|
575
|
+
"""The guard must clear the label, not leave a scan re-reading forever.
|
|
576
|
+
|
|
577
|
+
A review predating the re-applied label yields no `completed` entry in
|
|
578
|
+
`_finish_completed_review`, so that call returns before its label DELETE.
|
|
579
|
+
Without clearing the label here the label stays and every later scan re-reads
|
|
580
|
+
this head and re-logs `review-already-published` with no effect.
|
|
581
|
+
"""
|
|
582
|
+
_module, run = _reviewer(tmp_path, monkeypatch)
|
|
583
|
+
pr = {
|
|
584
|
+
"number": 2,
|
|
585
|
+
"head": {"sha": "head-2"},
|
|
586
|
+
"labels": [{"name": "openhands-review"}],
|
|
587
|
+
}
|
|
588
|
+
run.gh_pages = lambda path: (
|
|
589
|
+
[pr]
|
|
590
|
+
if path.startswith("/pulls?")
|
|
591
|
+
else _reviews(submitted_at="2025-12-31T00:00:00Z")
|
|
592
|
+
)
|
|
593
|
+
run.gh = Mock(side_effect=[{"id": 99}, pr, pr])
|
|
594
|
+
|
|
595
|
+
run.run()
|
|
596
|
+
|
|
597
|
+
run.dispatcher.deliver.assert_not_called()
|
|
598
|
+
deleted = [
|
|
599
|
+
call.args[1]
|
|
600
|
+
for call in run.gh.call_args_list
|
|
601
|
+
if call.args[0] == "DELETE"
|
|
602
|
+
]
|
|
603
|
+
assert "/issues/2/labels/openhands-review" in deleted
|
|
329
604
|
|
|
330
605
|
|
|
331
606
|
def test_reviewer_does_not_trust_another_reviewers_verdict(tmp_path, monkeypatch):
|
|
@@ -349,9 +624,12 @@ def test_reviewer_does_not_trust_another_reviewers_verdict(tmp_path, monkeypatch
|
|
|
349
624
|
run.dispatcher.deliver.assert_called_once()
|
|
350
625
|
|
|
351
626
|
|
|
352
|
-
def
|
|
627
|
+
def test_reviewer_ignores_a_draft_unlabeled_unrequested_pr(tmp_path, monkeypatch):
|
|
628
|
+
"""The unrequested scan skips a draft: it is not reviewable yet."""
|
|
353
629
|
_module, run = _reviewer(tmp_path, monkeypatch)
|
|
354
|
-
run.gh_pages = lambda path: [
|
|
630
|
+
run.gh_pages = lambda path: [
|
|
631
|
+
{"number": 1, "labels": [], "head": {"sha": "head"}, "draft": True}
|
|
632
|
+
]
|
|
355
633
|
run.gh = Mock(return_value={"id": 99})
|
|
356
634
|
submit = Mock()
|
|
357
635
|
run.dispatcher.deliver = submit
|
|
@@ -359,6 +637,8 @@ def test_reviewer_ignores_unlabeled_prs(tmp_path, monkeypatch):
|
|
|
359
637
|
run.run()
|
|
360
638
|
|
|
361
639
|
submit.assert_not_called()
|
|
640
|
+
# Skipped before the full PR read, so only the repository-id lookup ran.
|
|
641
|
+
assert run.gh.call_count == 1
|
|
362
642
|
|
|
363
643
|
|
|
364
644
|
def test_reviewer_continues_after_one_submission_fails_then_reports_run_failure(
|
|
@@ -551,3 +831,1964 @@ def test_reviewer_hands_scope_stop_to_maintainer(tmp_path, monkeypatch):
|
|
|
551
831
|
"/issues/2/labels/openhands-review",
|
|
552
832
|
)
|
|
553
833
|
run.dispatcher.deliver.assert_not_called()
|
|
834
|
+
|
|
835
|
+
|
|
836
|
+
# --------------------------------------------------------------------------- #
|
|
837
|
+
# Head-eligibility gate: a deterministically blocked head spends no agent work.
|
|
838
|
+
# --------------------------------------------------------------------------- #
|
|
839
|
+
|
|
840
|
+
|
|
841
|
+
def _labeled_pr(number=2, sha="head-2"):
|
|
842
|
+
return {
|
|
843
|
+
"number": number,
|
|
844
|
+
"head": {"sha": sha},
|
|
845
|
+
"labels": [{"name": "openhands-review"}],
|
|
846
|
+
}
|
|
847
|
+
|
|
848
|
+
|
|
849
|
+
def _comment(body, login="all-hands-bot", comment_id=900):
|
|
850
|
+
return {
|
|
851
|
+
"id": comment_id,
|
|
852
|
+
"body": body,
|
|
853
|
+
"user": {"login": login},
|
|
854
|
+
}
|
|
855
|
+
|
|
856
|
+
|
|
857
|
+
def _gate_pages(pr, comments):
|
|
858
|
+
return lambda path: (
|
|
859
|
+
[pr] if path.startswith("/pulls?") else comments
|
|
860
|
+
)
|
|
861
|
+
|
|
862
|
+
|
|
863
|
+
def _gate_comment_calls(run, number=2):
|
|
864
|
+
return [
|
|
865
|
+
call
|
|
866
|
+
for call in run.gh.call_args_list
|
|
867
|
+
if call.args[1] == f"/issues/{number}/comments"
|
|
868
|
+
]
|
|
869
|
+
|
|
870
|
+
|
|
871
|
+
def test_reviewer_blocking_current_head_check_creates_no_conversation(
|
|
872
|
+
tmp_path, monkeypatch
|
|
873
|
+
):
|
|
874
|
+
_module, run = _reviewer(tmp_path, monkeypatch)
|
|
875
|
+
pr = _labeled_pr()
|
|
876
|
+
run.gh_pages = _gate_pages(pr, [])
|
|
877
|
+
run.gh = Mock(side_effect=[{"id": 99}, pr, {"id": 1234}])
|
|
878
|
+
run.check_runs = lambda sha: _checks(
|
|
879
|
+
("Validate PR description", "completed", "failure"), sha=sha
|
|
880
|
+
)
|
|
881
|
+
|
|
882
|
+
run.run()
|
|
883
|
+
|
|
884
|
+
run.dispatcher.deliver.assert_not_called()
|
|
885
|
+
posted = [call for call in _gate_comment_calls(run) if call.args[0] == "POST"]
|
|
886
|
+
assert len(posted) == 1
|
|
887
|
+
body = posted[0].args[2]["body"]
|
|
888
|
+
assert "<!-- openhands-review-gate:blocked:head-2 -->" in body
|
|
889
|
+
assert "`Validate PR description`" in body
|
|
890
|
+
assert "no AI was used to generate this comment" in body
|
|
891
|
+
|
|
892
|
+
|
|
893
|
+
def test_reviewer_green_current_head_creates_exactly_one_conversation(
|
|
894
|
+
tmp_path, monkeypatch
|
|
895
|
+
):
|
|
896
|
+
_module, run = _reviewer(tmp_path, monkeypatch)
|
|
897
|
+
pr = _labeled_pr()
|
|
898
|
+
run.gh_pages = _gate_pages(pr, [])
|
|
899
|
+
run.gh = Mock(side_effect=[{"id": 99}, pr])
|
|
900
|
+
run.check_runs = lambda sha: _checks(
|
|
901
|
+
("Validate PR description", "completed", "success"),
|
|
902
|
+
("lint", "completed", "skipped"),
|
|
903
|
+
("optional", "completed", "neutral"),
|
|
904
|
+
sha=sha,
|
|
905
|
+
)
|
|
906
|
+
run.dispatcher.deliver.return_value = {
|
|
907
|
+
"disposition": "created",
|
|
908
|
+
"conversation_id": "conversation",
|
|
909
|
+
}
|
|
910
|
+
|
|
911
|
+
run.run()
|
|
912
|
+
|
|
913
|
+
run.dispatcher.deliver.assert_called_once()
|
|
914
|
+
assert run.dispatcher.deliver.call_args.kwargs["delivery"] == "7:head-2"
|
|
915
|
+
|
|
916
|
+
|
|
917
|
+
def test_reviewer_pending_current_head_check_waits_without_a_conversation(
|
|
918
|
+
tmp_path, monkeypatch
|
|
919
|
+
):
|
|
920
|
+
_module, run = _reviewer(tmp_path, monkeypatch)
|
|
921
|
+
pr = _labeled_pr()
|
|
922
|
+
run.gh_pages = _gate_pages(pr, [])
|
|
923
|
+
run.gh = Mock(side_effect=[{"id": 99}, pr, {"id": 1234}])
|
|
924
|
+
run.check_runs = lambda sha: _checks(
|
|
925
|
+
("ci", "completed", "success"),
|
|
926
|
+
("slow-e2e", "in_progress", None),
|
|
927
|
+
sha=sha,
|
|
928
|
+
)
|
|
929
|
+
|
|
930
|
+
run.run()
|
|
931
|
+
|
|
932
|
+
run.dispatcher.deliver.assert_not_called()
|
|
933
|
+
posted = [call for call in _gate_comment_calls(run) if call.args[0] == "POST"]
|
|
934
|
+
assert len(posted) == 1
|
|
935
|
+
body = posted[0].args[2]["body"]
|
|
936
|
+
assert "<!-- openhands-review-gate:waiting:head-2 -->" in body
|
|
937
|
+
assert "`slow-e2e`" in body
|
|
938
|
+
assert "waiting on checks" in body
|
|
939
|
+
|
|
940
|
+
|
|
941
|
+
def test_reviewer_ignores_checks_from_an_obsolete_head(tmp_path, monkeypatch):
|
|
942
|
+
_module, run = _reviewer(tmp_path, monkeypatch)
|
|
943
|
+
pr = _labeled_pr()
|
|
944
|
+
run.gh_pages = _gate_pages(pr, [])
|
|
945
|
+
run.gh = Mock(side_effect=[{"id": 99}, pr])
|
|
946
|
+
run.check_runs = lambda sha: _checks(
|
|
947
|
+
("Validate PR description", "completed", "failure"), sha="head-1"
|
|
948
|
+
)
|
|
949
|
+
run.dispatcher.deliver.return_value = {
|
|
950
|
+
"disposition": "created",
|
|
951
|
+
"conversation_id": "conversation",
|
|
952
|
+
}
|
|
953
|
+
|
|
954
|
+
run.run()
|
|
955
|
+
|
|
956
|
+
run.dispatcher.deliver.assert_called_once()
|
|
957
|
+
assert _gate_comment_calls(run) == []
|
|
958
|
+
|
|
959
|
+
|
|
960
|
+
def test_reviewer_blocks_a_workflow_that_failed_with_no_check_runs(
|
|
961
|
+
tmp_path, monkeypatch
|
|
962
|
+
):
|
|
963
|
+
"""The #426 case: green check-run rollup, but a workflow failed before jobs.
|
|
964
|
+
|
|
965
|
+
`Tests`, `Check Extensions`, and `Deprecation deadlines` failed with zero
|
|
966
|
+
jobs, so they contributed no check runs and the commit's rollup was a green
|
|
967
|
+
`pr-title`. Reading only check runs launched the reviewer for red CI.
|
|
968
|
+
"""
|
|
969
|
+
_module, run = _reviewer(tmp_path, monkeypatch)
|
|
970
|
+
pr = _labeled_pr()
|
|
971
|
+
run.gh_pages = _gate_pages(pr, [])
|
|
972
|
+
run.gh = Mock(side_effect=[{"id": 99}, pr, {"id": 1234}])
|
|
973
|
+
run.check_runs = lambda sha: _checks(
|
|
974
|
+
("pr-title / Lint PR title (conventional)", "completed", "success"), sha=sha
|
|
975
|
+
)
|
|
976
|
+
run.workflow_runs = lambda sha: [
|
|
977
|
+
_workflow_run(
|
|
978
|
+
"Tests",
|
|
979
|
+
"completed",
|
|
980
|
+
"failure",
|
|
981
|
+
run_id=3,
|
|
982
|
+
suite_id=3003,
|
|
983
|
+
workflow_id=236324519,
|
|
984
|
+
sha=sha,
|
|
985
|
+
),
|
|
986
|
+
_workflow_run(
|
|
987
|
+
"Check Extensions",
|
|
988
|
+
"completed",
|
|
989
|
+
"failure",
|
|
990
|
+
run_id=2,
|
|
991
|
+
suite_id=3002,
|
|
992
|
+
workflow_id=260888338,
|
|
993
|
+
sha=sha,
|
|
994
|
+
),
|
|
995
|
+
_workflow_run(
|
|
996
|
+
"Deprecation deadlines",
|
|
997
|
+
"completed",
|
|
998
|
+
"failure",
|
|
999
|
+
run_id=1,
|
|
1000
|
+
suite_id=3001,
|
|
1001
|
+
workflow_id=311570275,
|
|
1002
|
+
sha=sha,
|
|
1003
|
+
),
|
|
1004
|
+
]
|
|
1005
|
+
|
|
1006
|
+
run.run()
|
|
1007
|
+
|
|
1008
|
+
run.dispatcher.deliver.assert_not_called()
|
|
1009
|
+
posted = [call for call in _gate_comment_calls(run) if call.args[0] == "POST"]
|
|
1010
|
+
assert len(posted) == 1
|
|
1011
|
+
body = posted[0].args[2]["body"]
|
|
1012
|
+
assert "<!-- openhands-review-gate:blocked:head-2 -->" in body
|
|
1013
|
+
assert "`Tests`" in body
|
|
1014
|
+
assert "`Check Extensions`" in body
|
|
1015
|
+
assert "`Deprecation deadlines`" in body
|
|
1016
|
+
|
|
1017
|
+
|
|
1018
|
+
def test_reviewer_does_not_double_report_a_workflow_that_has_check_runs(
|
|
1019
|
+
tmp_path, monkeypatch
|
|
1020
|
+
):
|
|
1021
|
+
"""A workflow whose suite already has check runs is not reported twice."""
|
|
1022
|
+
_module, run = _reviewer(tmp_path, monkeypatch)
|
|
1023
|
+
pr = _labeled_pr()
|
|
1024
|
+
run.gh_pages = _gate_pages(pr, [])
|
|
1025
|
+
run.gh = Mock(side_effect=[{"id": 99}, pr])
|
|
1026
|
+
# The suite the workflow run points at is the one that reported the check,
|
|
1027
|
+
# so the workflow run is redundant and must not be reported a second time.
|
|
1028
|
+
run.check_runs = lambda sha: [
|
|
1029
|
+
_run(
|
|
1030
|
+
"Validate PR description",
|
|
1031
|
+
"completed",
|
|
1032
|
+
"success",
|
|
1033
|
+
started_at="2026-09-22T13:00:00Z",
|
|
1034
|
+
run_id=1,
|
|
1035
|
+
sha=sha,
|
|
1036
|
+
suite_id=900,
|
|
1037
|
+
)
|
|
1038
|
+
]
|
|
1039
|
+
run.workflow_runs = lambda sha: [
|
|
1040
|
+
_workflow_run(
|
|
1041
|
+
"PR Description Check",
|
|
1042
|
+
"completed",
|
|
1043
|
+
"success",
|
|
1044
|
+
run_id=5,
|
|
1045
|
+
suite_id=900,
|
|
1046
|
+
workflow_id=341671185,
|
|
1047
|
+
sha=sha,
|
|
1048
|
+
)
|
|
1049
|
+
]
|
|
1050
|
+
run.dispatcher.deliver.return_value = {
|
|
1051
|
+
"disposition": "created",
|
|
1052
|
+
"conversation_id": "conversation",
|
|
1053
|
+
}
|
|
1054
|
+
|
|
1055
|
+
run.run()
|
|
1056
|
+
|
|
1057
|
+
run.dispatcher.deliver.assert_called_once()
|
|
1058
|
+
assert _gate_comment_calls(run) == []
|
|
1059
|
+
|
|
1060
|
+
|
|
1061
|
+
def test_reviewer_waits_for_a_workflow_run_that_has_not_finished(
|
|
1062
|
+
tmp_path, monkeypatch
|
|
1063
|
+
):
|
|
1064
|
+
"""A current-head workflow run still in progress makes the head wait."""
|
|
1065
|
+
_module, run = _reviewer(tmp_path, monkeypatch)
|
|
1066
|
+
pr = _labeled_pr()
|
|
1067
|
+
run.gh_pages = _gate_pages(pr, [])
|
|
1068
|
+
run.gh = Mock(side_effect=[{"id": 99}, pr, {"id": 1234}])
|
|
1069
|
+
run.workflow_runs = lambda sha: [
|
|
1070
|
+
_workflow_run(
|
|
1071
|
+
"Tests",
|
|
1072
|
+
"in_progress",
|
|
1073
|
+
None,
|
|
1074
|
+
run_id=4,
|
|
1075
|
+
suite_id=4004,
|
|
1076
|
+
workflow_id=236324519,
|
|
1077
|
+
sha=sha,
|
|
1078
|
+
)
|
|
1079
|
+
]
|
|
1080
|
+
|
|
1081
|
+
run.run()
|
|
1082
|
+
|
|
1083
|
+
run.dispatcher.deliver.assert_not_called()
|
|
1084
|
+
posted = [call for call in _gate_comment_calls(run) if call.args[0] == "POST"]
|
|
1085
|
+
assert len(posted) == 1
|
|
1086
|
+
body = posted[0].args[2]["body"]
|
|
1087
|
+
assert "<!-- openhands-review-gate:waiting:head-2 -->" in body
|
|
1088
|
+
assert "`Tests`" in body
|
|
1089
|
+
|
|
1090
|
+
|
|
1091
|
+
def test_reviewer_ignores_workflow_runs_from_an_obsolete_head(
|
|
1092
|
+
tmp_path, monkeypatch
|
|
1093
|
+
):
|
|
1094
|
+
"""A failed workflow on an earlier head must not block the push that fixed it."""
|
|
1095
|
+
_module, run = _reviewer(tmp_path, monkeypatch)
|
|
1096
|
+
pr = _labeled_pr()
|
|
1097
|
+
run.gh_pages = _gate_pages(pr, [])
|
|
1098
|
+
run.gh = Mock(side_effect=[{"id": 99}, pr])
|
|
1099
|
+
run.workflow_runs = lambda sha: [
|
|
1100
|
+
_workflow_run(
|
|
1101
|
+
"Tests",
|
|
1102
|
+
"completed",
|
|
1103
|
+
"failure",
|
|
1104
|
+
run_id=1,
|
|
1105
|
+
suite_id=3001,
|
|
1106
|
+
workflow_id=236324519,
|
|
1107
|
+
sha="head-1",
|
|
1108
|
+
)
|
|
1109
|
+
]
|
|
1110
|
+
run.dispatcher.deliver.return_value = {
|
|
1111
|
+
"disposition": "created",
|
|
1112
|
+
"conversation_id": "conversation",
|
|
1113
|
+
}
|
|
1114
|
+
|
|
1115
|
+
run.run()
|
|
1116
|
+
|
|
1117
|
+
run.dispatcher.deliver.assert_called_once()
|
|
1118
|
+
assert _gate_comment_calls(run) == []
|
|
1119
|
+
|
|
1120
|
+
|
|
1121
|
+
def test_reviewer_lets_a_green_workflow_rerun_supersede_a_failed_one(
|
|
1122
|
+
tmp_path, monkeypatch
|
|
1123
|
+
):
|
|
1124
|
+
"""The latest run of a workflow decides, so a successful re-run clears it."""
|
|
1125
|
+
_module, run = _reviewer(tmp_path, monkeypatch)
|
|
1126
|
+
pr = _labeled_pr()
|
|
1127
|
+
run.gh_pages = _gate_pages(pr, [])
|
|
1128
|
+
run.gh = Mock(side_effect=[{"id": 99}, pr])
|
|
1129
|
+
run.workflow_runs = lambda sha: [
|
|
1130
|
+
_workflow_run(
|
|
1131
|
+
"Tests",
|
|
1132
|
+
"completed",
|
|
1133
|
+
"failure",
|
|
1134
|
+
run_id=1,
|
|
1135
|
+
suite_id=3001,
|
|
1136
|
+
workflow_id=236324519,
|
|
1137
|
+
sha=sha,
|
|
1138
|
+
started_at="2026-09-22T13:00:00Z",
|
|
1139
|
+
),
|
|
1140
|
+
_workflow_run(
|
|
1141
|
+
"Tests",
|
|
1142
|
+
"completed",
|
|
1143
|
+
"success",
|
|
1144
|
+
run_id=2,
|
|
1145
|
+
suite_id=3002,
|
|
1146
|
+
workflow_id=236324519,
|
|
1147
|
+
sha=sha,
|
|
1148
|
+
started_at="2026-09-22T13:10:00Z",
|
|
1149
|
+
),
|
|
1150
|
+
]
|
|
1151
|
+
run.dispatcher.deliver.return_value = {
|
|
1152
|
+
"disposition": "created",
|
|
1153
|
+
"conversation_id": "conversation",
|
|
1154
|
+
}
|
|
1155
|
+
|
|
1156
|
+
run.run()
|
|
1157
|
+
|
|
1158
|
+
run.dispatcher.deliver.assert_called_once()
|
|
1159
|
+
assert _gate_comment_calls(run) == []
|
|
1160
|
+
|
|
1161
|
+
|
|
1162
|
+
def test_reviewer_ignores_an_optional_zero_job_workflow_on_a_green_required_head(
|
|
1163
|
+
tmp_path, monkeypatch
|
|
1164
|
+
):
|
|
1165
|
+
"""The #17200 case: the only required check passes, an optional workflow does not.
|
|
1166
|
+
|
|
1167
|
+
OpenHands/OpenHands#17200 passes `test-and-build (ubuntu)`, its only
|
|
1168
|
+
required check, while the optional `release ready` workflow has a zero-job
|
|
1169
|
+
`startup_failure`. Scheduled discovery must classify on the required check
|
|
1170
|
+
and dispatch, not block on the optional workflow.
|
|
1171
|
+
"""
|
|
1172
|
+
_module, run = _reviewer(tmp_path, monkeypatch)
|
|
1173
|
+
pr = _labeled_pr()
|
|
1174
|
+
run.gh_pages = _gate_pages(pr, [])
|
|
1175
|
+
run.gh = Mock(side_effect=[{"id": 99}, pr])
|
|
1176
|
+
run.required_check_contexts = lambda number: [
|
|
1177
|
+
{"name": "test-and-build (ubuntu)", "kind": "CheckRun"}
|
|
1178
|
+
]
|
|
1179
|
+
run.check_runs = lambda sha: _checks(
|
|
1180
|
+
("test-and-build (ubuntu)", "completed", "success"), sha=sha
|
|
1181
|
+
)
|
|
1182
|
+
run.workflow_runs = lambda sha: [
|
|
1183
|
+
_workflow_run(
|
|
1184
|
+
"release ready",
|
|
1185
|
+
"completed",
|
|
1186
|
+
"startup_failure",
|
|
1187
|
+
run_id=1,
|
|
1188
|
+
suite_id=1001,
|
|
1189
|
+
workflow_id=305675242,
|
|
1190
|
+
sha=sha,
|
|
1191
|
+
)
|
|
1192
|
+
]
|
|
1193
|
+
run.dispatcher.deliver.return_value = {
|
|
1194
|
+
"disposition": "created",
|
|
1195
|
+
"conversation_id": "conversation",
|
|
1196
|
+
}
|
|
1197
|
+
|
|
1198
|
+
run.run()
|
|
1199
|
+
|
|
1200
|
+
run.dispatcher.deliver.assert_called_once()
|
|
1201
|
+
assert _gate_comment_calls(run) == []
|
|
1202
|
+
|
|
1203
|
+
|
|
1204
|
+
def test_reviewer_blocks_when_a_required_check_fails(tmp_path, monkeypatch):
|
|
1205
|
+
"""A failed required check still blocks scheduled discovery."""
|
|
1206
|
+
_module, run = _reviewer(tmp_path, monkeypatch)
|
|
1207
|
+
pr = _labeled_pr()
|
|
1208
|
+
run.gh_pages = _gate_pages(pr, [])
|
|
1209
|
+
run.gh = Mock(side_effect=[{"id": 99}, pr, {"id": 1234}])
|
|
1210
|
+
run.required_check_contexts = lambda number: [
|
|
1211
|
+
{"name": "test-and-build (ubuntu)", "kind": "CheckRun"}
|
|
1212
|
+
]
|
|
1213
|
+
run.check_runs = lambda sha: _checks(
|
|
1214
|
+
("test-and-build (ubuntu)", "completed", "failure"),
|
|
1215
|
+
("optional", "completed", "startup_failure"),
|
|
1216
|
+
sha=sha,
|
|
1217
|
+
)
|
|
1218
|
+
|
|
1219
|
+
run.run()
|
|
1220
|
+
|
|
1221
|
+
run.dispatcher.deliver.assert_not_called()
|
|
1222
|
+
posted = [call for call in _gate_comment_calls(run) if call.args[0] == "POST"]
|
|
1223
|
+
assert len(posted) == 1
|
|
1224
|
+
body = posted[0].args[2]["body"]
|
|
1225
|
+
assert "<!-- openhands-review-gate:blocked:head-2 -->" in body
|
|
1226
|
+
assert "`test-and-build (ubuntu)`" in body
|
|
1227
|
+
assert "`optional`" not in body
|
|
1228
|
+
|
|
1229
|
+
|
|
1230
|
+
def test_reviewer_waits_for_a_pending_required_check(tmp_path, monkeypatch):
|
|
1231
|
+
"""A required check that has not finished makes scheduled discovery wait."""
|
|
1232
|
+
_module, run = _reviewer(tmp_path, monkeypatch)
|
|
1233
|
+
pr = _labeled_pr()
|
|
1234
|
+
run.gh_pages = _gate_pages(pr, [])
|
|
1235
|
+
run.gh = Mock(side_effect=[{"id": 99}, pr, {"id": 1234}])
|
|
1236
|
+
run.required_check_contexts = lambda number: [
|
|
1237
|
+
{"name": "test-and-build (ubuntu)", "kind": "CheckRun"}
|
|
1238
|
+
]
|
|
1239
|
+
run.check_runs = lambda sha: _checks(
|
|
1240
|
+
("test-and-build (ubuntu)", "in_progress", None), sha=sha
|
|
1241
|
+
)
|
|
1242
|
+
|
|
1243
|
+
run.run()
|
|
1244
|
+
|
|
1245
|
+
run.dispatcher.deliver.assert_not_called()
|
|
1246
|
+
posted = [call for call in _gate_comment_calls(run) if call.args[0] == "POST"]
|
|
1247
|
+
assert len(posted) == 1
|
|
1248
|
+
body = posted[0].args[2]["body"]
|
|
1249
|
+
assert "<!-- openhands-review-gate:waiting:head-2 -->" in body
|
|
1250
|
+
assert "`test-and-build (ubuntu)`" in body
|
|
1251
|
+
|
|
1252
|
+
|
|
1253
|
+
def test_reviewer_waits_for_a_required_context_with_no_current_head_run(
|
|
1254
|
+
tmp_path, monkeypatch
|
|
1255
|
+
):
|
|
1256
|
+
"""A required name that reported no check run is expected, so the head waits.
|
|
1257
|
+
|
|
1258
|
+
A required context can appear without a matching current-head check run (a
|
|
1259
|
+
required commit status, or a re-run whose name now resolves elsewhere).
|
|
1260
|
+
Unfulfilled is never approval.
|
|
1261
|
+
"""
|
|
1262
|
+
_module, run = _reviewer(tmp_path, monkeypatch)
|
|
1263
|
+
pr = _labeled_pr()
|
|
1264
|
+
run.gh_pages = _gate_pages(pr, [])
|
|
1265
|
+
run.gh = Mock(side_effect=[{"id": 99}, pr, {"id": 1234}])
|
|
1266
|
+
run.required_check_contexts = lambda number: [
|
|
1267
|
+
{"name": "test", "kind": "CheckRun"},
|
|
1268
|
+
]
|
|
1269
|
+
run.check_runs = lambda sha: []
|
|
1270
|
+
|
|
1271
|
+
run.run()
|
|
1272
|
+
|
|
1273
|
+
run.dispatcher.deliver.assert_not_called()
|
|
1274
|
+
posted = [call for call in _gate_comment_calls(run) if call.args[0] == "POST"]
|
|
1275
|
+
assert len(posted) == 1
|
|
1276
|
+
body = posted[0].args[2]["body"]
|
|
1277
|
+
assert "<!-- openhands-review-gate:waiting:head-2 -->" in body
|
|
1278
|
+
assert "`test`" in body
|
|
1279
|
+
|
|
1280
|
+
|
|
1281
|
+
def test_reviewer_falls_back_to_all_checks_when_required_reported_nothing(
|
|
1282
|
+
tmp_path, monkeypatch
|
|
1283
|
+
):
|
|
1284
|
+
"""The #426 shape through the required path: required checks never reported.
|
|
1285
|
+
|
|
1286
|
+
`isRequired` is a field on each context in the head's rollup, so a required
|
|
1287
|
+
check that failed before creating any check run has no node and the required
|
|
1288
|
+
set comes back empty. An empty set must fall back to the full current-head
|
|
1289
|
+
rollup rather than treating the head as green.
|
|
1290
|
+
"""
|
|
1291
|
+
_module, run = _reviewer(tmp_path, monkeypatch)
|
|
1292
|
+
pr = _labeled_pr()
|
|
1293
|
+
run.gh_pages = _gate_pages(pr, [])
|
|
1294
|
+
run.gh = Mock(side_effect=[{"id": 99}, pr, {"id": 1234}])
|
|
1295
|
+
run.required_check_contexts = lambda number: []
|
|
1296
|
+
run.check_runs = lambda sha: _checks(
|
|
1297
|
+
("pr-title / Lint PR title (conventional)", "completed", "success"), sha=sha
|
|
1298
|
+
)
|
|
1299
|
+
run.workflow_runs = lambda sha: [
|
|
1300
|
+
_workflow_run(
|
|
1301
|
+
"Tests",
|
|
1302
|
+
"completed",
|
|
1303
|
+
"failure",
|
|
1304
|
+
run_id=1,
|
|
1305
|
+
suite_id=3001,
|
|
1306
|
+
workflow_id=236324519,
|
|
1307
|
+
sha=sha,
|
|
1308
|
+
)
|
|
1309
|
+
]
|
|
1310
|
+
|
|
1311
|
+
run.run()
|
|
1312
|
+
|
|
1313
|
+
run.dispatcher.deliver.assert_not_called()
|
|
1314
|
+
posted = [call for call in _gate_comment_calls(run) if call.args[0] == "POST"]
|
|
1315
|
+
assert len(posted) == 1
|
|
1316
|
+
body = posted[0].args[2]["body"]
|
|
1317
|
+
assert "<!-- openhands-review-gate:blocked:head-2 -->" in body
|
|
1318
|
+
assert "`Tests`" in body
|
|
1319
|
+
|
|
1320
|
+
|
|
1321
|
+
def test_reviewer_ignores_an_optional_failure_but_reads_required_statuses(
|
|
1322
|
+
tmp_path, monkeypatch
|
|
1323
|
+
):
|
|
1324
|
+
"""A required commit status is classified alongside required check runs."""
|
|
1325
|
+
_module, run = _reviewer(tmp_path, monkeypatch)
|
|
1326
|
+
pr = _labeled_pr()
|
|
1327
|
+
run.gh_pages = _gate_pages(pr, [])
|
|
1328
|
+
run.gh = Mock(side_effect=[{"id": 99}, pr, {"id": 1234}])
|
|
1329
|
+
run.required_check_contexts = lambda number: [
|
|
1330
|
+
{"name": "test-and-build (ubuntu)", "kind": "CheckRun"},
|
|
1331
|
+
{"name": "ci/required", "kind": "StatusContext"},
|
|
1332
|
+
]
|
|
1333
|
+
run.check_runs = lambda sha: _checks(
|
|
1334
|
+
("test-and-build (ubuntu)", "completed", "success"), sha=sha
|
|
1335
|
+
)
|
|
1336
|
+
run.statuses = lambda sha: {"ci/required": "failure"}
|
|
1337
|
+
|
|
1338
|
+
run.run()
|
|
1339
|
+
|
|
1340
|
+
run.dispatcher.deliver.assert_not_called()
|
|
1341
|
+
posted = [call for call in _gate_comment_calls(run) if call.args[0] == "POST"]
|
|
1342
|
+
assert len(posted) == 1
|
|
1343
|
+
assert "`ci/required`" in posted[0].args[2]["body"]
|
|
1344
|
+
|
|
1345
|
+
|
|
1346
|
+
def test_reviewer_falls_back_to_every_check_when_required_signal_is_unavailable(
|
|
1347
|
+
tmp_path, monkeypatch
|
|
1348
|
+
):
|
|
1349
|
+
"""A failed required-check read must not approve silently."""
|
|
1350
|
+
|
|
1351
|
+
_module, run = _reviewer(tmp_path, monkeypatch)
|
|
1352
|
+
pr = _labeled_pr()
|
|
1353
|
+
run.gh_pages = _gate_pages(pr, [])
|
|
1354
|
+
run.gh = Mock(side_effect=[{"id": 99}, pr, {"id": 1234}])
|
|
1355
|
+
|
|
1356
|
+
def unavailable(number):
|
|
1357
|
+
raise RuntimeError("GitHub required-check query failed")
|
|
1358
|
+
|
|
1359
|
+
run.required_check_contexts = unavailable
|
|
1360
|
+
run.check_runs = lambda sha: _checks(("ci", "completed", "failure"), sha=sha)
|
|
1361
|
+
|
|
1362
|
+
run.run()
|
|
1363
|
+
|
|
1364
|
+
run.dispatcher.deliver.assert_not_called()
|
|
1365
|
+
posted = [call for call in _gate_comment_calls(run) if call.args[0] == "POST"]
|
|
1366
|
+
assert len(posted) == 1
|
|
1367
|
+
assert "<!-- openhands-review-gate:blocked:head-2 -->" in posted[0].args[2]["body"]
|
|
1368
|
+
|
|
1369
|
+
|
|
1370
|
+
def test_reviewer_fails_closed_on_an_unknown_conclusion(tmp_path, monkeypatch):
|
|
1371
|
+
_module, run = _reviewer(tmp_path, monkeypatch)
|
|
1372
|
+
pr = _labeled_pr()
|
|
1373
|
+
run.gh_pages = _gate_pages(pr, [])
|
|
1374
|
+
run.gh = Mock(side_effect=[{"id": 99}, pr, {"id": 1234}])
|
|
1375
|
+
run.check_runs = lambda sha: _checks(("mystery", "completed", "action_required"), sha=sha)
|
|
1376
|
+
|
|
1377
|
+
run.run()
|
|
1378
|
+
|
|
1379
|
+
run.dispatcher.deliver.assert_not_called()
|
|
1380
|
+
body = [
|
|
1381
|
+
call.args[2]["body"]
|
|
1382
|
+
for call in _gate_comment_calls(run)
|
|
1383
|
+
if call.args[0] == "POST"
|
|
1384
|
+
][0]
|
|
1385
|
+
assert "<!-- openhands-review-gate:blocked:head-2 -->" in body
|
|
1386
|
+
|
|
1387
|
+
|
|
1388
|
+
def test_reviewer_does_not_duplicate_the_gate_comment_for_the_same_head(
|
|
1389
|
+
tmp_path, monkeypatch
|
|
1390
|
+
):
|
|
1391
|
+
"""An identical explanation already on the PR is left untouched."""
|
|
1392
|
+
_module, run = _reviewer(tmp_path, monkeypatch)
|
|
1393
|
+
pr = _labeled_pr()
|
|
1394
|
+
existing = _comment(run._gate_body("head-2", "blocked", ["ci"], scheduled=True))
|
|
1395
|
+
run.gh_pages = _gate_pages(pr, [existing])
|
|
1396
|
+
run.gh = Mock(side_effect=[{"id": 99}, pr])
|
|
1397
|
+
run.check_runs = lambda sha: _checks(("ci", "completed", "failure"), sha=sha)
|
|
1398
|
+
|
|
1399
|
+
run.run()
|
|
1400
|
+
|
|
1401
|
+
run.dispatcher.deliver.assert_not_called()
|
|
1402
|
+
assert _gate_comment_calls(run) == []
|
|
1403
|
+
assert [call for call in run.gh.call_args_list if call.args[0] == "PATCH"] == []
|
|
1404
|
+
|
|
1405
|
+
|
|
1406
|
+
def test_reviewer_rewrites_a_stale_gate_comment_body_for_the_same_head(
|
|
1407
|
+
tmp_path, monkeypatch
|
|
1408
|
+
):
|
|
1409
|
+
"""Same head, changed wording: the managed comment is rewritten, not stacked."""
|
|
1410
|
+
_module, run = _reviewer(tmp_path, monkeypatch)
|
|
1411
|
+
pr = _labeled_pr()
|
|
1412
|
+
stale = _comment(
|
|
1413
|
+
"old wording\n\n<!-- openhands-review-gate:blocked:head-2 -->"
|
|
1414
|
+
)
|
|
1415
|
+
run.gh_pages = _gate_pages(pr, [stale])
|
|
1416
|
+
run.gh = Mock(side_effect=[{"id": 99}, pr, {}])
|
|
1417
|
+
run.check_runs = lambda sha: _checks(("ci", "completed", "failure"), sha=sha)
|
|
1418
|
+
|
|
1419
|
+
run.run()
|
|
1420
|
+
|
|
1421
|
+
assert [call for call in _gate_comment_calls(run) if call.args[0] == "POST"] == []
|
|
1422
|
+
patched = [call for call in run.gh.call_args_list if call.args[0] == "PATCH"]
|
|
1423
|
+
assert len(patched) == 1
|
|
1424
|
+
assert patched[0].args[1] == "/issues/comments/900"
|
|
1425
|
+
body = patched[0].args[2]["body"]
|
|
1426
|
+
assert "<!-- openhands-review-gate:blocked:head-2 -->" in body
|
|
1427
|
+
assert "old wording" not in body
|
|
1428
|
+
|
|
1429
|
+
|
|
1430
|
+
def test_reviewer_updates_its_gate_comment_when_the_head_moves(
|
|
1431
|
+
tmp_path, monkeypatch
|
|
1432
|
+
):
|
|
1433
|
+
_module, run = _reviewer(tmp_path, monkeypatch)
|
|
1434
|
+
pr = _labeled_pr()
|
|
1435
|
+
stale = _comment("stale head\n\n<!-- openhands-review-gate:blocked:head-1 -->")
|
|
1436
|
+
run.gh_pages = _gate_pages(pr, [stale])
|
|
1437
|
+
run.gh = Mock(side_effect=[{"id": 99}, pr, {}])
|
|
1438
|
+
run.check_runs = lambda sha: _checks(("ci", "completed", "failure"), sha=sha)
|
|
1439
|
+
|
|
1440
|
+
run.run()
|
|
1441
|
+
|
|
1442
|
+
patched = [call for call in run.gh.call_args_list if call.args[0] == "PATCH"]
|
|
1443
|
+
assert len(patched) == 1
|
|
1444
|
+
assert patched[0].args[1] == "/issues/comments/900"
|
|
1445
|
+
assert "<!-- openhands-review-gate:blocked:head-2 -->" in patched[0].args[2]["body"]
|
|
1446
|
+
assert run.dispatcher.deliver.call_count == 0
|
|
1447
|
+
|
|
1448
|
+
|
|
1449
|
+
def test_reviewer_defers_to_an_equivalent_workflow_remediation_comment(
|
|
1450
|
+
tmp_path, monkeypatch
|
|
1451
|
+
):
|
|
1452
|
+
_module, run = _reviewer(tmp_path, monkeypatch)
|
|
1453
|
+
pr = _labeled_pr()
|
|
1454
|
+
workflow = _comment(
|
|
1455
|
+
"The `Validate PR description` check still fails: the PR description's "
|
|
1456
|
+
"`HUMAN:` section needs at least 20 characters of what you tested.\n\n"
|
|
1457
|
+
"_This is an automated check - no AI was used to generate this comment._"
|
|
1458
|
+
)
|
|
1459
|
+
run.gh_pages = _gate_pages(pr, [workflow])
|
|
1460
|
+
run.gh = Mock(side_effect=[{"id": 99}, pr])
|
|
1461
|
+
run.check_runs = lambda sha: _checks(
|
|
1462
|
+
("Validate PR description", "completed", "failure"), sha=sha
|
|
1463
|
+
)
|
|
1464
|
+
|
|
1465
|
+
run.run()
|
|
1466
|
+
|
|
1467
|
+
run.dispatcher.deliver.assert_not_called()
|
|
1468
|
+
assert _gate_comment_calls(run) == []
|
|
1469
|
+
|
|
1470
|
+
|
|
1471
|
+
def test_reviewer_ignores_a_marked_comment_a_pr_author_wrote(
|
|
1472
|
+
tmp_path, monkeypatch
|
|
1473
|
+
):
|
|
1474
|
+
"""A forged marker is untrusted: it neither suppresses nor gets PATCHed."""
|
|
1475
|
+
_module, run = _reviewer(tmp_path, monkeypatch)
|
|
1476
|
+
pr = _labeled_pr()
|
|
1477
|
+
forged = _comment(
|
|
1478
|
+
"I already handled this\n\n<!-- openhands-review-gate:blocked:head-2 -->",
|
|
1479
|
+
login="neubig",
|
|
1480
|
+
)
|
|
1481
|
+
run.gh_pages = _gate_pages(pr, [forged])
|
|
1482
|
+
run.gh = Mock(side_effect=[{"id": 99}, pr, {"id": 1234}])
|
|
1483
|
+
run.check_runs = lambda sha: _checks(("ci", "completed", "failure"), sha=sha)
|
|
1484
|
+
|
|
1485
|
+
run.run()
|
|
1486
|
+
|
|
1487
|
+
run.dispatcher.deliver.assert_not_called()
|
|
1488
|
+
assert [call for call in run.gh.call_args_list if call.args[0] == "PATCH"] == []
|
|
1489
|
+
posted = [call for call in _gate_comment_calls(run) if call.args[0] == "POST"]
|
|
1490
|
+
assert len(posted) == 1
|
|
1491
|
+
assert "<!-- openhands-review-gate:blocked:head-2 -->" in posted[0].args[2]["body"]
|
|
1492
|
+
|
|
1493
|
+
|
|
1494
|
+
def test_reviewer_does_not_defer_to_a_workflow_comment_about_another_check(
|
|
1495
|
+
tmp_path, monkeypatch
|
|
1496
|
+
):
|
|
1497
|
+
"""A disclosure for a different check must not suppress this head's gate."""
|
|
1498
|
+
_module, run = _reviewer(tmp_path, monkeypatch)
|
|
1499
|
+
pr = _labeled_pr()
|
|
1500
|
+
unrelated = _comment(
|
|
1501
|
+
"The `Validate PR description` check still fails.\n\n"
|
|
1502
|
+
"_This is an automated check - no AI was used to generate this comment._"
|
|
1503
|
+
)
|
|
1504
|
+
run.gh_pages = _gate_pages(pr, [unrelated])
|
|
1505
|
+
run.gh = Mock(side_effect=[{"id": 99}, pr, {"id": 1234}])
|
|
1506
|
+
run.check_runs = lambda sha: _checks(("lint", "completed", "failure"), sha=sha)
|
|
1507
|
+
|
|
1508
|
+
run.run()
|
|
1509
|
+
|
|
1510
|
+
posted = [call for call in _gate_comment_calls(run) if call.args[0] == "POST"]
|
|
1511
|
+
assert len(posted) == 1
|
|
1512
|
+
assert "`lint`" in posted[0].args[2]["body"]
|
|
1513
|
+
assert "<!-- openhands-review-gate:blocked:head-2 -->" in posted[0].args[2]["body"]
|
|
1514
|
+
|
|
1515
|
+
|
|
1516
|
+
def test_reviewer_explicit_request_bypasses_the_ci_gate(tmp_path, monkeypatch):
|
|
1517
|
+
"""An explicit `all-hands-bot` request is the intake exception.
|
|
1518
|
+
|
|
1519
|
+
The caller asked for this head by name, so a red or pending required check
|
|
1520
|
+
must not stop the dispatch - and the gate leaves no explanatory comment.
|
|
1521
|
+
"""
|
|
1522
|
+
_module, run = _reviewer(tmp_path, monkeypatch)
|
|
1523
|
+
_event(monkeypatch)
|
|
1524
|
+
pr = {"number": 2, "head": {"sha": "head-2"}, "labels": []}
|
|
1525
|
+
request = {
|
|
1526
|
+
"id": 42,
|
|
1527
|
+
"event": "review_requested",
|
|
1528
|
+
"created_at": "2026-01-01T00:00:00Z",
|
|
1529
|
+
"requested_reviewer": {"login": "all-hands-bot"},
|
|
1530
|
+
}
|
|
1531
|
+
run.gh = Mock(side_effect=[{"id": 99}, pr])
|
|
1532
|
+
run.gh_pages = lambda path: [request] if path.endswith("/events") else []
|
|
1533
|
+
run.check_runs = lambda sha: _checks(
|
|
1534
|
+
("Validate PR description", "completed", "failure"), sha=sha
|
|
1535
|
+
)
|
|
1536
|
+
run.required_check_contexts = lambda number: [
|
|
1537
|
+
{"name": "Validate PR description", "kind": "CheckRun"}
|
|
1538
|
+
]
|
|
1539
|
+
run.dispatcher.deliver.return_value = {
|
|
1540
|
+
"disposition": "created",
|
|
1541
|
+
"conversation_id": "conversation",
|
|
1542
|
+
}
|
|
1543
|
+
|
|
1544
|
+
run.run()
|
|
1545
|
+
|
|
1546
|
+
run.dispatcher.deliver.assert_called_once()
|
|
1547
|
+
assert run.dispatcher.deliver.call_args.kwargs["delivery"] == "42:head-2"
|
|
1548
|
+
assert _gate_comment_calls(run) == []
|
|
1549
|
+
|
|
1550
|
+
|
|
1551
|
+
def test_reviewer_waiting_head_does_not_consume_a_later_green_request(
|
|
1552
|
+
tmp_path, monkeypatch
|
|
1553
|
+
):
|
|
1554
|
+
"""The trigger survives a waiting run, so the head is reviewed once green."""
|
|
1555
|
+
_module, run = _reviewer(tmp_path, monkeypatch)
|
|
1556
|
+
pr = _labeled_pr()
|
|
1557
|
+
pending = _checks(("slow-e2e", "queued", None), sha="head-2")
|
|
1558
|
+
state = {"runs": pending}
|
|
1559
|
+
run.gh_pages = _gate_pages(pr, [])
|
|
1560
|
+
run.gh = Mock(side_effect=[{"id": 99}, pr, {"id": 1234}, pr])
|
|
1561
|
+
run.check_runs = lambda sha: state["runs"]
|
|
1562
|
+
run.dispatcher.deliver.return_value = {
|
|
1563
|
+
"disposition": "created",
|
|
1564
|
+
"conversation_id": "conversation",
|
|
1565
|
+
}
|
|
1566
|
+
|
|
1567
|
+
run.run()
|
|
1568
|
+
assert run.dispatcher.deliver.call_count == 0
|
|
1569
|
+
|
|
1570
|
+
run.gh_pages = _gate_pages(pr, [])
|
|
1571
|
+
run.gh = Mock(side_effect=[{"id": 99}, pr])
|
|
1572
|
+
state["runs"] = _checks(("slow-e2e", "completed", "success"), sha="head-2")
|
|
1573
|
+
run.run()
|
|
1574
|
+
assert run.dispatcher.deliver.call_count == 1
|
|
1575
|
+
|
|
1576
|
+
|
|
1577
|
+
|
|
1578
|
+
def test_reviewer_ignores_a_superseded_failure_for_the_same_check(
|
|
1579
|
+
tmp_path, monkeypatch
|
|
1580
|
+
):
|
|
1581
|
+
"""The live #647/#649 case: an early failure and a later success, one SHA.
|
|
1582
|
+
|
|
1583
|
+
GitHub lists both `Validate PR description` runs on a single commit when the
|
|
1584
|
+
PR body is fixed without a code push, so only the latest run may count.
|
|
1585
|
+
"""
|
|
1586
|
+
_module, run = _reviewer(tmp_path, monkeypatch)
|
|
1587
|
+
pr = _labeled_pr()
|
|
1588
|
+
run.gh_pages = _gate_pages(pr, [])
|
|
1589
|
+
run.gh = Mock(side_effect=[{"id": 99}, pr])
|
|
1590
|
+
run.check_runs = lambda sha: [
|
|
1591
|
+
_run(
|
|
1592
|
+
"Validate PR description",
|
|
1593
|
+
"completed",
|
|
1594
|
+
"success",
|
|
1595
|
+
started_at="2026-09-22T13:18:41Z",
|
|
1596
|
+
run_id=106761769135,
|
|
1597
|
+
),
|
|
1598
|
+
_run(
|
|
1599
|
+
"Validate PR description",
|
|
1600
|
+
"completed",
|
|
1601
|
+
"failure",
|
|
1602
|
+
started_at="2026-09-22T13:04:07Z",
|
|
1603
|
+
run_id=106756418507,
|
|
1604
|
+
),
|
|
1605
|
+
]
|
|
1606
|
+
run.dispatcher.deliver.return_value = {
|
|
1607
|
+
"disposition": "created",
|
|
1608
|
+
"conversation_id": "conversation",
|
|
1609
|
+
}
|
|
1610
|
+
|
|
1611
|
+
run.run()
|
|
1612
|
+
|
|
1613
|
+
run.dispatcher.deliver.assert_called_once()
|
|
1614
|
+
assert _gate_comment_calls(run) == []
|
|
1615
|
+
|
|
1616
|
+
|
|
1617
|
+
def test_reviewer_prefers_run_id_to_break_a_start_time_tie(tmp_path, monkeypatch):
|
|
1618
|
+
"""Two runs sharing a start time are ordered by run ID, not list order."""
|
|
1619
|
+
_module, run = _reviewer(tmp_path, monkeypatch)
|
|
1620
|
+
pr = _labeled_pr()
|
|
1621
|
+
run.gh_pages = _gate_pages(pr, [])
|
|
1622
|
+
run.gh = Mock(side_effect=[{"id": 99}, pr])
|
|
1623
|
+
run.check_runs = lambda sha: [
|
|
1624
|
+
_run(
|
|
1625
|
+
"ci",
|
|
1626
|
+
"completed",
|
|
1627
|
+
"failure",
|
|
1628
|
+
started_at="2026-09-22T13:04:07Z",
|
|
1629
|
+
run_id=106756418507,
|
|
1630
|
+
),
|
|
1631
|
+
_run(
|
|
1632
|
+
"ci",
|
|
1633
|
+
"completed",
|
|
1634
|
+
"success",
|
|
1635
|
+
started_at="2026-09-22T13:04:07Z",
|
|
1636
|
+
run_id=106761769135,
|
|
1637
|
+
),
|
|
1638
|
+
]
|
|
1639
|
+
run.dispatcher.deliver.return_value = {
|
|
1640
|
+
"disposition": "created",
|
|
1641
|
+
"conversation_id": "conversation",
|
|
1642
|
+
}
|
|
1643
|
+
|
|
1644
|
+
run.run()
|
|
1645
|
+
|
|
1646
|
+
run.dispatcher.deliver.assert_called_once()
|
|
1647
|
+
assert _gate_comment_calls(run) == []
|
|
1648
|
+
|
|
1649
|
+
|
|
1650
|
+
def test_reviewer_waits_for_a_newer_rerun_that_superseded_a_success(
|
|
1651
|
+
tmp_path, monkeypatch
|
|
1652
|
+
):
|
|
1653
|
+
"""A later queued re-run of the same check makes the head wait, not pass."""
|
|
1654
|
+
_module, run = _reviewer(tmp_path, monkeypatch)
|
|
1655
|
+
pr = _labeled_pr()
|
|
1656
|
+
run.gh_pages = _gate_pages(pr, [])
|
|
1657
|
+
run.gh = Mock(side_effect=[{"id": 99}, pr, {"id": 1234}])
|
|
1658
|
+
run.check_runs = lambda sha: [
|
|
1659
|
+
_run("ci", "completed", "success", started_at="2026-09-22T13:00:00Z", run_id=1),
|
|
1660
|
+
_run("ci", "queued", None, started_at="2026-09-22T13:10:00Z", run_id=2),
|
|
1661
|
+
]
|
|
1662
|
+
|
|
1663
|
+
run.run()
|
|
1664
|
+
|
|
1665
|
+
run.dispatcher.deliver.assert_not_called()
|
|
1666
|
+
posted = [call for call in _gate_comment_calls(run) if call.args[0] == "POST"]
|
|
1667
|
+
assert len(posted) == 1
|
|
1668
|
+
body = posted[0].args[2]["body"]
|
|
1669
|
+
assert "<!-- openhands-review-gate:waiting:head-2 -->" in body
|
|
1670
|
+
assert "`ci`" in body
|
|
1671
|
+
|
|
1672
|
+
|
|
1673
|
+
def test_reviewer_treats_same_name_from_different_apps_as_distinct_checks(
|
|
1674
|
+
tmp_path, monkeypatch
|
|
1675
|
+
):
|
|
1676
|
+
"""Grouping by name and app identity keeps two apps' checks independent."""
|
|
1677
|
+
_module, run = _reviewer(tmp_path, monkeypatch)
|
|
1678
|
+
pr = _labeled_pr()
|
|
1679
|
+
run.gh_pages = _gate_pages(pr, [])
|
|
1680
|
+
run.gh = Mock(side_effect=[{"id": 99}, pr, {"id": 1234}])
|
|
1681
|
+
run.check_runs = lambda sha: [
|
|
1682
|
+
_run(
|
|
1683
|
+
"ci",
|
|
1684
|
+
"completed",
|
|
1685
|
+
"success",
|
|
1686
|
+
app="github-actions",
|
|
1687
|
+
started_at="2026-09-22T13:10:00Z",
|
|
1688
|
+
run_id=2,
|
|
1689
|
+
),
|
|
1690
|
+
_run(
|
|
1691
|
+
"ci",
|
|
1692
|
+
"completed",
|
|
1693
|
+
"failure",
|
|
1694
|
+
app="custom-ci",
|
|
1695
|
+
started_at="2026-09-22T13:00:00Z",
|
|
1696
|
+
run_id=1,
|
|
1697
|
+
),
|
|
1698
|
+
]
|
|
1699
|
+
|
|
1700
|
+
run.run()
|
|
1701
|
+
|
|
1702
|
+
run.dispatcher.deliver.assert_not_called()
|
|
1703
|
+
posted = [call for call in _gate_comment_calls(run) if call.args[0] == "POST"]
|
|
1704
|
+
assert len(posted) == 1
|
|
1705
|
+
assert "<!-- openhands-review-gate:blocked:head-2 -->" in posted[0].args[2]["body"]
|
|
1706
|
+
|
|
1707
|
+
|
|
1708
|
+
def test_reviewer_waits_for_a_newer_queued_run_without_a_start_time(
|
|
1709
|
+
tmp_path, monkeypatch
|
|
1710
|
+
):
|
|
1711
|
+
"""A queued re-run with `started_at: null` still supersedes an earlier run.
|
|
1712
|
+
|
|
1713
|
+
GitHub allows a queued or requested check run before it has a start
|
|
1714
|
+
timestamp, so ordering must not treat the absent start time as earlier than
|
|
1715
|
+
a completed run's timestamp and let the stale success launch the review.
|
|
1716
|
+
"""
|
|
1717
|
+
_module, run = _reviewer(tmp_path, monkeypatch)
|
|
1718
|
+
pr = _labeled_pr()
|
|
1719
|
+
run.gh_pages = _gate_pages(pr, [])
|
|
1720
|
+
run.gh = Mock(side_effect=[{"id": 99}, pr, {"id": 1234}])
|
|
1721
|
+
run.check_runs = lambda sha: [
|
|
1722
|
+
_run(
|
|
1723
|
+
"ci",
|
|
1724
|
+
"completed",
|
|
1725
|
+
"success",
|
|
1726
|
+
started_at="2026-09-22T13:00:00Z",
|
|
1727
|
+
run_id=1,
|
|
1728
|
+
),
|
|
1729
|
+
_run("ci", "queued", None, started_at=None, run_id=2),
|
|
1730
|
+
]
|
|
1731
|
+
|
|
1732
|
+
run.run()
|
|
1733
|
+
|
|
1734
|
+
run.dispatcher.deliver.assert_not_called()
|
|
1735
|
+
posted = [call for call in _gate_comment_calls(run) if call.args[0] == "POST"]
|
|
1736
|
+
assert len(posted) == 1
|
|
1737
|
+
body = posted[0].args[2]["body"]
|
|
1738
|
+
assert "<!-- openhands-review-gate:waiting:head-2 -->" in body
|
|
1739
|
+
assert "`ci`" in body
|
|
1740
|
+
|
|
1741
|
+
|
|
1742
|
+
# Scheduled retry: a five-minute scan resumes an outstanding review request.
|
|
1743
|
+
# --------------------------------------------------------------------------- #
|
|
1744
|
+
|
|
1745
|
+
|
|
1746
|
+
def _requested_pr(number=2, sha="head-2", draft=False):
|
|
1747
|
+
return {
|
|
1748
|
+
"number": number,
|
|
1749
|
+
"head": {"sha": sha},
|
|
1750
|
+
"labels": [],
|
|
1751
|
+
"draft": draft,
|
|
1752
|
+
"requested_reviewers": [{"login": "all-hands-bot"}],
|
|
1753
|
+
}
|
|
1754
|
+
|
|
1755
|
+
|
|
1756
|
+
def _review_request_event(request_id=42):
|
|
1757
|
+
return {
|
|
1758
|
+
"id": request_id,
|
|
1759
|
+
"event": "review_requested",
|
|
1760
|
+
"created_at": "2026-01-01T00:00:00Z",
|
|
1761
|
+
"requested_reviewer": {"login": "all-hands-bot"},
|
|
1762
|
+
}
|
|
1763
|
+
|
|
1764
|
+
|
|
1765
|
+
def _request_pages(pr, request, comments):
|
|
1766
|
+
return lambda path: (
|
|
1767
|
+
[pr]
|
|
1768
|
+
if path.startswith("/pulls?")
|
|
1769
|
+
else ([request] if path.endswith("/events") else comments)
|
|
1770
|
+
)
|
|
1771
|
+
|
|
1772
|
+
|
|
1773
|
+
@pytest.mark.parametrize("draft", [False, True])
|
|
1774
|
+
def test_reviewer_scheduled_scan_reviews_an_outstanding_request(tmp_path, monkeypatch, draft):
|
|
1775
|
+
"""A request made before CI finished is reviewed once the head is green."""
|
|
1776
|
+
_module, run = _reviewer(tmp_path, monkeypatch)
|
|
1777
|
+
pr = _requested_pr(draft=draft)
|
|
1778
|
+
state = {"runs": _checks(("slow-e2e", "queued", None), sha="head-2")}
|
|
1779
|
+
run.check_runs = lambda sha: state["runs"]
|
|
1780
|
+
run.gh_pages = _request_pages(pr, _review_request_event(), [])
|
|
1781
|
+
run.gh = Mock(side_effect=[{"id": 99}, pr, {"id": 1234}, pr])
|
|
1782
|
+
run.dispatcher.deliver.return_value = {
|
|
1783
|
+
"disposition": "created",
|
|
1784
|
+
"conversation_id": "conversation",
|
|
1785
|
+
}
|
|
1786
|
+
|
|
1787
|
+
run.run()
|
|
1788
|
+
assert run.dispatcher.deliver.call_count == 0
|
|
1789
|
+
posted = [call for call in _gate_comment_calls(run) if call.args[0] == "POST"]
|
|
1790
|
+
assert len(posted) == 1
|
|
1791
|
+
waiting = posted[0].args[2]["body"]
|
|
1792
|
+
assert "<!-- openhands-review-gate:waiting:head-2 -->" in waiting
|
|
1793
|
+
assert "The scheduled scan retries" in waiting
|
|
1794
|
+
|
|
1795
|
+
run.gh_pages = _request_pages(pr, _review_request_event(), [])
|
|
1796
|
+
run.gh = Mock(side_effect=[{"id": 99}, pr])
|
|
1797
|
+
state["runs"] = _checks(("slow-e2e", "completed", "success"), sha="head-2")
|
|
1798
|
+
run.run()
|
|
1799
|
+
|
|
1800
|
+
run.dispatcher.deliver.assert_called_once()
|
|
1801
|
+
call = run.dispatcher.deliver.call_args.kwargs
|
|
1802
|
+
assert call["delivery"] == "42:head-2"
|
|
1803
|
+
assert "latest review request for `all-hands-bot` event 42" in call["prompt"]
|
|
1804
|
+
|
|
1805
|
+
|
|
1806
|
+
@pytest.mark.parametrize("draft", [False, True])
|
|
1807
|
+
def test_reviewer_repeated_scans_do_not_duplicate_a_requested_review(
|
|
1808
|
+
tmp_path, monkeypatch, draft
|
|
1809
|
+
):
|
|
1810
|
+
"""Two scans over one outstanding request produce one stable delivery key."""
|
|
1811
|
+
_module, run = _reviewer(tmp_path, monkeypatch)
|
|
1812
|
+
pr = _requested_pr(draft=draft)
|
|
1813
|
+
run.check_runs = lambda sha: _checks(("ci", "completed", "success"), sha=sha)
|
|
1814
|
+
run.dispatcher.deliver.return_value = {
|
|
1815
|
+
"disposition": "created",
|
|
1816
|
+
"conversation_id": "conversation",
|
|
1817
|
+
}
|
|
1818
|
+
|
|
1819
|
+
for _ in range(2):
|
|
1820
|
+
run.gh_pages = _request_pages(pr, _review_request_event(), [])
|
|
1821
|
+
run.gh = Mock(side_effect=[{"id": 99}, pr])
|
|
1822
|
+
run.run()
|
|
1823
|
+
|
|
1824
|
+
assert run.dispatcher.deliver.call_count == 2
|
|
1825
|
+
deliveries = {
|
|
1826
|
+
call.kwargs["delivery"] for call in run.dispatcher.deliver.call_args_list
|
|
1827
|
+
}
|
|
1828
|
+
assert deliveries == {"42:head-2"}
|
|
1829
|
+
|
|
1830
|
+
|
|
1831
|
+
def test_reviewer_scheduled_scan_ignores_a_draft_requested_from_someone_else(
|
|
1832
|
+
tmp_path, monkeypatch
|
|
1833
|
+
):
|
|
1834
|
+
_module, run = _reviewer(tmp_path, monkeypatch)
|
|
1835
|
+
pr = _requested_pr(draft=True)
|
|
1836
|
+
pr["requested_reviewers"] = [{"login": "another-reviewer"}]
|
|
1837
|
+
run.gh_pages = lambda path: [pr]
|
|
1838
|
+
run.gh = Mock(return_value={"id": 99})
|
|
1839
|
+
|
|
1840
|
+
run.run()
|
|
1841
|
+
|
|
1842
|
+
run.dispatcher.deliver.assert_not_called()
|
|
1843
|
+
|
|
1844
|
+
|
|
1845
|
+
def test_reviewer_scheduled_scan_reviews_an_unrequested_green_pr(
|
|
1846
|
+
tmp_path, monkeypatch
|
|
1847
|
+
):
|
|
1848
|
+
"""A green PR with no request and no label is still reviewed on a scan."""
|
|
1849
|
+
_module, run = _reviewer(tmp_path, monkeypatch)
|
|
1850
|
+
pr = {"number": 3, "head": {"sha": "head-3"}, "labels": [], "draft": False}
|
|
1851
|
+
run.gh_pages = lambda path: (
|
|
1852
|
+
[] if path.endswith("/reviews") else [pr]
|
|
1853
|
+
)
|
|
1854
|
+
run.gh = Mock(side_effect=[{"id": 99}, pr])
|
|
1855
|
+
run.check_runs = lambda sha: _checks(("ci", "completed", "success"), sha=sha)
|
|
1856
|
+
run.dispatcher.deliver.return_value = {
|
|
1857
|
+
"disposition": "created",
|
|
1858
|
+
"conversation_id": "conversation",
|
|
1859
|
+
}
|
|
1860
|
+
|
|
1861
|
+
run.run()
|
|
1862
|
+
|
|
1863
|
+
run.dispatcher.deliver.assert_called_once()
|
|
1864
|
+
call = run.dispatcher.deliver.call_args.kwargs
|
|
1865
|
+
assert call["subject"] == "99:pr:3"
|
|
1866
|
+
assert call["delivery"] == "scan:owner/repo:3:head-3"
|
|
1867
|
+
assert "scheduled scan of open, non-draft pull requests" in call["prompt"]
|
|
1868
|
+
|
|
1869
|
+
|
|
1870
|
+
def test_reviewer_scheduled_scan_skips_a_pr_with_a_current_head_review(
|
|
1871
|
+
tmp_path, monkeypatch
|
|
1872
|
+
):
|
|
1873
|
+
"""An already-reviewed head is not re-reviewed by the unrequested scan."""
|
|
1874
|
+
_module, run = _reviewer(tmp_path, monkeypatch)
|
|
1875
|
+
pr = {"number": 3, "head": {"sha": "head-3"}, "labels": [], "draft": False}
|
|
1876
|
+
run.gh_pages = lambda path: (
|
|
1877
|
+
_reviews(sha="head-3") if path.endswith("/reviews") else [pr]
|
|
1878
|
+
)
|
|
1879
|
+
# The repository-id lookup, the full PR read, and the head re-read that the
|
|
1880
|
+
# completion path performs to confirm the reviewed head has not moved.
|
|
1881
|
+
run.gh = Mock(side_effect=[{"id": 99}, pr, pr])
|
|
1882
|
+
run.check_runs = lambda sha: _checks(("ci", "completed", "success"), sha=sha)
|
|
1883
|
+
|
|
1884
|
+
run.run()
|
|
1885
|
+
|
|
1886
|
+
run.dispatcher.deliver.assert_not_called()
|
|
1887
|
+
|
|
1888
|
+
|
|
1889
|
+
def test_reviewer_scheduled_scan_keeps_the_label_path_unchanged(
|
|
1890
|
+
tmp_path, monkeypatch
|
|
1891
|
+
):
|
|
1892
|
+
"""A labeled PR is still driven by its label event, not by a request."""
|
|
1893
|
+
_module, run = _reviewer(tmp_path, monkeypatch)
|
|
1894
|
+
pr = _labeled_pr()
|
|
1895
|
+
run.gh_pages = _gate_pages(pr, [])
|
|
1896
|
+
run.gh = Mock(side_effect=[{"id": 99}, pr])
|
|
1897
|
+
run.check_runs = lambda sha: _checks(("ci", "completed", "success"), sha=sha)
|
|
1898
|
+
run.dispatcher.deliver.return_value = {
|
|
1899
|
+
"disposition": "created",
|
|
1900
|
+
"conversation_id": "conversation",
|
|
1901
|
+
}
|
|
1902
|
+
|
|
1903
|
+
run.run()
|
|
1904
|
+
|
|
1905
|
+
run.dispatcher.deliver.assert_called_once()
|
|
1906
|
+
assert run.dispatcher.deliver.call_args.kwargs["delivery"] == "7:head-2"
|
|
1907
|
+
|
|
1908
|
+
|
|
1909
|
+
def test_reviewer_event_body_names_the_request_retry_not_a_scan(
|
|
1910
|
+
tmp_path, monkeypatch
|
|
1911
|
+
):
|
|
1912
|
+
"""The event-only body names the retry it actually honors.
|
|
1913
|
+
|
|
1914
|
+
Explicit `all-hands-bot` requests bypass the CI gate, so this body is not
|
|
1915
|
+
reached through `run()`; assert the wording contract directly.
|
|
1916
|
+
"""
|
|
1917
|
+
_module, run = _reviewer(tmp_path, monkeypatch)
|
|
1918
|
+
body = run._gate_body("head-2", "waiting", ["slow-e2e"], scheduled=False)
|
|
1919
|
+
|
|
1920
|
+
assert "remove the outstanding `all-hands-bot` request and request " in body
|
|
1921
|
+
assert "GitHub will not accept a second request" in body
|
|
1922
|
+
assert "scheduled" not in body.lower()
|
|
1923
|
+
|
|
1924
|
+
|
|
1925
|
+
def test_reviewer_event_blocked_body_names_the_request_retry_not_a_scan(
|
|
1926
|
+
tmp_path, monkeypatch
|
|
1927
|
+
):
|
|
1928
|
+
_module, run = _reviewer(tmp_path, monkeypatch)
|
|
1929
|
+
body = run._gate_body("head-2", "blocked", ["ci"], scheduled=False)
|
|
1930
|
+
|
|
1931
|
+
assert "remove the outstanding `all-hands-bot` request and request " in body
|
|
1932
|
+
assert "GitHub will not accept a second request" in body
|
|
1933
|
+
assert "scheduled" not in body.lower()
|
|
1934
|
+
|
|
1935
|
+
|
|
1936
|
+
def test_reviewer_rewords_the_managed_comment_when_an_event_deployment_becomes_scheduled(
|
|
1937
|
+
tmp_path, monkeypatch
|
|
1938
|
+
):
|
|
1939
|
+
"""An event-only gate comment is reworded once a cron scan exists.
|
|
1940
|
+
|
|
1941
|
+
A PR first gated by the event-only deployment keeps a "remove and re-request"
|
|
1942
|
+
comment on the same head. When the automation is switched to a scheduled scan
|
|
1943
|
+
the comment must name the retry that is now actually deployed, in place,
|
|
1944
|
+
without creating a second comment.
|
|
1945
|
+
"""
|
|
1946
|
+
_module, run = _reviewer(tmp_path, monkeypatch)
|
|
1947
|
+
pr = _requested_pr()
|
|
1948
|
+
request = _review_request_event()
|
|
1949
|
+
run.check_runs = lambda sha: _checks(("slow-e2e", "in_progress", None), sha=sha)
|
|
1950
|
+
|
|
1951
|
+
# The comment an event-only deployment would have left on this head.
|
|
1952
|
+
event_body = run._gate_body("head-2", "waiting", ["slow-e2e"], scheduled=False)
|
|
1953
|
+
assert "remove the outstanding `all-hands-bot` request" in event_body
|
|
1954
|
+
managed = _comment(event_body)
|
|
1955
|
+
|
|
1956
|
+
run.gh_pages = _request_pages(pr, request, [managed])
|
|
1957
|
+
run.gh = Mock(side_effect=[{"id": 99}, pr, {}])
|
|
1958
|
+
run.run()
|
|
1959
|
+
|
|
1960
|
+
assert [call for call in _gate_comment_calls(run) if call.args[0] == "POST"] == []
|
|
1961
|
+
patched = [call for call in run.gh.call_args_list if call.args[0] == "PATCH"]
|
|
1962
|
+
assert len(patched) == 1
|
|
1963
|
+
assert patched[0].args[1] == "/issues/comments/900"
|
|
1964
|
+
scheduled_body = patched[0].args[2]["body"]
|
|
1965
|
+
assert "<!-- openhands-review-gate:waiting:head-2 -->" in scheduled_body
|
|
1966
|
+
assert "The scheduled scan retries" in scheduled_body
|
|
1967
|
+
assert "remove the outstanding" not in scheduled_body
|
|
1968
|
+
|
|
1969
|
+
|
|
1970
|
+
def test_reviewer_scheduled_scan_blocked_comment_names_the_scan(
|
|
1971
|
+
tmp_path, monkeypatch
|
|
1972
|
+
):
|
|
1973
|
+
_module, run = _reviewer(tmp_path, monkeypatch)
|
|
1974
|
+
pr = _requested_pr()
|
|
1975
|
+
run.gh_pages = _request_pages(pr, _review_request_event(), [])
|
|
1976
|
+
run.gh = Mock(side_effect=[{"id": 99}, pr, {"id": 1234}])
|
|
1977
|
+
run.check_runs = lambda sha: _checks(("ci", "completed", "failure"), sha=sha)
|
|
1978
|
+
|
|
1979
|
+
run.run()
|
|
1980
|
+
|
|
1981
|
+
posted = [call for call in _gate_comment_calls(run) if call.args[0] == "POST"]
|
|
1982
|
+
body = posted[0].args[2]["body"]
|
|
1983
|
+
assert "The scheduled scan then starts the review" in body
|
|
1984
|
+
|
|
1985
|
+
|
|
1986
|
+
|
|
1987
|
+
# --------------------------------------------------------------------------- #
|
|
1988
|
+
# Bounded intake: one scheduled scan starts a small, configurable number of new
|
|
1989
|
+
# review conversations across every configured repository, oldest request first.
|
|
1990
|
+
# --------------------------------------------------------------------------- #
|
|
1991
|
+
|
|
1992
|
+
|
|
1993
|
+
class _DedupeDispatcher:
|
|
1994
|
+
"""An in-memory stand-in for the shipped dispatcher's dedupe contract.
|
|
1995
|
+
|
|
1996
|
+
The real AgentConversationDispatcher needs an agent server and SDK, so this
|
|
1997
|
+
reproduces only the contract the worker relies on: a delivery keyed by
|
|
1998
|
+
(subject, delivery) starts one conversation and reports `created` once, then
|
|
1999
|
+
reports `deduplicated` for the same key. That is what lets a repeat scan
|
|
2000
|
+
reuse a conversation without spending another intake slot. It also records
|
|
2001
|
+
the head the caller named, which the worker's own pre-dispatch guard uses to
|
|
2002
|
+
suppress a second review of an unchanged head.
|
|
2003
|
+
"""
|
|
2004
|
+
|
|
2005
|
+
def __init__(self):
|
|
2006
|
+
self.seen = {}
|
|
2007
|
+
self.heads = []
|
|
2008
|
+
self.calls = []
|
|
2009
|
+
self.prompts = []
|
|
2010
|
+
|
|
2011
|
+
def deliver(self, *, subject, delivery, prompt, head=""):
|
|
2012
|
+
self.calls.append((subject, delivery))
|
|
2013
|
+
self.prompts.append(prompt)
|
|
2014
|
+
self.heads.append(head)
|
|
2015
|
+
if self.seen.get(subject) == delivery:
|
|
2016
|
+
return {"disposition": "deduplicated", "conversation_id": subject}
|
|
2017
|
+
self.seen[subject] = delivery
|
|
2018
|
+
return {"disposition": "created", "conversation_id": subject}
|
|
2019
|
+
|
|
2020
|
+
|
|
2021
|
+
def _scan_reviewers(tmp_path, monkeypatch, repositories, *, max_new=None):
|
|
2022
|
+
"""One module and one reviewer per repository, as the shipped scan builds them."""
|
|
2023
|
+
module = worker("github-pr-reviewer", tmp_path, monkeypatch)
|
|
2024
|
+
monkeypatch.delenv("AUTOMATION_EVENT_PAYLOAD", raising=False)
|
|
2025
|
+
monkeypatch.setattr(
|
|
2026
|
+
module.workflow,
|
|
2027
|
+
"_latest_trigger_label_event",
|
|
2028
|
+
lambda *args: {"id": 7, "created_at": "2026-01-01T00:00:00Z"},
|
|
2029
|
+
)
|
|
2030
|
+
reviewers = []
|
|
2031
|
+
for repository in repositories:
|
|
2032
|
+
run = object.__new__(module.PullRequestReviewer)
|
|
2033
|
+
run.config = {"trigger_label": "openhands-review"}
|
|
2034
|
+
if max_new is not None:
|
|
2035
|
+
run.config["max_new_per_run"] = max_new
|
|
2036
|
+
run.repository = repository
|
|
2037
|
+
run.token = "token"
|
|
2038
|
+
run.token_name = "FACTORY_GITHUB_REVIEWER_TOKEN"
|
|
2039
|
+
run.github_login = "all-hands-bot"
|
|
2040
|
+
run.dispatcher = _DedupeDispatcher()
|
|
2041
|
+
run.check_runs = lambda sha: []
|
|
2042
|
+
run.workflow_runs = lambda sha: []
|
|
2043
|
+
reviewers.append(run)
|
|
2044
|
+
return module, reviewers
|
|
2045
|
+
|
|
2046
|
+
|
|
2047
|
+
def _run_scan(module, reviewers):
|
|
2048
|
+
"""Drive the shipped scheduled-scan control flow over the given reviewers.
|
|
2049
|
+
|
|
2050
|
+
`module.run_scan` is the shipped entrypoint function, so the shared-intake
|
|
2051
|
+
and drain wiring under test is the one that runs in production. Only the
|
|
2052
|
+
repository objects it drives are supplied here: the GitHub transport is
|
|
2053
|
+
already stubbed on each reviewer, so `run_repositories` is replaced by a
|
|
2054
|
+
loop over them in the order a scan would visit the configured repositories.
|
|
2055
|
+
"""
|
|
2056
|
+
dispatcher = reviewers[0].dispatcher if reviewers else None
|
|
2057
|
+
by_repository = {run.repository: run for run in reviewers}
|
|
2058
|
+
|
|
2059
|
+
def fake_run_repositories(automation_type, conversation=None, dispatcher=None):
|
|
2060
|
+
for source in by_repository.values():
|
|
2061
|
+
run = object.__new__(automation_type)
|
|
2062
|
+
run.__dict__.update(source.__dict__)
|
|
2063
|
+
run.run()
|
|
2064
|
+
|
|
2065
|
+
original = module.run_repositories
|
|
2066
|
+
module.run_repositories = fake_run_repositories
|
|
2067
|
+
try:
|
|
2068
|
+
module.run_scan(dispatcher)
|
|
2069
|
+
finally:
|
|
2070
|
+
module.run_repositories = original
|
|
2071
|
+
module.PullRequestReviewer.scan_intake = None
|
|
2072
|
+
|
|
2073
|
+
|
|
2074
|
+
def _eligible_pr(number, sha, request_id, created_at, *, checks=()):
|
|
2075
|
+
"""An open non-draft PR holding an outstanding all-hands-bot request."""
|
|
2076
|
+
return {
|
|
2077
|
+
"number": number,
|
|
2078
|
+
"head": {"sha": sha},
|
|
2079
|
+
"labels": [],
|
|
2080
|
+
"draft": False,
|
|
2081
|
+
"requested_reviewers": [{"login": "all-hands-bot"}],
|
|
2082
|
+
"_request": {
|
|
2083
|
+
"id": request_id,
|
|
2084
|
+
"event": "review_requested",
|
|
2085
|
+
"created_at": created_at,
|
|
2086
|
+
"requested_reviewer": {"login": "all-hands-bot"},
|
|
2087
|
+
},
|
|
2088
|
+
"_checks": list(checks),
|
|
2089
|
+
}
|
|
2090
|
+
|
|
2091
|
+
|
|
2092
|
+
def _unrequested_pr(
|
|
2093
|
+
number,
|
|
2094
|
+
sha,
|
|
2095
|
+
*,
|
|
2096
|
+
created_at="2026-01-01T00:00:00Z",
|
|
2097
|
+
checks=(),
|
|
2098
|
+
workflows=(),
|
|
2099
|
+
reviews=(),
|
|
2100
|
+
author="someone",
|
|
2101
|
+
):
|
|
2102
|
+
"""An open non-draft PR nobody requested, as the unrequested scan sees it."""
|
|
2103
|
+
return {
|
|
2104
|
+
"number": number,
|
|
2105
|
+
"head": {"sha": sha},
|
|
2106
|
+
"labels": [],
|
|
2107
|
+
"draft": False,
|
|
2108
|
+
"created_at": created_at,
|
|
2109
|
+
"user": {"login": author},
|
|
2110
|
+
"requested_reviewers": [],
|
|
2111
|
+
"_checks": list(checks),
|
|
2112
|
+
"_workflows": list(workflows),
|
|
2113
|
+
"_reviews": list(reviews),
|
|
2114
|
+
}
|
|
2115
|
+
|
|
2116
|
+
|
|
2117
|
+
def _wire_scan(run, prs, repository_id):
|
|
2118
|
+
"""Point one reviewer at its PRs, requests, reviews, and runs."""
|
|
2119
|
+
by_number = {pr["number"]: pr for pr in prs}
|
|
2120
|
+
|
|
2121
|
+
def gh(method, path, body=None):
|
|
2122
|
+
if path == "":
|
|
2123
|
+
return {"id": repository_id}
|
|
2124
|
+
if method == "GET" and path.startswith("/pulls/"):
|
|
2125
|
+
return by_number[int(path.split("/")[2])]
|
|
2126
|
+
if method == "POST" and path.endswith("/comments"):
|
|
2127
|
+
return {"id": 1234}
|
|
2128
|
+
return {}
|
|
2129
|
+
|
|
2130
|
+
def gh_pages(path):
|
|
2131
|
+
if path.startswith("/pulls?"):
|
|
2132
|
+
return list(prs)
|
|
2133
|
+
if path.endswith("/reviews"):
|
|
2134
|
+
return list(by_number[int(path.split("/")[2])].get("_reviews", []))
|
|
2135
|
+
if path.endswith("/events"):
|
|
2136
|
+
request = by_number[int(path.split("/")[2])].get("_request")
|
|
2137
|
+
return [request] if request else []
|
|
2138
|
+
return []
|
|
2139
|
+
|
|
2140
|
+
def check_runs(sha):
|
|
2141
|
+
for pr in by_number.values():
|
|
2142
|
+
if pr["head"]["sha"] == sha:
|
|
2143
|
+
return [
|
|
2144
|
+
{
|
|
2145
|
+
"name": name,
|
|
2146
|
+
"status": status,
|
|
2147
|
+
"conclusion": conclusion,
|
|
2148
|
+
"head_sha": sha,
|
|
2149
|
+
}
|
|
2150
|
+
for name, status, conclusion in pr.get("_checks", [])
|
|
2151
|
+
]
|
|
2152
|
+
return []
|
|
2153
|
+
|
|
2154
|
+
def workflow_runs(sha):
|
|
2155
|
+
for pr in by_number.values():
|
|
2156
|
+
if pr["head"]["sha"] == sha:
|
|
2157
|
+
return [
|
|
2158
|
+
{
|
|
2159
|
+
"name": name,
|
|
2160
|
+
"status": status,
|
|
2161
|
+
"conclusion": conclusion,
|
|
2162
|
+
"head_sha": sha,
|
|
2163
|
+
"id": run_id,
|
|
2164
|
+
"check_suite_id": suite_id,
|
|
2165
|
+
"workflow_id": workflow_id,
|
|
2166
|
+
"run_started_at": "",
|
|
2167
|
+
"created_at": "",
|
|
2168
|
+
}
|
|
2169
|
+
for name, status, conclusion, run_id, suite_id, workflow_id in pr.get(
|
|
2170
|
+
"_workflows", []
|
|
2171
|
+
)
|
|
2172
|
+
]
|
|
2173
|
+
return []
|
|
2174
|
+
|
|
2175
|
+
run.gh = gh
|
|
2176
|
+
run.gh_pages = gh_pages
|
|
2177
|
+
run.check_runs = check_runs
|
|
2178
|
+
run.workflow_runs = workflow_runs
|
|
2179
|
+
|
|
2180
|
+
|
|
2181
|
+
def test_one_scan_starts_at_most_the_maximum_and_a_later_scan_reaches_the_rest(
|
|
2182
|
+
tmp_path, monkeypatch
|
|
2183
|
+
):
|
|
2184
|
+
"""Three eligible PRs across two repositories, default maximum of two."""
|
|
2185
|
+
module, (one, two) = _scan_reviewers(
|
|
2186
|
+
tmp_path, monkeypatch, ["owner/one", "owner/two"]
|
|
2187
|
+
)
|
|
2188
|
+
# Oldest request first: #5 (Jan 1), then #2 (Jan 2), then #9 (Jan 3).
|
|
2189
|
+
_wire_scan(
|
|
2190
|
+
one,
|
|
2191
|
+
[
|
|
2192
|
+
_eligible_pr(5, "head-5", 500, "2026-01-01T00:00:00Z"),
|
|
2193
|
+
_eligible_pr(9, "head-9", 900, "2026-01-03T00:00:00Z"),
|
|
2194
|
+
],
|
|
2195
|
+
101,
|
|
2196
|
+
)
|
|
2197
|
+
_wire_scan(two, [_eligible_pr(2, "head-2", 200, "2026-01-02T00:00:00Z")], 102)
|
|
2198
|
+
|
|
2199
|
+
_run_scan(module, [one, two])
|
|
2200
|
+
|
|
2201
|
+
started = set(one.dispatcher.calls) | set(two.dispatcher.calls)
|
|
2202
|
+
assert started == {("101:pr:5", "500:head-5"), ("102:pr:2", "200:head-2")}
|
|
2203
|
+
assert one.dispatcher.seen == {"101:pr:5": "500:head-5"}
|
|
2204
|
+
assert two.dispatcher.seen == {"102:pr:2": "200:head-2"}
|
|
2205
|
+
|
|
2206
|
+
# A later scan sees the first two as already-running and reaches the rest.
|
|
2207
|
+
_run_scan(module, [one, two])
|
|
2208
|
+
|
|
2209
|
+
assert one.dispatcher.seen == {
|
|
2210
|
+
"101:pr:5": "500:head-5",
|
|
2211
|
+
"101:pr:9": "900:head-9",
|
|
2212
|
+
}
|
|
2213
|
+
assert two.dispatcher.seen == {"102:pr:2": "200:head-2"}
|
|
2214
|
+
|
|
2215
|
+
|
|
2216
|
+
def test_a_pending_check_candidate_consumes_no_slot_and_does_not_block_the_queue(
|
|
2217
|
+
tmp_path, monkeypatch
|
|
2218
|
+
):
|
|
2219
|
+
"""The oldest candidate is waiting on CI, so the next two drain instead."""
|
|
2220
|
+
module, (one, two) = _scan_reviewers(
|
|
2221
|
+
tmp_path, monkeypatch, ["owner/one", "owner/two"]
|
|
2222
|
+
)
|
|
2223
|
+
_wire_scan(
|
|
2224
|
+
one,
|
|
2225
|
+
[
|
|
2226
|
+
# Oldest request, but its exact head still has a queued check.
|
|
2227
|
+
_eligible_pr(
|
|
2228
|
+
5,
|
|
2229
|
+
"head-5",
|
|
2230
|
+
500,
|
|
2231
|
+
"2026-01-01T00:00:00Z",
|
|
2232
|
+
checks=[("slow-e2e", "queued", None)],
|
|
2233
|
+
),
|
|
2234
|
+
_eligible_pr(9, "head-9", 900, "2026-01-03T00:00:00Z"),
|
|
2235
|
+
],
|
|
2236
|
+
101,
|
|
2237
|
+
)
|
|
2238
|
+
_wire_scan(two, [_eligible_pr(2, "head-2", 200, "2026-01-02T00:00:00Z")], 102)
|
|
2239
|
+
|
|
2240
|
+
_run_scan(module, [one, two])
|
|
2241
|
+
|
|
2242
|
+
assert one.dispatcher.seen == {"101:pr:9": "900:head-9"}
|
|
2243
|
+
assert two.dispatcher.seen == {"102:pr:2": "200:head-2"}
|
|
2244
|
+
|
|
2245
|
+
|
|
2246
|
+
def test_non_dispatching_work_still_runs_after_the_maximum_is_reached(
|
|
2247
|
+
tmp_path, monkeypatch
|
|
2248
|
+
):
|
|
2249
|
+
"""A blocked candidate past the maximum still gets its gate explanation."""
|
|
2250
|
+
module, (one,) = _scan_reviewers(tmp_path, monkeypatch, ["owner/one"])
|
|
2251
|
+
_wire_scan(
|
|
2252
|
+
one,
|
|
2253
|
+
[
|
|
2254
|
+
_eligible_pr(5, "head-5", 500, "2026-01-01T00:00:00Z"),
|
|
2255
|
+
_eligible_pr(6, "head-6", 600, "2026-01-02T00:00:00Z"),
|
|
2256
|
+
_eligible_pr(
|
|
2257
|
+
7,
|
|
2258
|
+
"head-7",
|
|
2259
|
+
700,
|
|
2260
|
+
"2026-01-04T00:00:00Z",
|
|
2261
|
+
checks=[("ci", "completed", "failure")],
|
|
2262
|
+
),
|
|
2263
|
+
],
|
|
2264
|
+
101,
|
|
2265
|
+
)
|
|
2266
|
+
one.gh = Mock(side_effect=one.gh)
|
|
2267
|
+
|
|
2268
|
+
_run_scan(module, [one])
|
|
2269
|
+
|
|
2270
|
+
# Only the two oldest start; the blocked third consumes no slot.
|
|
2271
|
+
assert set(one.dispatcher.seen) == {"101:pr:5", "101:pr:6"}
|
|
2272
|
+
posted = [call for call in one.gh.call_args_list if call.args[0] == "POST"]
|
|
2273
|
+
assert len(posted) == 1
|
|
2274
|
+
assert "<!-- openhands-review-gate:blocked:head-7 -->" in posted[0].args[2]["body"]
|
|
2275
|
+
|
|
2276
|
+
|
|
2277
|
+
def test_a_failing_dispatch_is_reported_without_consuming_a_slot_or_aborting(
|
|
2278
|
+
tmp_path, monkeypatch
|
|
2279
|
+
):
|
|
2280
|
+
"""The oldest dispatch raises; the next candidates still get their turn."""
|
|
2281
|
+
module, (one,) = _scan_reviewers(tmp_path, monkeypatch, ["owner/one"])
|
|
2282
|
+
_wire_scan(
|
|
2283
|
+
one,
|
|
2284
|
+
[
|
|
2285
|
+
_eligible_pr(5, "head-5", 500, "2026-01-01T00:00:00Z"),
|
|
2286
|
+
_eligible_pr(6, "head-6", 600, "2026-01-02T00:00:00Z"),
|
|
2287
|
+
_eligible_pr(7, "head-7", 700, "2026-01-03T00:00:00Z"),
|
|
2288
|
+
],
|
|
2289
|
+
101,
|
|
2290
|
+
)
|
|
2291
|
+
|
|
2292
|
+
def deliver(*, subject, delivery, prompt, head=""):
|
|
2293
|
+
if subject == "101:pr:5":
|
|
2294
|
+
raise RuntimeError("agent server unavailable")
|
|
2295
|
+
one.dispatcher.seen[subject] = delivery
|
|
2296
|
+
return {"disposition": "created", "conversation_id": subject}
|
|
2297
|
+
|
|
2298
|
+
one.dispatcher.deliver = deliver
|
|
2299
|
+
|
|
2300
|
+
with pytest.raises(RuntimeError, match="#5"):
|
|
2301
|
+
_run_scan(module, [one])
|
|
2302
|
+
|
|
2303
|
+
assert set(one.dispatcher.seen) == {"101:pr:6", "101:pr:7"}
|
|
2304
|
+
|
|
2305
|
+
|
|
2306
|
+
def test_the_maximum_is_configurable_through_the_rendered_config(tmp_path, monkeypatch):
|
|
2307
|
+
module, (one,) = _scan_reviewers(tmp_path, monkeypatch, ["owner/one"], max_new=1)
|
|
2308
|
+
_wire_scan(
|
|
2309
|
+
one,
|
|
2310
|
+
[
|
|
2311
|
+
_eligible_pr(5, "head-5", 500, "2026-01-01T00:00:00Z"),
|
|
2312
|
+
_eligible_pr(6, "head-6", 600, "2026-01-02T00:00:00Z"),
|
|
2313
|
+
],
|
|
2314
|
+
101,
|
|
2315
|
+
)
|
|
2316
|
+
|
|
2317
|
+
_run_scan(module, [one])
|
|
2318
|
+
|
|
2319
|
+
assert set(one.dispatcher.seen) == {"101:pr:5"}
|
|
2320
|
+
|
|
2321
|
+
|
|
2322
|
+
def test_the_rendered_config_defaults_the_maximum_to_two(tmp_path, monkeypatch):
|
|
2323
|
+
"""An unset max_new_per_run still bounds a scheduled scan to two."""
|
|
2324
|
+
module, (one,) = _scan_reviewers(tmp_path, monkeypatch, ["owner/one"])
|
|
2325
|
+
_wire_scan(
|
|
2326
|
+
one,
|
|
2327
|
+
[
|
|
2328
|
+
_eligible_pr(5, "head-5", 500, "2026-01-01T00:00:00Z"),
|
|
2329
|
+
_eligible_pr(6, "head-6", 600, "2026-01-02T00:00:00Z"),
|
|
2330
|
+
_eligible_pr(7, "head-7", 700, "2026-01-03T00:00:00Z"),
|
|
2331
|
+
],
|
|
2332
|
+
101,
|
|
2333
|
+
)
|
|
2334
|
+
|
|
2335
|
+
_run_scan(module, [one])
|
|
2336
|
+
|
|
2337
|
+
assert module.workflow.MAX_NEW_PER_RUN == 2
|
|
2338
|
+
assert set(one.dispatcher.seen) == {"101:pr:5", "101:pr:6"}
|
|
2339
|
+
|
|
2340
|
+
|
|
2341
|
+
def test_the_explicit_request_event_path_is_not_bounded(tmp_path, monkeypatch):
|
|
2342
|
+
"""Only the scheduled scan's new conversations are bounded."""
|
|
2343
|
+
module, (one,) = _scan_reviewers(tmp_path, monkeypatch, ["owner/one"])
|
|
2344
|
+
monkeypatch.setenv(
|
|
2345
|
+
"AUTOMATION_EVENT_PAYLOAD",
|
|
2346
|
+
json.dumps(
|
|
2347
|
+
{
|
|
2348
|
+
"automation_id": "automation",
|
|
2349
|
+
"event": {
|
|
2350
|
+
"payload": {
|
|
2351
|
+
"action": "review_requested",
|
|
2352
|
+
"repository": {"full_name": "owner/one"},
|
|
2353
|
+
"pull_request": {"number": 5},
|
|
2354
|
+
"requested_reviewer": {"login": "all-hands-bot"},
|
|
2355
|
+
}
|
|
2356
|
+
},
|
|
2357
|
+
}
|
|
2358
|
+
),
|
|
2359
|
+
)
|
|
2360
|
+
pr = {"number": 5, "head": {"sha": "head-5"}, "labels": []}
|
|
2361
|
+
one.gh = Mock(side_effect=[{"id": 101}, pr])
|
|
2362
|
+
one.gh_pages = lambda path: (
|
|
2363
|
+
[
|
|
2364
|
+
{
|
|
2365
|
+
"id": 500,
|
|
2366
|
+
"event": "review_requested",
|
|
2367
|
+
"created_at": "2026-01-01T00:00:00Z",
|
|
2368
|
+
"requested_reviewer": {"login": "all-hands-bot"},
|
|
2369
|
+
}
|
|
2370
|
+
]
|
|
2371
|
+
if path.endswith("/events")
|
|
2372
|
+
else []
|
|
2373
|
+
)
|
|
2374
|
+
one.workflow_runs = lambda sha: []
|
|
2375
|
+
|
|
2376
|
+
one.run()
|
|
2377
|
+
|
|
2378
|
+
assert one.dispatcher.seen == {"101:pr:5": "500:head-5"}
|
|
2379
|
+
|
|
2380
|
+
|
|
2381
|
+
def test_the_rendered_config_reads_a_positive_max_new_per_run(tmp_path, monkeypatch):
|
|
2382
|
+
"""The rendered config overrides the default, following the key the other
|
|
2383
|
+
automations already use."""
|
|
2384
|
+
module = worker("github-pr-reviewer", tmp_path, monkeypatch)
|
|
2385
|
+
(tmp_path / "github-pr-reviewer" / "config.json").write_text(
|
|
2386
|
+
json.dumps(
|
|
2387
|
+
{
|
|
2388
|
+
"repos": ["owner/one"],
|
|
2389
|
+
"max_new_per_run": 3,
|
|
2390
|
+
"github_token_secret": "GITHUB_TOKEN",
|
|
2391
|
+
}
|
|
2392
|
+
)
|
|
2393
|
+
)
|
|
2394
|
+
|
|
2395
|
+
config = module.workflow.load_config(tmp_path / "github-pr-reviewer")
|
|
2396
|
+
|
|
2397
|
+
assert config["max_new_per_run"] == 3
|
|
2398
|
+
|
|
2399
|
+
|
|
2400
|
+
@pytest.mark.parametrize("bad", [0, -1, True, "2"])
|
|
2401
|
+
def test_the_rendered_config_rejects_a_misbehaving_max_new_per_run(
|
|
2402
|
+
tmp_path, monkeypatch, bad
|
|
2403
|
+
):
|
|
2404
|
+
"""A non-positive or non-integer bound is a hard error, not a coercion: the
|
|
2405
|
+
alternative is a scan that either never starts or starts `True` conversations."""
|
|
2406
|
+
module = worker("github-pr-reviewer", tmp_path, monkeypatch)
|
|
2407
|
+
(tmp_path / "github-pr-reviewer" / "config.json").write_text(
|
|
2408
|
+
json.dumps({"repos": ["owner/one"], "max_new_per_run": bad})
|
|
2409
|
+
)
|
|
2410
|
+
|
|
2411
|
+
with pytest.raises(SystemExit):
|
|
2412
|
+
module.workflow.load_config(tmp_path / "github-pr-reviewer")
|
|
2413
|
+
|
|
2414
|
+
# --------------------------------------------------------------------------- #
|
|
2415
|
+
# Unrequested scheduled scan: an open, non-draft PR with a green current head and
|
|
2416
|
+
# no all-hands-bot review is reviewed without any request or trigger label.
|
|
2417
|
+
# --------------------------------------------------------------------------- #
|
|
2418
|
+
|
|
2419
|
+
|
|
2420
|
+
def test_unrequested_scan_reviews_a_green_pr_with_no_request_or_label(
|
|
2421
|
+
tmp_path, monkeypatch
|
|
2422
|
+
):
|
|
2423
|
+
"""The core new behavior: a green PR nobody requested starts a review."""
|
|
2424
|
+
module, (one,) = _scan_reviewers(tmp_path, monkeypatch, ["owner/one"])
|
|
2425
|
+
_wire_scan(one, [_unrequested_pr(5, "head-5")], 101)
|
|
2426
|
+
|
|
2427
|
+
_run_scan(module, [one])
|
|
2428
|
+
|
|
2429
|
+
assert one.dispatcher.seen == {"101:pr:5": "scan:owner/one:5:head-5"}
|
|
2430
|
+
|
|
2431
|
+
|
|
2432
|
+
def test_unrequested_scan_blocks_a_failed_zero_job_workflow(tmp_path, monkeypatch):
|
|
2433
|
+
"""A green check-run rollup with a failed zero-job workflow still blocks.
|
|
2434
|
+
|
|
2435
|
+
The workflow failed before creating any check run, so the commit's rollup is
|
|
2436
|
+
green while the Actions run is `completed`/`failure`. The gate must read the
|
|
2437
|
+
workflow run and start nothing - and, because nobody requested this PR, it
|
|
2438
|
+
must do so without posting a managed gate comment.
|
|
2439
|
+
"""
|
|
2440
|
+
module, (one,) = _scan_reviewers(tmp_path, monkeypatch, ["owner/one"])
|
|
2441
|
+
_wire_scan(
|
|
2442
|
+
one,
|
|
2443
|
+
[
|
|
2444
|
+
_unrequested_pr(
|
|
2445
|
+
5,
|
|
2446
|
+
"head-5",
|
|
2447
|
+
checks=(("pr-title", "completed", "success"),),
|
|
2448
|
+
workflows=(("Tests", "completed", "failure", 3, 3003, 236324519),),
|
|
2449
|
+
)
|
|
2450
|
+
],
|
|
2451
|
+
101,
|
|
2452
|
+
)
|
|
2453
|
+
one.gh = Mock(side_effect=one.gh)
|
|
2454
|
+
|
|
2455
|
+
_run_scan(module, [one])
|
|
2456
|
+
|
|
2457
|
+
assert one.dispatcher.seen == {}
|
|
2458
|
+
assert [call for call in one.gh.call_args_list if call.args[0] == "POST"] == []
|
|
2459
|
+
|
|
2460
|
+
|
|
2461
|
+
def test_unrequested_scan_waits_on_a_pending_workflow(tmp_path, monkeypatch):
|
|
2462
|
+
"""A current-head workflow still in progress makes the unrequested head wait.
|
|
2463
|
+
|
|
2464
|
+
The unrequested head starts no conversation and, being unrequested, gets no
|
|
2465
|
+
managed gate comment either.
|
|
2466
|
+
"""
|
|
2467
|
+
module, (one,) = _scan_reviewers(tmp_path, monkeypatch, ["owner/one"])
|
|
2468
|
+
_wire_scan(
|
|
2469
|
+
one,
|
|
2470
|
+
[
|
|
2471
|
+
_unrequested_pr(
|
|
2472
|
+
5,
|
|
2473
|
+
"head-5",
|
|
2474
|
+
workflows=(("Tests", "in_progress", None, 4, 4004, 236324519),),
|
|
2475
|
+
)
|
|
2476
|
+
],
|
|
2477
|
+
101,
|
|
2478
|
+
)
|
|
2479
|
+
one.gh = Mock(side_effect=one.gh)
|
|
2480
|
+
|
|
2481
|
+
_run_scan(module, [one])
|
|
2482
|
+
|
|
2483
|
+
assert one.dispatcher.seen == {}
|
|
2484
|
+
assert [call for call in one.gh.call_args_list if call.args[0] == "POST"] == []
|
|
2485
|
+
|
|
2486
|
+
|
|
2487
|
+
def test_unrequested_scan_does_not_duplicate_across_repeated_scans(
|
|
2488
|
+
tmp_path, monkeypatch
|
|
2489
|
+
):
|
|
2490
|
+
"""The stable repository/PR/head key makes a repeat scan a no-op."""
|
|
2491
|
+
module, (one,) = _scan_reviewers(tmp_path, monkeypatch, ["owner/one"])
|
|
2492
|
+
_wire_scan(one, [_unrequested_pr(5, "head-5")], 101)
|
|
2493
|
+
|
|
2494
|
+
_run_scan(module, [one])
|
|
2495
|
+
_run_scan(module, [one])
|
|
2496
|
+
|
|
2497
|
+
assert one.dispatcher.calls == [
|
|
2498
|
+
("101:pr:5", "scan:owner/one:5:head-5"),
|
|
2499
|
+
("101:pr:5", "scan:owner/one:5:head-5"),
|
|
2500
|
+
]
|
|
2501
|
+
assert one.dispatcher.seen == {"101:pr:5": "scan:owner/one:5:head-5"}
|
|
2502
|
+
|
|
2503
|
+
|
|
2504
|
+
def test_unrequested_scan_reviews_a_changed_head_again_under_a_new_key(
|
|
2505
|
+
tmp_path, monkeypatch
|
|
2506
|
+
):
|
|
2507
|
+
"""A new head is eligible again and reviewed once under its own key."""
|
|
2508
|
+
module, (one,) = _scan_reviewers(tmp_path, monkeypatch, ["owner/one"])
|
|
2509
|
+
_wire_scan(one, [_unrequested_pr(5, "head-5")], 101)
|
|
2510
|
+
_run_scan(module, [one])
|
|
2511
|
+
|
|
2512
|
+
# The PR is pushed to a new head, so the old key no longer matches.
|
|
2513
|
+
_wire_scan(one, [_unrequested_pr(5, "head-6")], 101)
|
|
2514
|
+
_run_scan(module, [one])
|
|
2515
|
+
|
|
2516
|
+
assert one.dispatcher.seen == {"101:pr:5": "scan:owner/one:5:head-6"}
|
|
2517
|
+
assert one.dispatcher.calls[-1] == ("101:pr:5", "scan:owner/one:5:head-6")
|
|
2518
|
+
|
|
2519
|
+
|
|
2520
|
+
def test_unrequested_scan_bound_spans_repositories_with_explicit_priority(
|
|
2521
|
+
tmp_path, monkeypatch
|
|
2522
|
+
):
|
|
2523
|
+
"""The cap is global; an explicit request outranks unrequested candidates.
|
|
2524
|
+
|
|
2525
|
+
Repository `owner/one` holds an unrequested PR older than `owner/two`'s
|
|
2526
|
+
explicit request. The request must win the single slot even though it is
|
|
2527
|
+
younger, because explicit requests are ordered first.
|
|
2528
|
+
"""
|
|
2529
|
+
module, (one, two) = _scan_reviewers(
|
|
2530
|
+
tmp_path, monkeypatch, ["owner/one", "owner/two"], max_new=1
|
|
2531
|
+
)
|
|
2532
|
+
_wire_scan(
|
|
2533
|
+
one, [_unrequested_pr(5, "head-5", created_at="2026-01-01T00:00:00Z")], 101
|
|
2534
|
+
)
|
|
2535
|
+
_wire_scan(two, [_eligible_pr(2, "head-2", 200, "2026-01-05T00:00:00Z")], 102)
|
|
2536
|
+
|
|
2537
|
+
_run_scan(module, [one, two])
|
|
2538
|
+
|
|
2539
|
+
assert one.dispatcher.seen == {}
|
|
2540
|
+
assert two.dispatcher.seen == {"102:pr:2": "200:head-2"}
|
|
2541
|
+
|
|
2542
|
+
|
|
2543
|
+
def test_unrequested_scan_reviews_the_oldest_eligible_prs_under_the_cap(
|
|
2544
|
+
tmp_path, monkeypatch
|
|
2545
|
+
):
|
|
2546
|
+
"""With no requests, the cap starts the oldest unrequested PRs first."""
|
|
2547
|
+
module, (one,) = _scan_reviewers(tmp_path, monkeypatch, ["owner/one"])
|
|
2548
|
+
_wire_scan(
|
|
2549
|
+
one,
|
|
2550
|
+
[
|
|
2551
|
+
_unrequested_pr(9, "head-9", created_at="2026-01-03T00:00:00Z"),
|
|
2552
|
+
_unrequested_pr(5, "head-5", created_at="2026-01-01T00:00:00Z"),
|
|
2553
|
+
_unrequested_pr(7, "head-7", created_at="2026-01-02T00:00:00Z"),
|
|
2554
|
+
],
|
|
2555
|
+
101,
|
|
2556
|
+
)
|
|
2557
|
+
|
|
2558
|
+
_run_scan(module, [one])
|
|
2559
|
+
|
|
2560
|
+
assert one.dispatcher.seen == {
|
|
2561
|
+
"101:pr:5": "scan:owner/one:5:head-5",
|
|
2562
|
+
"101:pr:7": "scan:owner/one:7:head-7",
|
|
2563
|
+
}
|
|
2564
|
+
|
|
2565
|
+
|
|
2566
|
+
def test_unrequested_scan_reconciles_a_completed_review_and_hands_off(
|
|
2567
|
+
tmp_path, monkeypatch
|
|
2568
|
+
):
|
|
2569
|
+
"""A head that already carries a completed review is not re-dispatched.
|
|
2570
|
+
|
|
2571
|
+
The review is found on the current head, so the scan reconciles it and runs
|
|
2572
|
+
the existing maintainer handoff instead of starting a second conversation.
|
|
2573
|
+
"""
|
|
2574
|
+
module, (one,) = _scan_reviewers(tmp_path, monkeypatch, ["owner/one"])
|
|
2575
|
+
_wire_scan(
|
|
2576
|
+
one,
|
|
2577
|
+
[
|
|
2578
|
+
_unrequested_pr(
|
|
2579
|
+
5,
|
|
2580
|
+
"head-5",
|
|
2581
|
+
reviews=[
|
|
2582
|
+
{
|
|
2583
|
+
"body": "Review body\n\n✅ APPROVED",
|
|
2584
|
+
"commit_id": "head-5",
|
|
2585
|
+
"submitted_at": "2026-01-02T00:00:00Z",
|
|
2586
|
+
"user": {"login": "all-hands-bot"},
|
|
2587
|
+
}
|
|
2588
|
+
],
|
|
2589
|
+
)
|
|
2590
|
+
],
|
|
2591
|
+
101,
|
|
2592
|
+
)
|
|
2593
|
+
one.config["maintainers"] = "neubig"
|
|
2594
|
+
handoff = Mock(return_value="neubig")
|
|
2595
|
+
monkeypatch.setattr(module, "request_maintainer_review", handoff)
|
|
2596
|
+
|
|
2597
|
+
_run_scan(module, [one])
|
|
2598
|
+
|
|
2599
|
+
assert one.dispatcher.seen == {}
|
|
2600
|
+
handoff.assert_called_once()
|
|
2601
|
+
|
|
2602
|
+
|
|
2603
|
+
def test_unrequested_scan_still_gates_a_blocked_pr_past_the_cap(
|
|
2604
|
+
tmp_path, monkeypatch
|
|
2605
|
+
):
|
|
2606
|
+
"""An ineligible unrequested PR past the cap consumes no slot and stays silent.
|
|
2607
|
+
|
|
2608
|
+
Two green unrequested PRs fill the cap, and a third blocked one - older than
|
|
2609
|
+
them - is skipped without consuming a slot, without aborting the scan, and
|
|
2610
|
+
without a managed gate comment, because nobody requested it.
|
|
2611
|
+
"""
|
|
2612
|
+
module, (one,) = _scan_reviewers(tmp_path, monkeypatch, ["owner/one"])
|
|
2613
|
+
_wire_scan(
|
|
2614
|
+
one,
|
|
2615
|
+
[
|
|
2616
|
+
_unrequested_pr(
|
|
2617
|
+
5,
|
|
2618
|
+
"head-5",
|
|
2619
|
+
created_at="2026-01-01T00:00:00Z",
|
|
2620
|
+
checks=(("ci", "completed", "failure"),),
|
|
2621
|
+
),
|
|
2622
|
+
_unrequested_pr(6, "head-6", created_at="2026-01-02T00:00:00Z"),
|
|
2623
|
+
_unrequested_pr(7, "head-7", created_at="2026-01-03T00:00:00Z"),
|
|
2624
|
+
],
|
|
2625
|
+
101,
|
|
2626
|
+
)
|
|
2627
|
+
one.gh = Mock(side_effect=one.gh)
|
|
2628
|
+
|
|
2629
|
+
_run_scan(module, [one])
|
|
2630
|
+
|
|
2631
|
+
assert set(one.dispatcher.seen) == {"101:pr:6", "101:pr:7"}
|
|
2632
|
+
assert [call for call in one.gh.call_args_list if call.args[0] == "POST"] == []
|
|
2633
|
+
|
|
2634
|
+
|
|
2635
|
+
def test_an_explicit_request_still_gets_its_managed_gate_comment(
|
|
2636
|
+
tmp_path, monkeypatch
|
|
2637
|
+
):
|
|
2638
|
+
"""The managed comment answers an explicit request, red or pending.
|
|
2639
|
+
|
|
2640
|
+
Suppressing the unrequested comment must not silence the explanation for the
|
|
2641
|
+
request a caller actually made: an outstanding request whose head is failing
|
|
2642
|
+
still gets its blocked gate comment.
|
|
2643
|
+
"""
|
|
2644
|
+
module, (one,) = _scan_reviewers(tmp_path, monkeypatch, ["owner/one"])
|
|
2645
|
+
_wire_scan(
|
|
2646
|
+
one,
|
|
2647
|
+
[
|
|
2648
|
+
_eligible_pr(
|
|
2649
|
+
5,
|
|
2650
|
+
"head-5",
|
|
2651
|
+
500,
|
|
2652
|
+
"2026-01-01T00:00:00Z",
|
|
2653
|
+
checks=(("ci", "completed", "failure"),),
|
|
2654
|
+
)
|
|
2655
|
+
],
|
|
2656
|
+
101,
|
|
2657
|
+
)
|
|
2658
|
+
one.gh = Mock(side_effect=one.gh)
|
|
2659
|
+
|
|
2660
|
+
_run_scan(module, [one])
|
|
2661
|
+
|
|
2662
|
+
assert one.dispatcher.seen == {}
|
|
2663
|
+
posted = [call for call in one.gh.call_args_list if call.args[0] == "POST"]
|
|
2664
|
+
assert len(posted) == 1
|
|
2665
|
+
assert "<!-- openhands-review-gate:blocked:head-5 -->" in posted[0].args[2]["body"]
|
|
2666
|
+
|
|
2667
|
+
|
|
2668
|
+
def test_unrequested_scan_marks_a_self_authored_pr_for_the_comment_verdict(
|
|
2669
|
+
tmp_path, monkeypatch
|
|
2670
|
+
):
|
|
2671
|
+
"""A bot-authored PR is told to use the non-approval verdict path.
|
|
2672
|
+
|
|
2673
|
+
GitHub ignores a self-review request, so the reviewer must publish the clean
|
|
2674
|
+
review as a `COMMENT` event that keeps the approved verdict instead of being
|
|
2675
|
+
skipped.
|
|
2676
|
+
"""
|
|
2677
|
+
module, (one,) = _scan_reviewers(tmp_path, monkeypatch, ["owner/one"])
|
|
2678
|
+
_wire_scan(one, [_unrequested_pr(5, "head-5", author="all-hands-bot")], 101)
|
|
2679
|
+
|
|
2680
|
+
_run_scan(module, [one])
|
|
2681
|
+
|
|
2682
|
+
assert one.dispatcher.seen == {"101:pr:5": "scan:owner/one:5:head-5"}
|
|
2683
|
+
prompt = one.dispatcher.prompts[-1]
|
|
2684
|
+
assert "authored by the configured reviewer account" in prompt
|
|
2685
|
+
assert "keep the approved verdict" in prompt
|
|
2686
|
+
|
|
2687
|
+
|
|
2688
|
+
# --------------------------------------------------------------------------- #
|
|
2689
|
+
# Comprehensive scheduled scan: inspect the backlog, cap only review launches.
|
|
2690
|
+
# --------------------------------------------------------------------------- #
|
|
2691
|
+
|
|
2692
|
+
|
|
2693
|
+
def _many_unrequested(count, *, checks=()):
|
|
2694
|
+
"""`count` open, non-draft PRs nobody requested, numbered 1..count."""
|
|
2695
|
+
return [
|
|
2696
|
+
_unrequested_pr(
|
|
2697
|
+
number,
|
|
2698
|
+
f"head-{number}",
|
|
2699
|
+
created_at=f"2026-01-{number:02d}T00:00:00Z",
|
|
2700
|
+
checks=checks,
|
|
2701
|
+
)
|
|
2702
|
+
for number in range(1, count + 1)
|
|
2703
|
+
]
|
|
2704
|
+
|
|
2705
|
+
|
|
2706
|
+
def _examined(run):
|
|
2707
|
+
"""The PR numbers a scan actually dispatched, in call order."""
|
|
2708
|
+
return [int(subject.split(":")[-1]) for subject, _ in run.dispatcher.calls]
|
|
2709
|
+
|
|
2710
|
+
|
|
2711
|
+
def test_scan_skips_blocked_heads_until_it_fills_the_review_quota(
|
|
2712
|
+
tmp_path, monkeypatch
|
|
2713
|
+
):
|
|
2714
|
+
"""The quota limits launches, not how many backlog entries are inspected.
|
|
2715
|
+
|
|
2716
|
+
Ten blocked heads at the front of the backlog must not consume the ten-review
|
|
2717
|
+
launch quota or prevent the ten green heads behind them from being reviewed.
|
|
2718
|
+
"""
|
|
2719
|
+
module, (one,) = _scan_reviewers(tmp_path, monkeypatch, ["owner/one"], max_new=10)
|
|
2720
|
+
blocked = [
|
|
2721
|
+
_unrequested_pr(
|
|
2722
|
+
number,
|
|
2723
|
+
f"head-{number}",
|
|
2724
|
+
created_at=f"2026-01-{number:02d}T00:00:00Z",
|
|
2725
|
+
checks=(("ci", "completed", "failure"),),
|
|
2726
|
+
)
|
|
2727
|
+
for number in range(1, 11)
|
|
2728
|
+
]
|
|
2729
|
+
green = [
|
|
2730
|
+
_unrequested_pr(
|
|
2731
|
+
number,
|
|
2732
|
+
f"head-{number}",
|
|
2733
|
+
created_at=f"2026-01-{number:02d}T00:00:00Z",
|
|
2734
|
+
)
|
|
2735
|
+
for number in range(11, 21)
|
|
2736
|
+
]
|
|
2737
|
+
_wire_scan(one, blocked + green, 101)
|
|
2738
|
+
|
|
2739
|
+
_run_scan(module, [one])
|
|
2740
|
+
|
|
2741
|
+
assert _examined(one) == list(range(11, 21))
|
|
2742
|
+
|
|
2743
|
+
|
|
2744
|
+
def test_unrequested_red_and_pending_prs_post_no_gate_comments(
|
|
2745
|
+
tmp_path, monkeypatch
|
|
2746
|
+
):
|
|
2747
|
+
"""A scan over a large unrequested backlog posts no managed gate comments.
|
|
2748
|
+
|
|
2749
|
+
This is the storm the canary exposed: every red or pending head used to get a
|
|
2750
|
+
comment. Unrequested heads now stay silent; only an explicit request gets the
|
|
2751
|
+
managed explanation.
|
|
2752
|
+
"""
|
|
2753
|
+
module, (one,) = _scan_reviewers(tmp_path, monkeypatch, ["owner/one"], max_new=5)
|
|
2754
|
+
_wire_scan(
|
|
2755
|
+
one,
|
|
2756
|
+
[
|
|
2757
|
+
_unrequested_pr(
|
|
2758
|
+
5, "head-5", checks=(("ci", "completed", "failure"),)
|
|
2759
|
+
),
|
|
2760
|
+
_unrequested_pr(
|
|
2761
|
+
6, "head-6", checks=(("slow-e2e", "in_progress", None),)
|
|
2762
|
+
),
|
|
2763
|
+
_unrequested_pr(
|
|
2764
|
+
7, "head-7", checks=(("ci", "completed", "success"),)
|
|
2765
|
+
),
|
|
2766
|
+
],
|
|
2767
|
+
101,
|
|
2768
|
+
)
|
|
2769
|
+
one.gh = Mock(side_effect=one.gh)
|
|
2770
|
+
|
|
2771
|
+
_run_scan(module, [one])
|
|
2772
|
+
|
|
2773
|
+
assert [call for call in one.gh.call_args_list if call.args[0] == "POST"] == []
|
|
2774
|
+
# The green unrequested head still reaches the bounded dispatch queue.
|
|
2775
|
+
assert one.dispatcher.seen == {"101:pr:7": "scan:owner/one:7:head-7"}
|
|
2776
|
+
|
|
2777
|
+
|
|
2778
|
+
def test_scan_reads_every_pull_request_but_starts_only_the_quota(
|
|
2779
|
+
tmp_path, monkeypatch
|
|
2780
|
+
):
|
|
2781
|
+
"""Classification is comprehensive while launches remain bounded."""
|
|
2782
|
+
module, (one,) = _scan_reviewers(tmp_path, monkeypatch, ["owner/one"], max_new=5)
|
|
2783
|
+
_wire_scan(one, _many_unrequested(40), 101)
|
|
2784
|
+
one.gh = Mock(side_effect=one.gh)
|
|
2785
|
+
|
|
2786
|
+
_run_scan(module, [one])
|
|
2787
|
+
|
|
2788
|
+
full_reads = [
|
|
2789
|
+
call
|
|
2790
|
+
for call in one.gh.call_args_list
|
|
2791
|
+
if call.args[0] == "GET" and str(call.args[1]).startswith("/pulls/")
|
|
2792
|
+
]
|
|
2793
|
+
assert len(full_reads) == 40
|
|
2794
|
+
assert _examined(one) == [1, 2, 3, 4, 5]
|