@muggleai/works 5.12.0-staging.85 → 5.12.0-staging.87
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/plugin/scripts/pr-watch-arm.sh +117 -0
- package/dist/plugin/scripts/pr-watch-fetch.sh +56 -0
- package/dist/plugin/scripts/pr-watch-guards.sh +18 -0
- package/dist/plugin/scripts/pr-watch-loop.sh +19 -50
- package/dist/plugin/skills/muggle-pr-followup/arm-watcher.md +15 -3
- package/dist/release-manifest.json +3 -3
- package/package.json +1 -1
- package/plugin/scripts/pr-watch-arm.sh +117 -0
- package/plugin/scripts/pr-watch-fetch.sh +56 -0
- package/plugin/scripts/pr-watch-guards.sh +18 -0
- package/plugin/scripts/pr-watch-loop.sh +19 -50
- package/plugin/skills/muggle-pr-followup/arm-watcher.md +15 -3
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# Arm the watch on one pull request: read its state once, report what is already
|
|
3
|
+
# outstanding, seed the watermark from that same read, and start the loop.
|
|
4
|
+
#
|
|
5
|
+
# Arming used to be three prose steps a caller performed by hand, and skipping
|
|
6
|
+
# the middle one — writing watch-watermark.env — produced a loop that held its
|
|
7
|
+
# PID lease and touched its heartbeat while polling nothing, reporting nothing
|
|
8
|
+
# and never reaching its terminal check. Every health signal said fine. Doing
|
|
9
|
+
# the three steps here, in one command, is what makes that unskippable.
|
|
10
|
+
#
|
|
11
|
+
# The floors are taken from the drain's own read rather than from a fresh fetch
|
|
12
|
+
# afterwards: anything that lands between the two would otherwise be marked seen
|
|
13
|
+
# without ever being reported.
|
|
14
|
+
set -uo pipefail
|
|
15
|
+
|
|
16
|
+
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
17
|
+
|
|
18
|
+
slot="" repo="" pr_number="" base_branch="" exec_loop=1
|
|
19
|
+
|
|
20
|
+
while [ $# -gt 0 ]; do
|
|
21
|
+
case "$1" in
|
|
22
|
+
--slot) slot="$2"; shift 2 ;;
|
|
23
|
+
--repo) repo="$2"; shift 2 ;;
|
|
24
|
+
--pr) pr_number="$2"; shift 2 ;;
|
|
25
|
+
--base) base_branch="$2"; shift 2 ;;
|
|
26
|
+
--no-exec) exec_loop=0; shift ;;
|
|
27
|
+
*) echo "pr-watch-arm: unknown argument $1" >&2; exit 2 ;;
|
|
28
|
+
esac
|
|
29
|
+
done
|
|
30
|
+
|
|
31
|
+
if [ -z "$slot" ] || [ -z "$repo" ] || [ -z "$pr_number" ] || [ -z "$base_branch" ]; then
|
|
32
|
+
echo "usage: pr-watch-arm.sh --slot <dir> --repo <owner/repo> --pr <n> --base <branch> [--no-exec]" >&2
|
|
33
|
+
exit 2
|
|
34
|
+
fi
|
|
35
|
+
|
|
36
|
+
for lib in pr-watch-guards.sh pr-watch-events.sh pr-watch-fetch.sh; do
|
|
37
|
+
if [ ! -f "${script_dir}/${lib}" ]; then
|
|
38
|
+
echo "pr-watch-arm: missing ${lib} beside this script" >&2
|
|
39
|
+
exit 2
|
|
40
|
+
fi
|
|
41
|
+
# shellcheck source=/dev/null
|
|
42
|
+
. "${script_dir}/${lib}"
|
|
43
|
+
done
|
|
44
|
+
|
|
45
|
+
if [ ! -f "${script_dir}/pr-watch-state.jq" ]; then
|
|
46
|
+
echo "pr-watch-arm: missing pr-watch-state.jq beside this script" >&2
|
|
47
|
+
exit 2
|
|
48
|
+
fi
|
|
49
|
+
state_projection="$(cat "${script_dir}/pr-watch-state.jq")"
|
|
50
|
+
|
|
51
|
+
mkdir -p "$slot"
|
|
52
|
+
|
|
53
|
+
pinned_token="$(gh auth token 2>/dev/null)"
|
|
54
|
+
[ -n "$pinned_token" ] && export GH_TOKEN="$pinned_token"
|
|
55
|
+
|
|
56
|
+
state_line="$(watch_fetch_state "$repo" "$pr_number" "$slot" "$state_projection")"
|
|
57
|
+
if [ -z "$state_line" ]; then
|
|
58
|
+
echo "ARM-FAIL pr=$pr_number could not read PR state — refusing to arm a watch with no floors"
|
|
59
|
+
exit 1
|
|
60
|
+
fi
|
|
61
|
+
|
|
62
|
+
mapfile -t fields < <(watch_split_state "$state_line")
|
|
63
|
+
pr_state="${fields[0]-}"
|
|
64
|
+
head_sha="${fields[1]-}"
|
|
65
|
+
base_sha="${fields[2]-}"
|
|
66
|
+
mergeable="${fields[3]-}"
|
|
67
|
+
latest_review="${fields[4]-}"
|
|
68
|
+
latest_comment="${fields[5]-}"
|
|
69
|
+
unresolved_threads="${fields[6]-}"
|
|
70
|
+
pending_checks="${fields[7]-}"
|
|
71
|
+
failed_checks="${fields[8]-}"
|
|
72
|
+
|
|
73
|
+
if [ "$pr_state" = "MERGED" ] || [ "$pr_state" = "CLOSED" ]; then
|
|
74
|
+
echo "TERMINAL pr=$pr_number state=$pr_state — nothing to arm"
|
|
75
|
+
exit 0
|
|
76
|
+
fi
|
|
77
|
+
|
|
78
|
+
# A red head at arm time is the drain's to hand over, so it goes into the floor
|
|
79
|
+
# and does not re-fire on the loop's first pass. Pending checks are not red yet.
|
|
80
|
+
ci_red_floor=""
|
|
81
|
+
if [ "${pending_checks:-0}" -eq 0 ] 2>/dev/null && [ "${failed_checks:-0}" -gt 0 ] 2>/dev/null; then
|
|
82
|
+
ci_red_floor="$head_sha"
|
|
83
|
+
fi
|
|
84
|
+
|
|
85
|
+
# Same for staleness: keyed on the head/base pair, so it re-arms when either moves.
|
|
86
|
+
rebase_floor=""
|
|
87
|
+
behind_count="$(watch_fetch_behind_count "$repo" "$base_branch" "$head_sha" "$slot")"
|
|
88
|
+
if [ "$mergeable" = "CONFLICTING" ] || { [ -n "$behind_count" ] && [ "$behind_count" -gt 0 ] 2>/dev/null; }; then
|
|
89
|
+
rebase_floor="${head_sha}..${base_sha}"
|
|
90
|
+
fi
|
|
91
|
+
|
|
92
|
+
thread_count=0
|
|
93
|
+
[ -n "$unresolved_threads" ] && thread_count=$(printf '%s' "$unresolved_threads" | tr ';' '\n' | grep -c .)
|
|
94
|
+
|
|
95
|
+
# Printed, not swallowed: these are the things the arming session owes a decision
|
|
96
|
+
# on before the monitor takes over, and the monitor only ever sees what arrives
|
|
97
|
+
# after this point.
|
|
98
|
+
echo "DRAIN pr=$pr_number state=$pr_state head=${head_sha:0:8}"
|
|
99
|
+
echo "DRAIN unresolved-threads=$thread_count latest-review=${latest_review:-0} latest-comment=${latest_comment:-0}"
|
|
100
|
+
echo "DRAIN checks pending=${pending_checks:-0} failed=${failed_checks:-0} behind-base=${behind_count:-unknown} mergeable=${mergeable:-UNKNOWN}"
|
|
101
|
+
[ -n "$ci_red_floor" ] && echo "DRAIN ci already red at arm time — handled by the arming session, floored"
|
|
102
|
+
[ -n "$rebase_floor" ] && echo "DRAIN branch already stale at arm time — handled by the arming session, floored"
|
|
103
|
+
|
|
104
|
+
{
|
|
105
|
+
printf 'REV=%s\n' "${latest_review:-0}"
|
|
106
|
+
printf 'COM=%s\n' "${latest_comment:-0}"
|
|
107
|
+
printf 'THREADS="%s"\n' "$unresolved_threads"
|
|
108
|
+
printf 'CIRED="%s"\n' "$ci_red_floor"
|
|
109
|
+
printf 'REBASED="%s"\n' "$rebase_floor"
|
|
110
|
+
printf 'BLOCKED_CIDIGEST=""\n'
|
|
111
|
+
} > "${slot}/watch-watermark.env"
|
|
112
|
+
|
|
113
|
+
echo "ARMED pr=$pr_number watermark seeded at ${slot}/watch-watermark.env"
|
|
114
|
+
|
|
115
|
+
[ "$exec_loop" -eq 1 ] || exit 0
|
|
116
|
+
|
|
117
|
+
exec bash "${script_dir}/pr-watch-loop.sh" --slot "$slot" --repo "$repo" --pr "$pr_number" --base "$base_branch"
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# One read of a pull request's watch-relevant state, shared by the loop that
|
|
3
|
+
# polls it and the arm step that seeds its watermark.
|
|
4
|
+
#
|
|
5
|
+
# Both must see the PR through exactly the same projection: the watermark is a
|
|
6
|
+
# set of floors taken from these fields, and a floor read through a different
|
|
7
|
+
# shape than the one it is later compared against is worse than no floor at all.
|
|
8
|
+
# Keeping the query in one place is what makes "seed from the drain's own read"
|
|
9
|
+
# a fact rather than an instruction.
|
|
10
|
+
|
|
11
|
+
watch_fetch_state() {
|
|
12
|
+
local repo="$1" pr_number="$2" slot="$3" state_projection="$4"
|
|
13
|
+
gh api graphql -F owner="${repo%%/*}" -F name="${repo##*/}" -F number="$pr_number" -f query='
|
|
14
|
+
query($owner: String!, $name: String!, $number: Int!) {
|
|
15
|
+
repository(owner: $owner, name: $name) {
|
|
16
|
+
pullRequest(number: $number) {
|
|
17
|
+
state
|
|
18
|
+
headRefOid
|
|
19
|
+
baseRefOid
|
|
20
|
+
mergeable
|
|
21
|
+
commits(last: 1) {
|
|
22
|
+
nodes {
|
|
23
|
+
commit {
|
|
24
|
+
statusCheckRollup {
|
|
25
|
+
contexts(first: 100) {
|
|
26
|
+
nodes {
|
|
27
|
+
__typename
|
|
28
|
+
... on CheckRun { name status conclusion }
|
|
29
|
+
... on StatusContext { context state }
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
reviews(last: 20, states: [COMMENTED, APPROVED, CHANGES_REQUESTED, DISMISSED]) { nodes { databaseId body } }
|
|
37
|
+
reviewThreads(first: 100) {
|
|
38
|
+
nodes {
|
|
39
|
+
id
|
|
40
|
+
isResolved
|
|
41
|
+
isOutdated
|
|
42
|
+
comments(last: 1) { nodes { databaseId body pullRequestReview { databaseId state } } }
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
}' --jq "$state_projection" 2>>"${slot}/watch-fetch.log"
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
# behind_by needs its own call — see watch_wake_rebase for why no field on the
|
|
51
|
+
# PR carries it. The loop makes it only when the head/base pair moved, so the
|
|
52
|
+
# steady state stays one request per iteration.
|
|
53
|
+
watch_fetch_behind_count() {
|
|
54
|
+
local repo="$1" base_branch="$2" head_sha="$3" slot="$4"
|
|
55
|
+
gh api "repos/${repo}/compare/${base_branch}...${head_sha}" --jq '.behind_by' 2>>"${slot}/watch-fetch.log"
|
|
56
|
+
}
|
|
@@ -30,6 +30,15 @@ MUGGLE_PR_WATCH_POLL_INTERVAL="${MUGGLE_PR_WATCH_POLL_INTERVAL:-60}"
|
|
|
30
30
|
# revoked auth) exhausts it.
|
|
31
31
|
MUGGLE_PR_WATCH_MAX_FETCH_FAILURES="${MUGGLE_PR_WATCH_MAX_FETCH_FAILURES:-60}"
|
|
32
32
|
|
|
33
|
+
# Seconds a loop will wait for the arming session to write watch-watermark.env
|
|
34
|
+
# before giving up. The loop cannot evaluate a single wake condition without it,
|
|
35
|
+
# so an unseeded slot is a watcher that will never report anything — and one that
|
|
36
|
+
# looks perfectly healthy while it does nothing, because it still holds its PID
|
|
37
|
+
# lease and still touches its heartbeat. Arming writes the file in the same turn
|
|
38
|
+
# it starts the loop, so anything past a couple of minutes means the arming
|
|
39
|
+
# sequence was not followed and the watch is inert.
|
|
40
|
+
MUGGLE_PR_WATCH_MAX_UNSEEDED="${MUGGLE_PR_WATCH_MAX_UNSEEDED:-180}"
|
|
41
|
+
|
|
33
42
|
# Seconds to sleep after `fails` consecutive failed fetches: the poll interval,
|
|
34
43
|
# then a linear back-off capped at 5 minutes so a sustained outage is retried
|
|
35
44
|
# calmly rather than hammered every 60s.
|
|
@@ -59,6 +68,15 @@ watcher_lifetime_exceeded() {
|
|
|
59
68
|
[ $((now - started)) -ge "$max" ]
|
|
60
69
|
}
|
|
61
70
|
|
|
71
|
+
# True when the slot has gone unseeded longer than the cap allows. 0 is
|
|
72
|
+
# unbounded, matching `watcher_lifetime_exceeded`, for a caller that seeds the
|
|
73
|
+
# watermark out of band.
|
|
74
|
+
watcher_unseeded_too_long() {
|
|
75
|
+
local waited="$1" max="${2:-$MUGGLE_PR_WATCH_MAX_UNSEEDED}"
|
|
76
|
+
[ "$max" -eq 0 ] 2>/dev/null && return 1
|
|
77
|
+
[ "$waited" -ge "$max" ]
|
|
78
|
+
}
|
|
79
|
+
|
|
62
80
|
# True when pid names a running process. `kill -0` sends no signal; EPERM means
|
|
63
81
|
# the process exists but is foreign, which still counts as alive. Used by
|
|
64
82
|
# arm-watcher's pre-arm dedup to decide whether a watcher already owns the slot.
|
|
@@ -39,7 +39,7 @@ fi
|
|
|
39
39
|
# gone. Guards missing means the plugin moved or upgraded underneath this loop —
|
|
40
40
|
# a newer version's watcher owns the slot now, so step down rather than run on
|
|
41
41
|
# without the supersede check.
|
|
42
|
-
for lib in pr-watch-guards.sh pr-watch-events.sh; do
|
|
42
|
+
for lib in pr-watch-guards.sh pr-watch-events.sh pr-watch-fetch.sh; do
|
|
43
43
|
[ -f "${script_dir}/${lib}" ] || exit 0
|
|
44
44
|
# shellcheck source=/dev/null
|
|
45
45
|
. "${script_dir}/${lib}"
|
|
@@ -54,6 +54,7 @@ state_projection="$(cat "${script_dir}/pr-watch-state.jq")"
|
|
|
54
54
|
echo "$$" > "${slot}/watch.pid"
|
|
55
55
|
started=$(date +%s)
|
|
56
56
|
fails=0
|
|
57
|
+
unseeded=0
|
|
57
58
|
|
|
58
59
|
# In-memory floors, above the on-disk watermark. The watermark is advanced by
|
|
59
60
|
# the session after it handles a wave; these stop the loop re-reporting an event
|
|
@@ -70,51 +71,6 @@ floor_blocked_digest=""
|
|
|
70
71
|
pinned_token="$(gh auth token 2>/dev/null)"
|
|
71
72
|
[ -n "$pinned_token" ] && export GH_TOKEN="$pinned_token"
|
|
72
73
|
|
|
73
|
-
fetch_pr_state() {
|
|
74
|
-
gh api graphql -F owner="${repo%%/*}" -F name="${repo##*/}" -F number="$pr_number" -f query='
|
|
75
|
-
query($owner: String!, $name: String!, $number: Int!) {
|
|
76
|
-
repository(owner: $owner, name: $name) {
|
|
77
|
-
pullRequest(number: $number) {
|
|
78
|
-
state
|
|
79
|
-
headRefOid
|
|
80
|
-
baseRefOid
|
|
81
|
-
mergeable
|
|
82
|
-
commits(last: 1) {
|
|
83
|
-
nodes {
|
|
84
|
-
commit {
|
|
85
|
-
statusCheckRollup {
|
|
86
|
-
contexts(first: 100) {
|
|
87
|
-
nodes {
|
|
88
|
-
__typename
|
|
89
|
-
... on CheckRun { name status conclusion }
|
|
90
|
-
... on StatusContext { context state }
|
|
91
|
-
}
|
|
92
|
-
}
|
|
93
|
-
}
|
|
94
|
-
}
|
|
95
|
-
}
|
|
96
|
-
}
|
|
97
|
-
reviews(last: 20, states: [COMMENTED, APPROVED, CHANGES_REQUESTED, DISMISSED]) { nodes { databaseId body } }
|
|
98
|
-
reviewThreads(first: 100) {
|
|
99
|
-
nodes {
|
|
100
|
-
id
|
|
101
|
-
isResolved
|
|
102
|
-
isOutdated
|
|
103
|
-
comments(last: 1) { nodes { databaseId body pullRequestReview { databaseId state } } }
|
|
104
|
-
}
|
|
105
|
-
}
|
|
106
|
-
}
|
|
107
|
-
}
|
|
108
|
-
}' --jq "$state_projection" 2>>"${slot}/watch-fetch.log"
|
|
109
|
-
}
|
|
110
|
-
|
|
111
|
-
# behind_by needs its own call — see watch_wake_rebase for why no field on the
|
|
112
|
-
# PR carries it. Made only when the head/base pair moved, so the steady state
|
|
113
|
-
# stays one request per iteration.
|
|
114
|
-
fetch_behind_count() {
|
|
115
|
-
local head_sha="$1"
|
|
116
|
-
gh api "repos/${repo}/compare/${base_branch}...${head_sha}" --jq '.behind_by' 2>>"${slot}/watch-fetch.log"
|
|
117
|
-
}
|
|
118
74
|
|
|
119
75
|
read_watermark_value() {
|
|
120
76
|
local key="$1" line value=""
|
|
@@ -137,11 +93,24 @@ while :; do
|
|
|
137
93
|
touch "${slot}/watch-heartbeat" 2>/dev/null
|
|
138
94
|
|
|
139
95
|
# No watermark yet means the arming session has not finished seeding. Wait
|
|
140
|
-
# rather than treat every floor as zero, which would fire the whole backlog
|
|
96
|
+
# rather than treat every floor as zero, which would fire the whole backlog —
|
|
97
|
+
# but not forever. Every wake condition below is evaluated against a floor
|
|
98
|
+
# read from this file, so a loop that never gets one polls nothing, reports
|
|
99
|
+
# nothing and never reaches the terminal check, while still holding its PID
|
|
100
|
+
# lease and touching its heartbeat. That combination is the worst kind of
|
|
101
|
+
# broken: reconcile reads the live lease and fresh beacon as healthy and
|
|
102
|
+
# declines to re-arm, so the PR is silently unwatched for as long as the
|
|
103
|
+
# session lives. Fail loudly instead.
|
|
141
104
|
if [ ! -f "${slot}/watch-watermark.env" ]; then
|
|
105
|
+
unseeded=$((unseeded + MUGGLE_PR_WATCH_POLL_INTERVAL))
|
|
106
|
+
if watcher_unseeded_too_long "$unseeded"; then
|
|
107
|
+
echo "WATCH-FAIL pr=$pr_number no watch-watermark.env after ${unseeded}s — arming never seeded it, so this watch can detect nothing (arm-watcher.md steps 1-2)"
|
|
108
|
+
exit 1
|
|
109
|
+
fi
|
|
142
110
|
sleep "$MUGGLE_PR_WATCH_POLL_INTERVAL"
|
|
143
111
|
continue
|
|
144
112
|
fi
|
|
113
|
+
unseeded=0
|
|
145
114
|
|
|
146
115
|
watermark_review=$(read_watermark_value REV)
|
|
147
116
|
watermark_comment=$(read_watermark_value COM)
|
|
@@ -150,10 +119,10 @@ while :; do
|
|
|
150
119
|
watermark_rebase=$(read_watermark_value REBASED)
|
|
151
120
|
watermark_blocked_digest=$(read_watermark_value BLOCKED_CIDIGEST)
|
|
152
121
|
|
|
153
|
-
state_line=$(
|
|
122
|
+
state_line=$(watch_fetch_state "$repo" "$pr_number" "$slot" "$state_projection")
|
|
154
123
|
# One quick retry before counting a strike: a single flaky call should not
|
|
155
124
|
# advance the failure budget.
|
|
156
|
-
[ -z "$state_line" ] && { sleep 3; state_line=$(
|
|
125
|
+
[ -z "$state_line" ] && { sleep 3; state_line=$(watch_fetch_state "$repo" "$pr_number" "$slot" "$state_projection"); }
|
|
157
126
|
|
|
158
127
|
if [ -z "$state_line" ]; then
|
|
159
128
|
fails=$((fails + 1))
|
|
@@ -220,7 +189,7 @@ while :; do
|
|
|
220
189
|
|
|
221
190
|
rebase_key="${head_sha}..${base_sha}"
|
|
222
191
|
if [ "$rebase_key" != "$watermark_rebase" ] && [ "$rebase_key" != "$floor_rebase" ]; then
|
|
223
|
-
behind_count=$(
|
|
192
|
+
behind_count=$(watch_fetch_behind_count "$repo" "$base_branch" "$head_sha" "$slot")
|
|
224
193
|
if watch_wake_rebase "$pr_number" "$mergeable" "$behind_count" "$rebase_key" ""; then
|
|
225
194
|
floor_rebase="$rebase_key"
|
|
226
195
|
fi
|
|
@@ -2,18 +2,30 @@
|
|
|
2
2
|
|
|
3
3
|
How an orchestrating session starts the watch on one PR. Every arming point runs this same sequence: [`bootstrap.md`](bootstrap.md) Step 8, [`auto-track.md`](auto-track.md) Step 6, and the executor's post-cycle settle.
|
|
4
4
|
|
|
5
|
+
**Run [`../../scripts/pr-watch-arm.sh`](../../scripts/pr-watch-arm.sh); do not perform the steps by hand.** It performs the drain of [`contract.md`](contract.md) — reading the PR once and printing what is already outstanding as `DRAIN` lines — then seeds the watermark from that same read and starts the loop:
|
|
6
|
+
|
|
7
|
+
```sh
|
|
8
|
+
bash "<abs>/scripts/pr-watch-arm.sh" --slot "<slot>" --repo "<owner>/<repo>" --pr <n> --base <base-branch>
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
The same argument that forbids authoring a per-slot `watch.sh` applies to arming itself. Performed from prose, the middle step — writing `watch-watermark.env` — is the one that gets dropped, and dropping it is silent: the loop gates every wake condition on that file, so it polls nothing, reports nothing and never reaches its terminal check, while still holding its PID lease and touching its heartbeat. Eleven such watchers once ran for hours across one session, missed nine merges and a red CI run between them, and every health signal said they were fine. Running one command cannot half-happen.
|
|
12
|
+
|
|
13
|
+
Read the `DRAIN` lines it prints: they are the outstanding work the monitor will *not* tell you about, because the floors it just wrote mark that state as already seen. Steps 1-3 below are what the script does, and why.
|
|
14
|
+
|
|
5
15
|
1. **Drain.** Run one tick per [`contract.md`](contract.md). It acts on everything already outstanding — actionable threads (`gitlab`: discussions), body-only reviews past the watermark (GitHub-only — GitLab has no review envelope), a stale branch, red CI — and finalizes a terminal PR. If the tick dispatched a cycle, stop here: the cycle's exit path settles the watch when it finishes.
|
|
6
|
-
2. **Seed the watermark.** Resolve the provider once per [`../_shared/vcs/detect-vcs.md`](../_shared/vcs/detect-vcs.md) — every fetch in this sequence uses that provider's recipes. Write the slot's watch watermark ([`state-schemas.md`](state-schemas.md#watch-watermarkenv)) to the ids the **drain itself read** — the max review-id and comment-id observed at the drain's own fetch (Step 1), snapshotted at that read. Never let the loop capture its own baseline — the arming session writes it; and **never** from a fresh fetch taken after the drain, which would include a comment that arrived after the drain read the wave and mark it seen unread. Seeded to the drain's floor, anything landing after that read stays above the watermark and the monitor's first iteration surfaces it. Seed the CI floor (`CIRED`) from the same drain read: set it to the head SHA when the checks have **already settled red** at that read (no check pending, one or more in the `fail` bucket per [`../_shared/vcs/common/ci-rollup.md`](../_shared/vcs/common/ci-rollup.md)) — that red is what the drain just handled — and empty otherwise, so an escalated red head the drain already saw does not re-fire on the loop's first iteration. Seed the rebase floor (`REBASED`) the same way, from the drain's branch-standing read per [`../_shared/vcs/common/branch-standing.md`](../_shared/vcs/common/branch-standing.md): set it to the current `rebase_key` (`<head_sha>..<base_tip_sha>`) when the drain found the branch already behind or conflicting — that staleness is what the drain just handled — and empty otherwise, so a branch the drain already rebased or escalated does not re-fire on the loop's first iteration. Seed the blocked-CI floor (`BLOCKED_CIDIGEST`) to the blocked fingerprint's `ci_digest` when arming while `last_seen.blocked` is already set, and empty otherwise — empty is the not-blocked state, in which the loop's blocked-resume probe stays dormant.
|
|
16
|
+
2. **Seed the watermark.** *(`pr-watch-arm.sh` does this; the reasoning is kept because the floors are subtle and wrong ones are quiet.)* Resolve the provider once per [`../_shared/vcs/detect-vcs.md`](../_shared/vcs/detect-vcs.md) — every fetch in this sequence uses that provider's recipes. Write the slot's watch watermark ([`state-schemas.md`](state-schemas.md#watch-watermarkenv)) to the ids the **drain itself read** — the max review-id and comment-id observed at the drain's own fetch (Step 1), snapshotted at that read. Never let the loop capture its own baseline — the arming session writes it; and **never** from a fresh fetch taken after the drain, which would include a comment that arrived after the drain read the wave and mark it seen unread. Seeded to the drain's floor, anything landing after that read stays above the watermark and the monitor's first iteration surfaces it. Seed the CI floor (`CIRED`) from the same drain read: set it to the head SHA when the checks have **already settled red** at that read (no check pending, one or more in the `fail` bucket per [`../_shared/vcs/common/ci-rollup.md`](../_shared/vcs/common/ci-rollup.md)) — that red is what the drain just handled — and empty otherwise, so an escalated red head the drain already saw does not re-fire on the loop's first iteration. Seed the rebase floor (`REBASED`) the same way, from the drain's branch-standing read per [`../_shared/vcs/common/branch-standing.md`](../_shared/vcs/common/branch-standing.md): set it to the current `rebase_key` (`<head_sha>..<base_tip_sha>`) when the drain found the branch already behind or conflicting — that staleness is what the drain just handled — and empty otherwise, so a branch the drain already rebased or escalated does not re-fire on the loop's first iteration. Seed the blocked-CI floor (`BLOCKED_CIDIGEST`) to the blocked fingerprint's `ci_digest` when arming while `last_seen.blocked` is already set, and empty otherwise — empty is the not-blocked state, in which the loop's blocked-resume probe stays dormant.
|
|
7
17
|
3. **Dedup, then watch.** First read `<slot>/watch.pid` ([`state-schemas.md`](state-schemas.md#watchpid)): if it names a live process (`kill -0 "$pid"` succeeds), a watcher already owns this slot — **skip arming, do not start a second**. This is what stops orphaned watchers from accumulating: the in-session monitor dying does not stop the OS loop it launched (on Windows a detached Git Bash loop keeps running and polling `gh` forever after the session ends), so checking a live task list is not enough — the PID lease is.
|
|
8
18
|
|
|
9
19
|
Otherwise **claim the slot for this session** before starting anything: write `owner.json` ([`state-schemas.md`](state-schemas.md#ownerjson)) with `session_id` from `$CLAUDE_CODE_SESSION_ID` and `claimed_at` now. Arming is what establishes ownership, so every arming point records it here rather than each caller remembering to. If `$CLAUDE_CODE_SESSION_ID` is unset, write no `owner.json` — an unidentifiable owner is worse than none, since [`reconcile.md`](reconcile.md) would read a bogus id as some other session's claim and could never recover the slot.
|
|
10
20
|
|
|
11
|
-
Then start the
|
|
21
|
+
Then start the watch as a **persistent background monitor** in the orchestrating session, with the script path resolved to an absolute path at arm time (from `${CLAUDE_PLUGIN_ROOT}/scripts/`) so it still resolves after the arming session is gone. `pr-watch-arm.sh` performs Steps 1-2 and then execs the loop, so the monitor's command is the arm script:
|
|
12
22
|
|
|
13
23
|
```sh
|
|
14
|
-
bash "<abs>/scripts/pr-watch-
|
|
24
|
+
bash "<abs>/scripts/pr-watch-arm.sh" --slot "<slot>" --repo "<owner>/<repo>" --pr <n> --base <base-branch>
|
|
15
25
|
```
|
|
16
26
|
|
|
27
|
+
Pass `--no-exec` to seed a slot without starting a loop; it is otherwise the same sequence.
|
|
28
|
+
|
|
17
29
|
**Never author a per-slot `watch.sh`.** The loop ships as [`../../scripts/pr-watch-loop.sh`](../../scripts/pr-watch-loop.sh), with its wake conditions in [`../../scripts/pr-watch-events.sh`](../../scripts/pr-watch-events.sh) and its state projection in [`../../scripts/pr-watch-state.jq`](../../scripts/pr-watch-state.jq); arming runs it and passes arguments. Writing the loop from this prose was how it drifted — each arm produced an independent derivation, and a derivation that quietly dropped a wake still ran, still heartbeat, still logged, and simply never fired for the signal it lost. Two slots on one machine ended up without the behind-base wake, which left their PRs unmergeable under watchers that looked healthy. The prose below says *why* each wake exists; the shipped files are the only definition of *what* fires. A slot holding a legacy generated `watch.sh` keeps it until re-armed, at which point the supersede guard retires the old loop.
|
|
18
30
|
|
|
19
31
|
The label is `PR #<n> — <title>`. Label and command both matter: some task surfaces show one, some the other, and a slot-bearing command keeps the watch identifiable everywhere a raw script blob would not. One monitor per PR, alive from arm to terminal: it is the watch's visible handle, showing as a running task the entire time the PR is polled. Its loop checks about every 60 seconds, re-reading the watermark and touching the slot's `watch-heartbeat` file each iteration — the liveness beacon that tells [`reconcile.md`](reconcile.md) a quiet watch is still alive; on a newer submitted review, a newer thread comment (`gitlab`: a newer discussion note), a thread newly unresolved (`gitlab`: discussion), **the head SHA's checks settling red** (no check pending and one or more in the `fail` bucket per [`../_shared/vcs/common/ci-rollup.md`](../_shared/vcs/common/ci-rollup.md)), **the branch falling behind or conflicting with its base** (`behind_by > 0` or the conflict signal per [`../_shared/vcs/common/branch-standing.md`](../_shared/vcs/common/branch-standing.md)), or — **only while the watch is blocked** (`BLOCKED_CIDIGEST` non-empty) — **the head's CI digest changing in any way** (not just to red) — it prints one line and **keeps watching**, advancing its in-memory floor so each event fires the tick exactly once. The review and thread floors are monotonic ids; the other three are not. The CI-red floor is the **head SHA**, because the check rollup is non-monotonic — it flips green↔red and resets on every push — so recording the red head SHA fires CI once per red head, and a later push re-arms it on the new SHA. The rebase floor (`REBASED`) is the **`rebase_key`** — `<head_sha>..<base_tip_sha>` — because staleness is a function of both sides: keying on the pair fires once per newly-due pair and re-arms when either the head or the base moves, where a head-only key would wedge permanently the first time the base advances (the head cannot change while nobody pushes). A head whose checks are still **pending** is never a red wake, and a branch with `behind_by == 0` and `mergeable == UNKNOWN` is never a rebase wake: pending checks may yet go green and conflict state is still computing, and the tick would idle on either (Steps 5–6) regardless. The blocked-CI signal is different in kind — a **resume** probe, live only while the watch is blocked: it wakes on any move of the head's CI digest (the same bucket-plus-sorted-name/conclusion signature the blocked fingerprint records — [`blocked-tick.md`](blocked-tick.md)) away from `BLOCKED_CIDIGEST`, so a block waiting on a green pass, a rerun, or an external deploy check resumes as promptly as one waiting on red. Quiet iterations print nothing and cost nothing — no model tokens are spent while the watch is quiet.
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"release": "5.11.0",
|
|
3
|
-
"buildId": "run-
|
|
4
|
-
"commitSha": "
|
|
5
|
-
"buildTime": "2026-08-
|
|
3
|
+
"buildId": "run-87-1",
|
|
4
|
+
"commitSha": "bc0e4ca5afe97b15d235d10b222be9695d2a140b",
|
|
5
|
+
"buildTime": "2026-08-30T05:45:55Z",
|
|
6
6
|
"serviceName": "muggle-ai-works-mcp"
|
|
7
7
|
}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@muggleai/works",
|
|
3
3
|
"mcpName": "io.github.multiplex-ai/muggle",
|
|
4
|
-
"version": "5.12.0-staging.
|
|
4
|
+
"version": "5.12.0-staging.87",
|
|
5
5
|
"description": "Ship quality products with AI-powered E2E acceptance testing that validates your web app like a real user — from Claude Code and Cursor to PR.",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"main": "dist/index.js",
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# Arm the watch on one pull request: read its state once, report what is already
|
|
3
|
+
# outstanding, seed the watermark from that same read, and start the loop.
|
|
4
|
+
#
|
|
5
|
+
# Arming used to be three prose steps a caller performed by hand, and skipping
|
|
6
|
+
# the middle one — writing watch-watermark.env — produced a loop that held its
|
|
7
|
+
# PID lease and touched its heartbeat while polling nothing, reporting nothing
|
|
8
|
+
# and never reaching its terminal check. Every health signal said fine. Doing
|
|
9
|
+
# the three steps here, in one command, is what makes that unskippable.
|
|
10
|
+
#
|
|
11
|
+
# The floors are taken from the drain's own read rather than from a fresh fetch
|
|
12
|
+
# afterwards: anything that lands between the two would otherwise be marked seen
|
|
13
|
+
# without ever being reported.
|
|
14
|
+
set -uo pipefail
|
|
15
|
+
|
|
16
|
+
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
17
|
+
|
|
18
|
+
slot="" repo="" pr_number="" base_branch="" exec_loop=1
|
|
19
|
+
|
|
20
|
+
while [ $# -gt 0 ]; do
|
|
21
|
+
case "$1" in
|
|
22
|
+
--slot) slot="$2"; shift 2 ;;
|
|
23
|
+
--repo) repo="$2"; shift 2 ;;
|
|
24
|
+
--pr) pr_number="$2"; shift 2 ;;
|
|
25
|
+
--base) base_branch="$2"; shift 2 ;;
|
|
26
|
+
--no-exec) exec_loop=0; shift ;;
|
|
27
|
+
*) echo "pr-watch-arm: unknown argument $1" >&2; exit 2 ;;
|
|
28
|
+
esac
|
|
29
|
+
done
|
|
30
|
+
|
|
31
|
+
if [ -z "$slot" ] || [ -z "$repo" ] || [ -z "$pr_number" ] || [ -z "$base_branch" ]; then
|
|
32
|
+
echo "usage: pr-watch-arm.sh --slot <dir> --repo <owner/repo> --pr <n> --base <branch> [--no-exec]" >&2
|
|
33
|
+
exit 2
|
|
34
|
+
fi
|
|
35
|
+
|
|
36
|
+
for lib in pr-watch-guards.sh pr-watch-events.sh pr-watch-fetch.sh; do
|
|
37
|
+
if [ ! -f "${script_dir}/${lib}" ]; then
|
|
38
|
+
echo "pr-watch-arm: missing ${lib} beside this script" >&2
|
|
39
|
+
exit 2
|
|
40
|
+
fi
|
|
41
|
+
# shellcheck source=/dev/null
|
|
42
|
+
. "${script_dir}/${lib}"
|
|
43
|
+
done
|
|
44
|
+
|
|
45
|
+
if [ ! -f "${script_dir}/pr-watch-state.jq" ]; then
|
|
46
|
+
echo "pr-watch-arm: missing pr-watch-state.jq beside this script" >&2
|
|
47
|
+
exit 2
|
|
48
|
+
fi
|
|
49
|
+
state_projection="$(cat "${script_dir}/pr-watch-state.jq")"
|
|
50
|
+
|
|
51
|
+
mkdir -p "$slot"
|
|
52
|
+
|
|
53
|
+
pinned_token="$(gh auth token 2>/dev/null)"
|
|
54
|
+
[ -n "$pinned_token" ] && export GH_TOKEN="$pinned_token"
|
|
55
|
+
|
|
56
|
+
state_line="$(watch_fetch_state "$repo" "$pr_number" "$slot" "$state_projection")"
|
|
57
|
+
if [ -z "$state_line" ]; then
|
|
58
|
+
echo "ARM-FAIL pr=$pr_number could not read PR state — refusing to arm a watch with no floors"
|
|
59
|
+
exit 1
|
|
60
|
+
fi
|
|
61
|
+
|
|
62
|
+
mapfile -t fields < <(watch_split_state "$state_line")
|
|
63
|
+
pr_state="${fields[0]-}"
|
|
64
|
+
head_sha="${fields[1]-}"
|
|
65
|
+
base_sha="${fields[2]-}"
|
|
66
|
+
mergeable="${fields[3]-}"
|
|
67
|
+
latest_review="${fields[4]-}"
|
|
68
|
+
latest_comment="${fields[5]-}"
|
|
69
|
+
unresolved_threads="${fields[6]-}"
|
|
70
|
+
pending_checks="${fields[7]-}"
|
|
71
|
+
failed_checks="${fields[8]-}"
|
|
72
|
+
|
|
73
|
+
if [ "$pr_state" = "MERGED" ] || [ "$pr_state" = "CLOSED" ]; then
|
|
74
|
+
echo "TERMINAL pr=$pr_number state=$pr_state — nothing to arm"
|
|
75
|
+
exit 0
|
|
76
|
+
fi
|
|
77
|
+
|
|
78
|
+
# A red head at arm time is the drain's to hand over, so it goes into the floor
|
|
79
|
+
# and does not re-fire on the loop's first pass. Pending checks are not red yet.
|
|
80
|
+
ci_red_floor=""
|
|
81
|
+
if [ "${pending_checks:-0}" -eq 0 ] 2>/dev/null && [ "${failed_checks:-0}" -gt 0 ] 2>/dev/null; then
|
|
82
|
+
ci_red_floor="$head_sha"
|
|
83
|
+
fi
|
|
84
|
+
|
|
85
|
+
# Same for staleness: keyed on the head/base pair, so it re-arms when either moves.
|
|
86
|
+
rebase_floor=""
|
|
87
|
+
behind_count="$(watch_fetch_behind_count "$repo" "$base_branch" "$head_sha" "$slot")"
|
|
88
|
+
if [ "$mergeable" = "CONFLICTING" ] || { [ -n "$behind_count" ] && [ "$behind_count" -gt 0 ] 2>/dev/null; }; then
|
|
89
|
+
rebase_floor="${head_sha}..${base_sha}"
|
|
90
|
+
fi
|
|
91
|
+
|
|
92
|
+
thread_count=0
|
|
93
|
+
[ -n "$unresolved_threads" ] && thread_count=$(printf '%s' "$unresolved_threads" | tr ';' '\n' | grep -c .)
|
|
94
|
+
|
|
95
|
+
# Printed, not swallowed: these are the things the arming session owes a decision
|
|
96
|
+
# on before the monitor takes over, and the monitor only ever sees what arrives
|
|
97
|
+
# after this point.
|
|
98
|
+
echo "DRAIN pr=$pr_number state=$pr_state head=${head_sha:0:8}"
|
|
99
|
+
echo "DRAIN unresolved-threads=$thread_count latest-review=${latest_review:-0} latest-comment=${latest_comment:-0}"
|
|
100
|
+
echo "DRAIN checks pending=${pending_checks:-0} failed=${failed_checks:-0} behind-base=${behind_count:-unknown} mergeable=${mergeable:-UNKNOWN}"
|
|
101
|
+
[ -n "$ci_red_floor" ] && echo "DRAIN ci already red at arm time — handled by the arming session, floored"
|
|
102
|
+
[ -n "$rebase_floor" ] && echo "DRAIN branch already stale at arm time — handled by the arming session, floored"
|
|
103
|
+
|
|
104
|
+
{
|
|
105
|
+
printf 'REV=%s\n' "${latest_review:-0}"
|
|
106
|
+
printf 'COM=%s\n' "${latest_comment:-0}"
|
|
107
|
+
printf 'THREADS="%s"\n' "$unresolved_threads"
|
|
108
|
+
printf 'CIRED="%s"\n' "$ci_red_floor"
|
|
109
|
+
printf 'REBASED="%s"\n' "$rebase_floor"
|
|
110
|
+
printf 'BLOCKED_CIDIGEST=""\n'
|
|
111
|
+
} > "${slot}/watch-watermark.env"
|
|
112
|
+
|
|
113
|
+
echo "ARMED pr=$pr_number watermark seeded at ${slot}/watch-watermark.env"
|
|
114
|
+
|
|
115
|
+
[ "$exec_loop" -eq 1 ] || exit 0
|
|
116
|
+
|
|
117
|
+
exec bash "${script_dir}/pr-watch-loop.sh" --slot "$slot" --repo "$repo" --pr "$pr_number" --base "$base_branch"
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# One read of a pull request's watch-relevant state, shared by the loop that
|
|
3
|
+
# polls it and the arm step that seeds its watermark.
|
|
4
|
+
#
|
|
5
|
+
# Both must see the PR through exactly the same projection: the watermark is a
|
|
6
|
+
# set of floors taken from these fields, and a floor read through a different
|
|
7
|
+
# shape than the one it is later compared against is worse than no floor at all.
|
|
8
|
+
# Keeping the query in one place is what makes "seed from the drain's own read"
|
|
9
|
+
# a fact rather than an instruction.
|
|
10
|
+
|
|
11
|
+
watch_fetch_state() {
|
|
12
|
+
local repo="$1" pr_number="$2" slot="$3" state_projection="$4"
|
|
13
|
+
gh api graphql -F owner="${repo%%/*}" -F name="${repo##*/}" -F number="$pr_number" -f query='
|
|
14
|
+
query($owner: String!, $name: String!, $number: Int!) {
|
|
15
|
+
repository(owner: $owner, name: $name) {
|
|
16
|
+
pullRequest(number: $number) {
|
|
17
|
+
state
|
|
18
|
+
headRefOid
|
|
19
|
+
baseRefOid
|
|
20
|
+
mergeable
|
|
21
|
+
commits(last: 1) {
|
|
22
|
+
nodes {
|
|
23
|
+
commit {
|
|
24
|
+
statusCheckRollup {
|
|
25
|
+
contexts(first: 100) {
|
|
26
|
+
nodes {
|
|
27
|
+
__typename
|
|
28
|
+
... on CheckRun { name status conclusion }
|
|
29
|
+
... on StatusContext { context state }
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
reviews(last: 20, states: [COMMENTED, APPROVED, CHANGES_REQUESTED, DISMISSED]) { nodes { databaseId body } }
|
|
37
|
+
reviewThreads(first: 100) {
|
|
38
|
+
nodes {
|
|
39
|
+
id
|
|
40
|
+
isResolved
|
|
41
|
+
isOutdated
|
|
42
|
+
comments(last: 1) { nodes { databaseId body pullRequestReview { databaseId state } } }
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
}' --jq "$state_projection" 2>>"${slot}/watch-fetch.log"
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
# behind_by needs its own call — see watch_wake_rebase for why no field on the
|
|
51
|
+
# PR carries it. The loop makes it only when the head/base pair moved, so the
|
|
52
|
+
# steady state stays one request per iteration.
|
|
53
|
+
watch_fetch_behind_count() {
|
|
54
|
+
local repo="$1" base_branch="$2" head_sha="$3" slot="$4"
|
|
55
|
+
gh api "repos/${repo}/compare/${base_branch}...${head_sha}" --jq '.behind_by' 2>>"${slot}/watch-fetch.log"
|
|
56
|
+
}
|
|
@@ -30,6 +30,15 @@ MUGGLE_PR_WATCH_POLL_INTERVAL="${MUGGLE_PR_WATCH_POLL_INTERVAL:-60}"
|
|
|
30
30
|
# revoked auth) exhausts it.
|
|
31
31
|
MUGGLE_PR_WATCH_MAX_FETCH_FAILURES="${MUGGLE_PR_WATCH_MAX_FETCH_FAILURES:-60}"
|
|
32
32
|
|
|
33
|
+
# Seconds a loop will wait for the arming session to write watch-watermark.env
|
|
34
|
+
# before giving up. The loop cannot evaluate a single wake condition without it,
|
|
35
|
+
# so an unseeded slot is a watcher that will never report anything — and one that
|
|
36
|
+
# looks perfectly healthy while it does nothing, because it still holds its PID
|
|
37
|
+
# lease and still touches its heartbeat. Arming writes the file in the same turn
|
|
38
|
+
# it starts the loop, so anything past a couple of minutes means the arming
|
|
39
|
+
# sequence was not followed and the watch is inert.
|
|
40
|
+
MUGGLE_PR_WATCH_MAX_UNSEEDED="${MUGGLE_PR_WATCH_MAX_UNSEEDED:-180}"
|
|
41
|
+
|
|
33
42
|
# Seconds to sleep after `fails` consecutive failed fetches: the poll interval,
|
|
34
43
|
# then a linear back-off capped at 5 minutes so a sustained outage is retried
|
|
35
44
|
# calmly rather than hammered every 60s.
|
|
@@ -59,6 +68,15 @@ watcher_lifetime_exceeded() {
|
|
|
59
68
|
[ $((now - started)) -ge "$max" ]
|
|
60
69
|
}
|
|
61
70
|
|
|
71
|
+
# True when the slot has gone unseeded longer than the cap allows. 0 is
|
|
72
|
+
# unbounded, matching `watcher_lifetime_exceeded`, for a caller that seeds the
|
|
73
|
+
# watermark out of band.
|
|
74
|
+
watcher_unseeded_too_long() {
|
|
75
|
+
local waited="$1" max="${2:-$MUGGLE_PR_WATCH_MAX_UNSEEDED}"
|
|
76
|
+
[ "$max" -eq 0 ] 2>/dev/null && return 1
|
|
77
|
+
[ "$waited" -ge "$max" ]
|
|
78
|
+
}
|
|
79
|
+
|
|
62
80
|
# True when pid names a running process. `kill -0` sends no signal; EPERM means
|
|
63
81
|
# the process exists but is foreign, which still counts as alive. Used by
|
|
64
82
|
# arm-watcher's pre-arm dedup to decide whether a watcher already owns the slot.
|
|
@@ -39,7 +39,7 @@ fi
|
|
|
39
39
|
# gone. Guards missing means the plugin moved or upgraded underneath this loop —
|
|
40
40
|
# a newer version's watcher owns the slot now, so step down rather than run on
|
|
41
41
|
# without the supersede check.
|
|
42
|
-
for lib in pr-watch-guards.sh pr-watch-events.sh; do
|
|
42
|
+
for lib in pr-watch-guards.sh pr-watch-events.sh pr-watch-fetch.sh; do
|
|
43
43
|
[ -f "${script_dir}/${lib}" ] || exit 0
|
|
44
44
|
# shellcheck source=/dev/null
|
|
45
45
|
. "${script_dir}/${lib}"
|
|
@@ -54,6 +54,7 @@ state_projection="$(cat "${script_dir}/pr-watch-state.jq")"
|
|
|
54
54
|
echo "$$" > "${slot}/watch.pid"
|
|
55
55
|
started=$(date +%s)
|
|
56
56
|
fails=0
|
|
57
|
+
unseeded=0
|
|
57
58
|
|
|
58
59
|
# In-memory floors, above the on-disk watermark. The watermark is advanced by
|
|
59
60
|
# the session after it handles a wave; these stop the loop re-reporting an event
|
|
@@ -70,51 +71,6 @@ floor_blocked_digest=""
|
|
|
70
71
|
pinned_token="$(gh auth token 2>/dev/null)"
|
|
71
72
|
[ -n "$pinned_token" ] && export GH_TOKEN="$pinned_token"
|
|
72
73
|
|
|
73
|
-
fetch_pr_state() {
|
|
74
|
-
gh api graphql -F owner="${repo%%/*}" -F name="${repo##*/}" -F number="$pr_number" -f query='
|
|
75
|
-
query($owner: String!, $name: String!, $number: Int!) {
|
|
76
|
-
repository(owner: $owner, name: $name) {
|
|
77
|
-
pullRequest(number: $number) {
|
|
78
|
-
state
|
|
79
|
-
headRefOid
|
|
80
|
-
baseRefOid
|
|
81
|
-
mergeable
|
|
82
|
-
commits(last: 1) {
|
|
83
|
-
nodes {
|
|
84
|
-
commit {
|
|
85
|
-
statusCheckRollup {
|
|
86
|
-
contexts(first: 100) {
|
|
87
|
-
nodes {
|
|
88
|
-
__typename
|
|
89
|
-
... on CheckRun { name status conclusion }
|
|
90
|
-
... on StatusContext { context state }
|
|
91
|
-
}
|
|
92
|
-
}
|
|
93
|
-
}
|
|
94
|
-
}
|
|
95
|
-
}
|
|
96
|
-
}
|
|
97
|
-
reviews(last: 20, states: [COMMENTED, APPROVED, CHANGES_REQUESTED, DISMISSED]) { nodes { databaseId body } }
|
|
98
|
-
reviewThreads(first: 100) {
|
|
99
|
-
nodes {
|
|
100
|
-
id
|
|
101
|
-
isResolved
|
|
102
|
-
isOutdated
|
|
103
|
-
comments(last: 1) { nodes { databaseId body pullRequestReview { databaseId state } } }
|
|
104
|
-
}
|
|
105
|
-
}
|
|
106
|
-
}
|
|
107
|
-
}
|
|
108
|
-
}' --jq "$state_projection" 2>>"${slot}/watch-fetch.log"
|
|
109
|
-
}
|
|
110
|
-
|
|
111
|
-
# behind_by needs its own call — see watch_wake_rebase for why no field on the
|
|
112
|
-
# PR carries it. Made only when the head/base pair moved, so the steady state
|
|
113
|
-
# stays one request per iteration.
|
|
114
|
-
fetch_behind_count() {
|
|
115
|
-
local head_sha="$1"
|
|
116
|
-
gh api "repos/${repo}/compare/${base_branch}...${head_sha}" --jq '.behind_by' 2>>"${slot}/watch-fetch.log"
|
|
117
|
-
}
|
|
118
74
|
|
|
119
75
|
read_watermark_value() {
|
|
120
76
|
local key="$1" line value=""
|
|
@@ -137,11 +93,24 @@ while :; do
|
|
|
137
93
|
touch "${slot}/watch-heartbeat" 2>/dev/null
|
|
138
94
|
|
|
139
95
|
# No watermark yet means the arming session has not finished seeding. Wait
|
|
140
|
-
# rather than treat every floor as zero, which would fire the whole backlog
|
|
96
|
+
# rather than treat every floor as zero, which would fire the whole backlog —
|
|
97
|
+
# but not forever. Every wake condition below is evaluated against a floor
|
|
98
|
+
# read from this file, so a loop that never gets one polls nothing, reports
|
|
99
|
+
# nothing and never reaches the terminal check, while still holding its PID
|
|
100
|
+
# lease and touching its heartbeat. That combination is the worst kind of
|
|
101
|
+
# broken: reconcile reads the live lease and fresh beacon as healthy and
|
|
102
|
+
# declines to re-arm, so the PR is silently unwatched for as long as the
|
|
103
|
+
# session lives. Fail loudly instead.
|
|
141
104
|
if [ ! -f "${slot}/watch-watermark.env" ]; then
|
|
105
|
+
unseeded=$((unseeded + MUGGLE_PR_WATCH_POLL_INTERVAL))
|
|
106
|
+
if watcher_unseeded_too_long "$unseeded"; then
|
|
107
|
+
echo "WATCH-FAIL pr=$pr_number no watch-watermark.env after ${unseeded}s — arming never seeded it, so this watch can detect nothing (arm-watcher.md steps 1-2)"
|
|
108
|
+
exit 1
|
|
109
|
+
fi
|
|
142
110
|
sleep "$MUGGLE_PR_WATCH_POLL_INTERVAL"
|
|
143
111
|
continue
|
|
144
112
|
fi
|
|
113
|
+
unseeded=0
|
|
145
114
|
|
|
146
115
|
watermark_review=$(read_watermark_value REV)
|
|
147
116
|
watermark_comment=$(read_watermark_value COM)
|
|
@@ -150,10 +119,10 @@ while :; do
|
|
|
150
119
|
watermark_rebase=$(read_watermark_value REBASED)
|
|
151
120
|
watermark_blocked_digest=$(read_watermark_value BLOCKED_CIDIGEST)
|
|
152
121
|
|
|
153
|
-
state_line=$(
|
|
122
|
+
state_line=$(watch_fetch_state "$repo" "$pr_number" "$slot" "$state_projection")
|
|
154
123
|
# One quick retry before counting a strike: a single flaky call should not
|
|
155
124
|
# advance the failure budget.
|
|
156
|
-
[ -z "$state_line" ] && { sleep 3; state_line=$(
|
|
125
|
+
[ -z "$state_line" ] && { sleep 3; state_line=$(watch_fetch_state "$repo" "$pr_number" "$slot" "$state_projection"); }
|
|
157
126
|
|
|
158
127
|
if [ -z "$state_line" ]; then
|
|
159
128
|
fails=$((fails + 1))
|
|
@@ -220,7 +189,7 @@ while :; do
|
|
|
220
189
|
|
|
221
190
|
rebase_key="${head_sha}..${base_sha}"
|
|
222
191
|
if [ "$rebase_key" != "$watermark_rebase" ] && [ "$rebase_key" != "$floor_rebase" ]; then
|
|
223
|
-
behind_count=$(
|
|
192
|
+
behind_count=$(watch_fetch_behind_count "$repo" "$base_branch" "$head_sha" "$slot")
|
|
224
193
|
if watch_wake_rebase "$pr_number" "$mergeable" "$behind_count" "$rebase_key" ""; then
|
|
225
194
|
floor_rebase="$rebase_key"
|
|
226
195
|
fi
|
|
@@ -2,18 +2,30 @@
|
|
|
2
2
|
|
|
3
3
|
How an orchestrating session starts the watch on one PR. Every arming point runs this same sequence: [`bootstrap.md`](bootstrap.md) Step 8, [`auto-track.md`](auto-track.md) Step 6, and the executor's post-cycle settle.
|
|
4
4
|
|
|
5
|
+
**Run [`../../scripts/pr-watch-arm.sh`](../../scripts/pr-watch-arm.sh); do not perform the steps by hand.** It performs the drain of [`contract.md`](contract.md) — reading the PR once and printing what is already outstanding as `DRAIN` lines — then seeds the watermark from that same read and starts the loop:
|
|
6
|
+
|
|
7
|
+
```sh
|
|
8
|
+
bash "<abs>/scripts/pr-watch-arm.sh" --slot "<slot>" --repo "<owner>/<repo>" --pr <n> --base <base-branch>
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
The same argument that forbids authoring a per-slot `watch.sh` applies to arming itself. Performed from prose, the middle step — writing `watch-watermark.env` — is the one that gets dropped, and dropping it is silent: the loop gates every wake condition on that file, so it polls nothing, reports nothing and never reaches its terminal check, while still holding its PID lease and touching its heartbeat. Eleven such watchers once ran for hours across one session, missed nine merges and a red CI run between them, and every health signal said they were fine. Running one command cannot half-happen.
|
|
12
|
+
|
|
13
|
+
Read the `DRAIN` lines it prints: they are the outstanding work the monitor will *not* tell you about, because the floors it just wrote mark that state as already seen. Steps 1-3 below are what the script does, and why.
|
|
14
|
+
|
|
5
15
|
1. **Drain.** Run one tick per [`contract.md`](contract.md). It acts on everything already outstanding — actionable threads (`gitlab`: discussions), body-only reviews past the watermark (GitHub-only — GitLab has no review envelope), a stale branch, red CI — and finalizes a terminal PR. If the tick dispatched a cycle, stop here: the cycle's exit path settles the watch when it finishes.
|
|
6
|
-
2. **Seed the watermark.** Resolve the provider once per [`../_shared/vcs/detect-vcs.md`](../_shared/vcs/detect-vcs.md) — every fetch in this sequence uses that provider's recipes. Write the slot's watch watermark ([`state-schemas.md`](state-schemas.md#watch-watermarkenv)) to the ids the **drain itself read** — the max review-id and comment-id observed at the drain's own fetch (Step 1), snapshotted at that read. Never let the loop capture its own baseline — the arming session writes it; and **never** from a fresh fetch taken after the drain, which would include a comment that arrived after the drain read the wave and mark it seen unread. Seeded to the drain's floor, anything landing after that read stays above the watermark and the monitor's first iteration surfaces it. Seed the CI floor (`CIRED`) from the same drain read: set it to the head SHA when the checks have **already settled red** at that read (no check pending, one or more in the `fail` bucket per [`../_shared/vcs/common/ci-rollup.md`](../_shared/vcs/common/ci-rollup.md)) — that red is what the drain just handled — and empty otherwise, so an escalated red head the drain already saw does not re-fire on the loop's first iteration. Seed the rebase floor (`REBASED`) the same way, from the drain's branch-standing read per [`../_shared/vcs/common/branch-standing.md`](../_shared/vcs/common/branch-standing.md): set it to the current `rebase_key` (`<head_sha>..<base_tip_sha>`) when the drain found the branch already behind or conflicting — that staleness is what the drain just handled — and empty otherwise, so a branch the drain already rebased or escalated does not re-fire on the loop's first iteration. Seed the blocked-CI floor (`BLOCKED_CIDIGEST`) to the blocked fingerprint's `ci_digest` when arming while `last_seen.blocked` is already set, and empty otherwise — empty is the not-blocked state, in which the loop's blocked-resume probe stays dormant.
|
|
16
|
+
2. **Seed the watermark.** *(`pr-watch-arm.sh` does this; the reasoning is kept because the floors are subtle and wrong ones are quiet.)* Resolve the provider once per [`../_shared/vcs/detect-vcs.md`](../_shared/vcs/detect-vcs.md) — every fetch in this sequence uses that provider's recipes. Write the slot's watch watermark ([`state-schemas.md`](state-schemas.md#watch-watermarkenv)) to the ids the **drain itself read** — the max review-id and comment-id observed at the drain's own fetch (Step 1), snapshotted at that read. Never let the loop capture its own baseline — the arming session writes it; and **never** from a fresh fetch taken after the drain, which would include a comment that arrived after the drain read the wave and mark it seen unread. Seeded to the drain's floor, anything landing after that read stays above the watermark and the monitor's first iteration surfaces it. Seed the CI floor (`CIRED`) from the same drain read: set it to the head SHA when the checks have **already settled red** at that read (no check pending, one or more in the `fail` bucket per [`../_shared/vcs/common/ci-rollup.md`](../_shared/vcs/common/ci-rollup.md)) — that red is what the drain just handled — and empty otherwise, so an escalated red head the drain already saw does not re-fire on the loop's first iteration. Seed the rebase floor (`REBASED`) the same way, from the drain's branch-standing read per [`../_shared/vcs/common/branch-standing.md`](../_shared/vcs/common/branch-standing.md): set it to the current `rebase_key` (`<head_sha>..<base_tip_sha>`) when the drain found the branch already behind or conflicting — that staleness is what the drain just handled — and empty otherwise, so a branch the drain already rebased or escalated does not re-fire on the loop's first iteration. Seed the blocked-CI floor (`BLOCKED_CIDIGEST`) to the blocked fingerprint's `ci_digest` when arming while `last_seen.blocked` is already set, and empty otherwise — empty is the not-blocked state, in which the loop's blocked-resume probe stays dormant.
|
|
7
17
|
3. **Dedup, then watch.** First read `<slot>/watch.pid` ([`state-schemas.md`](state-schemas.md#watchpid)): if it names a live process (`kill -0 "$pid"` succeeds), a watcher already owns this slot — **skip arming, do not start a second**. This is what stops orphaned watchers from accumulating: the in-session monitor dying does not stop the OS loop it launched (on Windows a detached Git Bash loop keeps running and polling `gh` forever after the session ends), so checking a live task list is not enough — the PID lease is.
|
|
8
18
|
|
|
9
19
|
Otherwise **claim the slot for this session** before starting anything: write `owner.json` ([`state-schemas.md`](state-schemas.md#ownerjson)) with `session_id` from `$CLAUDE_CODE_SESSION_ID` and `claimed_at` now. Arming is what establishes ownership, so every arming point records it here rather than each caller remembering to. If `$CLAUDE_CODE_SESSION_ID` is unset, write no `owner.json` — an unidentifiable owner is worse than none, since [`reconcile.md`](reconcile.md) would read a bogus id as some other session's claim and could never recover the slot.
|
|
10
20
|
|
|
11
|
-
Then start the
|
|
21
|
+
Then start the watch as a **persistent background monitor** in the orchestrating session, with the script path resolved to an absolute path at arm time (from `${CLAUDE_PLUGIN_ROOT}/scripts/`) so it still resolves after the arming session is gone. `pr-watch-arm.sh` performs Steps 1-2 and then execs the loop, so the monitor's command is the arm script:
|
|
12
22
|
|
|
13
23
|
```sh
|
|
14
|
-
bash "<abs>/scripts/pr-watch-
|
|
24
|
+
bash "<abs>/scripts/pr-watch-arm.sh" --slot "<slot>" --repo "<owner>/<repo>" --pr <n> --base <base-branch>
|
|
15
25
|
```
|
|
16
26
|
|
|
27
|
+
Pass `--no-exec` to seed a slot without starting a loop; it is otherwise the same sequence.
|
|
28
|
+
|
|
17
29
|
**Never author a per-slot `watch.sh`.** The loop ships as [`../../scripts/pr-watch-loop.sh`](../../scripts/pr-watch-loop.sh), with its wake conditions in [`../../scripts/pr-watch-events.sh`](../../scripts/pr-watch-events.sh) and its state projection in [`../../scripts/pr-watch-state.jq`](../../scripts/pr-watch-state.jq); arming runs it and passes arguments. Writing the loop from this prose was how it drifted — each arm produced an independent derivation, and a derivation that quietly dropped a wake still ran, still heartbeat, still logged, and simply never fired for the signal it lost. Two slots on one machine ended up without the behind-base wake, which left their PRs unmergeable under watchers that looked healthy. The prose below says *why* each wake exists; the shipped files are the only definition of *what* fires. A slot holding a legacy generated `watch.sh` keeps it until re-armed, at which point the supersede guard retires the old loop.
|
|
18
30
|
|
|
19
31
|
The label is `PR #<n> — <title>`. Label and command both matter: some task surfaces show one, some the other, and a slot-bearing command keeps the watch identifiable everywhere a raw script blob would not. One monitor per PR, alive from arm to terminal: it is the watch's visible handle, showing as a running task the entire time the PR is polled. Its loop checks about every 60 seconds, re-reading the watermark and touching the slot's `watch-heartbeat` file each iteration — the liveness beacon that tells [`reconcile.md`](reconcile.md) a quiet watch is still alive; on a newer submitted review, a newer thread comment (`gitlab`: a newer discussion note), a thread newly unresolved (`gitlab`: discussion), **the head SHA's checks settling red** (no check pending and one or more in the `fail` bucket per [`../_shared/vcs/common/ci-rollup.md`](../_shared/vcs/common/ci-rollup.md)), **the branch falling behind or conflicting with its base** (`behind_by > 0` or the conflict signal per [`../_shared/vcs/common/branch-standing.md`](../_shared/vcs/common/branch-standing.md)), or — **only while the watch is blocked** (`BLOCKED_CIDIGEST` non-empty) — **the head's CI digest changing in any way** (not just to red) — it prints one line and **keeps watching**, advancing its in-memory floor so each event fires the tick exactly once. The review and thread floors are monotonic ids; the other three are not. The CI-red floor is the **head SHA**, because the check rollup is non-monotonic — it flips green↔red and resets on every push — so recording the red head SHA fires CI once per red head, and a later push re-arms it on the new SHA. The rebase floor (`REBASED`) is the **`rebase_key`** — `<head_sha>..<base_tip_sha>` — because staleness is a function of both sides: keying on the pair fires once per newly-due pair and re-arms when either the head or the base moves, where a head-only key would wedge permanently the first time the base advances (the head cannot change while nobody pushes). A head whose checks are still **pending** is never a red wake, and a branch with `behind_by == 0` and `mergeable == UNKNOWN` is never a rebase wake: pending checks may yet go green and conflict state is still computing, and the tick would idle on either (Steps 5–6) regardless. The blocked-CI signal is different in kind — a **resume** probe, live only while the watch is blocked: it wakes on any move of the head's CI digest (the same bucket-plus-sorted-name/conclusion signature the blocked fingerprint records — [`blocked-tick.md`](blocked-tick.md)) away from `BLOCKED_CIDIGEST`, so a block waiting on a green pass, a rerun, or an external deploy check resumes as promptly as one waiting on red. Quiet iterations print nothing and cost nothing — no model tokens are spent while the watch is quiet.
|