@plainconceptsplatform/workflows 1.6.0 → 1.6.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +0 -0
- package/loops/actions/decide-agent-retry/action.yml +31 -3
- package/loops/actions/decide-agent-retry/classify-failure.sh +50 -0
- package/loops/actions/housekeeping/action.yml +20 -0
- package/loops/actions/identify-gate-subject/action.yml +36 -19
- package/loops/actions/merge-train/action.yml +5 -0
- package/loops/actions/merge-train/merge-train.sh +130 -33
- package/loops/actions/push-agent-branch/action.yml +14 -1
- package/loops/actions/verify-route-matrix/verify-route-matrix.sh +177 -5
- package/loops/docs/diagrams.md +2 -2
- package/loops/scripts/compile-agent-workflows.mjs +20 -3
- package/loops/workflows/agent-implement.md +78 -32
- package/loops/workflows/agent-merge-gate.md +19 -9
- package/loops/workflows/agent-refine.md +9 -0
- package/loops/workflows/work-router.yml +54 -7
- package/package.json +9 -8
package/dist/index.js
CHANGED
|
File without changes
|
|
@@ -25,6 +25,9 @@ outputs:
|
|
|
25
25
|
minutes:
|
|
26
26
|
description: How long the agent job ran, or -1 when that could not be read.
|
|
27
27
|
value: ${{ steps.decide.outputs.minutes }}
|
|
28
|
+
cause:
|
|
29
|
+
description: 'Why a retry was chosen: short-run, runner-shutdown or cut-off. "none" when there is none.'
|
|
30
|
+
value: ${{ steps.decide.outputs.cause }}
|
|
28
31
|
runs:
|
|
29
32
|
using: composite
|
|
30
33
|
steps:
|
|
@@ -33,6 +36,13 @@ runs:
|
|
|
33
36
|
# repeating it buys the same wrong answer half an hour later. Duration is what separates them,
|
|
34
37
|
# and it needs no model and no log parsing to read.
|
|
35
38
|
#
|
|
39
|
+
# Except when the long run was killed rather than wrong. The VM was deleted under a forty
|
|
40
|
+
# minute implement (Pliny-Bot #699), and an AI gateway that answered nothing ended another
|
|
41
|
+
# run one turn after it began (#693's first attempt): neither is a verdict on the work, and
|
|
42
|
+
# both waited six hours for the janitor. classify-failure.sh reads those two from the job's
|
|
43
|
+
# annotations and from the line above the sandbox teardown, and they are repeated whatever
|
|
44
|
+
# the run lasted, within the same attempt budget.
|
|
45
|
+
#
|
|
36
46
|
# Written once here because the rule was in the implement worker alone, and refine, triage and
|
|
37
47
|
# apply-review parked instead -- an Odyssey refine died in three minutes on
|
|
38
48
|
# `Model 'glm-5-2' not found`, a gateway fault that clears in seconds, and waited on the
|
|
@@ -44,15 +54,16 @@ runs:
|
|
|
44
54
|
GH_TOKEN: ${{ inputs.token }}
|
|
45
55
|
REPO: ${{ github.repository }}
|
|
46
56
|
RUN_ID: ${{ github.run_id }}
|
|
57
|
+
ACTION_PATH: ${{ github.action_path }}
|
|
47
58
|
ATTEMPTS: ${{ inputs.attempts-so-far }}
|
|
48
59
|
PARK_AT: ${{ inputs.park-at }}
|
|
49
60
|
UNDER_MINUTES: ${{ inputs.under-minutes }}
|
|
50
61
|
run: |
|
|
51
62
|
set -euo pipefail
|
|
52
63
|
# The agent job belongs to this same run: a called workflow shares the caller's run id.
|
|
53
|
-
read -r started finished <<<"$(gh api "repos/$REPO/actions/runs/$RUN_ID/jobs?per_page=100" \
|
|
64
|
+
read -r job_id started finished <<<"$(gh api "repos/$REPO/actions/runs/$RUN_ID/jobs?per_page=100" \
|
|
54
65
|
--jq '[.jobs[] | select(.name | endswith("agent"))] | last // empty
|
|
55
|
-
| "\(.started_at // "") \(.completed_at // "")"')"
|
|
66
|
+
| "\(.id // "") \(.started_at // "") \(.completed_at // "")"')"
|
|
56
67
|
minutes=-1
|
|
57
68
|
if [ -n "${started:-}" ] && [ -n "${finished:-}" ]; then
|
|
58
69
|
minutes=$(( ( $(date -u -d "$finished" +%s) - $(date -u -d "$started" +%s) ) / 60 ))
|
|
@@ -60,13 +71,30 @@ runs:
|
|
|
60
71
|
attempts="${ATTEMPTS:-0}"
|
|
61
72
|
[ -n "$attempts" ] || attempts=0
|
|
62
73
|
retry=false
|
|
74
|
+
cause=none
|
|
63
75
|
# An unknown duration is treated as a long run: never retry on a guess.
|
|
64
76
|
if [ "$minutes" -ge 0 ] && [ "$minutes" -lt "$UNDER_MINUTES" ] && [ "$attempts" -lt "$PARK_AT" ]; then
|
|
65
77
|
retry=true
|
|
78
|
+
cause=short-run
|
|
79
|
+
fi
|
|
80
|
+
# A long run is repeated only when it was killed, not when it was wrong. The reads are best
|
|
81
|
+
# effort: a log or an annotation list that cannot be fetched leaves the run parked, exactly
|
|
82
|
+
# as before this check existed.
|
|
83
|
+
if [ "$retry" != true ] && [ "$attempts" -lt "$PARK_AT" ] && [ -n "${job_id:-}" ]; then
|
|
84
|
+
work="$(mktemp -d)"
|
|
85
|
+
gh api "repos/$REPO/check-runs/$job_id/annotations" --paginate --jq '.[].message' >"$work/annotations" 2>/dev/null || true
|
|
86
|
+
gh api "repos/$REPO/actions/jobs/$job_id/logs" >"$work/log" 2>/dev/null || true
|
|
87
|
+
kind="$(bash "$ACTION_PATH/classify-failure.sh" "$work/log" "$work/annotations" || echo none)"
|
|
88
|
+
if [ "$kind" != none ]; then
|
|
89
|
+
retry=true
|
|
90
|
+
cause="$kind"
|
|
91
|
+
fi
|
|
92
|
+
rm -rf "$work"
|
|
66
93
|
fi
|
|
67
94
|
{
|
|
68
95
|
echo "retry=$retry"
|
|
69
96
|
echo "next=$((attempts + 1))"
|
|
70
97
|
echo "minutes=$minutes"
|
|
98
|
+
echo "cause=$cause"
|
|
71
99
|
} >> "$GITHUB_OUTPUT"
|
|
72
|
-
echo "agent job ran for ${minutes}m; attempts so far ${attempts}; retry=${retry}"
|
|
100
|
+
echo "agent job ran for ${minutes}m; attempts so far ${attempts}; retry=${retry}; cause=${cause}"
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# Managed by @plainconceptsplatform/workflows. Source: loops/actions/decide-agent-retry/classify-failure.sh. Update with `workflows update --force`; consumer edits may be overwritten.
|
|
3
|
+
# Say why an agent job died, when the reason is one that repeating the run fixes.
|
|
4
|
+
#
|
|
5
|
+
# Duration alone could not tell the difference. A run that dies in three minutes produced nothing;
|
|
6
|
+
# a run that works for forty minutes and is then killed by its machine produced nothing either, and
|
|
7
|
+
# was parked for the janitor's six-hourly sweep as if the agent had been wrong. Two causes are
|
|
8
|
+
# infrastructure, not judgement:
|
|
9
|
+
#
|
|
10
|
+
# runner-shutdown the VM was taken away mid-job: the process is killed with SIGTERM and GitHub
|
|
11
|
+
# annotates the job "Process completed with exit code 143" (the log has the
|
|
12
|
+
# longer "The runner has received a shutdown signal"), whatever the model was
|
|
13
|
+
# doing. 137, a SIGKILL, is left out: an out-of-memory kill repeats itself
|
|
14
|
+
# cut-off the engine was stopped one turn after it started: the last thing the log shows
|
|
15
|
+
# before the sandbox stops is the header opencode prints when it sends a request
|
|
16
|
+
# to the model, with no reply under it. The AI gateway answered nothing, or an
|
|
17
|
+
# error opencode did not surface, and the run ended with the work unsaid
|
|
18
|
+
#
|
|
19
|
+
# A tool's output in the log can quote either phrase, so neither is searched for in the body: the
|
|
20
|
+
# shutdown is read from the annotations, which only GitHub writes, and the cut-off from the one line
|
|
21
|
+
# immediately above the sandbox teardown.
|
|
22
|
+
#
|
|
23
|
+
# Usage: classify-failure.sh LOG_FILE ANNOTATIONS_FILE
|
|
24
|
+
# Prints one word: runner-shutdown, cut-off or none.
|
|
25
|
+
|
|
26
|
+
set -euo pipefail
|
|
27
|
+
|
|
28
|
+
log="${1:-}"
|
|
29
|
+
annotations="${2:-}"
|
|
30
|
+
|
|
31
|
+
if [ -n "$annotations" ] && [ -f "$annotations" ] && grep -qiE 'exit code 143|runner has received a shutdown signal|lost communication with the server|runner.*(was|has been) (deleted|terminated)' "$annotations"; then
|
|
32
|
+
echo runner-shutdown
|
|
33
|
+
exit 0
|
|
34
|
+
fi
|
|
35
|
+
|
|
36
|
+
if [ -n "$log" ] && [ -f "$log" ]; then
|
|
37
|
+
# The line the sandbox teardown starts on. grep exits 1 for none, which is an answer here.
|
|
38
|
+
stop=$(grep -n '\[INFO\] Stopping containers' "$log" | head -1 | cut -d: -f1 || true)
|
|
39
|
+
if [ -n "$stop" ] && [ "$stop" -gt 1 ]; then
|
|
40
|
+
# The last line with any text above it, ignoring the ANSI colour codes opencode wraps its
|
|
41
|
+
# output in and the timestamp every runner line carries.
|
|
42
|
+
last=$(sed -n "1,$((stop - 1))p" "$log" | sed -E 's/\x1b\[[0-9;]*m//g' | grep -vE '^(\S+Z )?[[:space:]]*$' | tail -1 || true)
|
|
43
|
+
if printf '%s' "$last" | grep -qE '^(\S+Z )?> [A-Za-z0-9_.-]+ · [A-Za-z0-9_./:-]+[[:space:]]*$'; then
|
|
44
|
+
echo cut-off
|
|
45
|
+
exit 0
|
|
46
|
+
fi
|
|
47
|
+
fi
|
|
48
|
+
fi
|
|
49
|
+
|
|
50
|
+
echo none
|
|
@@ -71,13 +71,33 @@ runs:
|
|
|
71
71
|
// timeout. A stuck pending implement or refine run holds a fleet slot that the retry
|
|
72
72
|
// logic below can never free because the run never reaches a terminal state. This
|
|
73
73
|
// cancel runs on a GitHub-hosted runner, so it works even when the fleet is full.
|
|
74
|
+
//
|
|
75
|
+
// Only a run with nothing live ahead of it. The router queues implement runs FIFO
|
|
76
|
+
// (`queue: max`) so that labelling six issues at once implements all six, one after the
|
|
77
|
+
// other, and each of those takes an hour: the sixth waits five hours legitimately. A
|
|
78
|
+
// flat one-hour cutoff cancelled every run behind the first, which left its issue
|
|
79
|
+
// carrying bot-working with no run until the sweep cleared it (Pliny-Bot #706 and #644,
|
|
80
|
+
// both cancelled while #693 ran). A pending run is stuck when no run of its own kind
|
|
81
|
+
// ("Agent: Implement", "Agent: Merge Gate") is running or queued to hold the group.
|
|
74
82
|
const stuckCutoff = Date.now() - 3600 * 1000; // 1 hour ago
|
|
83
|
+
const runKind = (run) => String(run.display_title || '').split(' - ')[0];
|
|
84
|
+
const liveKinds = new Set();
|
|
85
|
+
for (const status of ['in_progress', 'queued']) {
|
|
86
|
+
for (const run of await github.paginate(github.rest.actions.listWorkflowRuns, {
|
|
87
|
+
owner: context.repo.owner, repo: context.repo.repo, workflow_id: 'work-router.yml',
|
|
88
|
+
status, per_page: 50,
|
|
89
|
+
})) liveKinds.add(runKind(run));
|
|
90
|
+
}
|
|
75
91
|
let cancelledRuns = 0;
|
|
76
92
|
for (const run of await github.paginate(github.rest.actions.listWorkflowRuns, {
|
|
77
93
|
owner: context.repo.owner, repo: context.repo.repo, workflow_id: 'work-router.yml',
|
|
78
94
|
status: 'pending', per_page: 50,
|
|
79
95
|
})) {
|
|
80
96
|
if (new Date(run.created_at).getTime() > stuckCutoff) continue;
|
|
97
|
+
if (liveKinds.has(runKind(run))) {
|
|
98
|
+
core.info(`run #${run.id} (${run.display_title}) waits behind a live ${runKind(run)} run; not stuck`);
|
|
99
|
+
continue;
|
|
100
|
+
}
|
|
81
101
|
await act(`cancelling stuck pending run #${run.id} (queued ${new Date(run.created_at).toISOString()})`, async () => {
|
|
82
102
|
await github.rest.actions.cancelWorkflowRun({
|
|
83
103
|
owner: context.repo.owner, repo: context.repo.repo, run_id: run.id,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Managed by @plainconceptsplatform/workflows. Source: loops/actions/identify-gate-subject/action.yml. Update with workflows update --force; consumer edits may be overwritten.
|
|
2
2
|
name: Identify gate subject
|
|
3
|
-
description: Confirm the pull request is one the gate may act on,
|
|
3
|
+
description: Confirm the pull request is one the gate may act on (same repository, opened by a trusted bot or someone with write access) and resolve the open issue it closes, if any.
|
|
4
4
|
inputs:
|
|
5
5
|
token:
|
|
6
6
|
description: GitHub token with issues and pull-requests read access.
|
|
@@ -24,10 +24,6 @@ inputs:
|
|
|
24
24
|
description: Issue the pull request closes, when the router already resolved it.
|
|
25
25
|
required: false
|
|
26
26
|
default: ''
|
|
27
|
-
require-label:
|
|
28
|
-
description: Label a bot pull request's closing issue must carry. Empty disables the check. A person's pull request needs neither an issue nor the label.
|
|
29
|
-
required: false
|
|
30
|
-
default: ''
|
|
31
27
|
outputs:
|
|
32
28
|
found:
|
|
33
29
|
description: Whether a valid gate subject was found.
|
|
@@ -66,7 +62,6 @@ runs:
|
|
|
66
62
|
CONCLUSION: ${{ inputs.ci-conclusion }}
|
|
67
63
|
CI_RUN_ID: ${{ inputs.ci-run-id }}
|
|
68
64
|
LINKED_ISSUE: ${{ inputs.linked-issue }}
|
|
69
|
-
REQUIRE_LABEL: ${{ inputs.require-label }}
|
|
70
65
|
TRUSTED_BOTS: ${{ inputs.trusted-bots }}
|
|
71
66
|
run: |
|
|
72
67
|
set -euo pipefail
|
|
@@ -115,19 +110,16 @@ runs:
|
|
|
115
110
|
grep -oE '[0-9]+' | head -n 1 || true)"
|
|
116
111
|
fi
|
|
117
112
|
|
|
118
|
-
#
|
|
119
|
-
#
|
|
120
|
-
#
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
if ! jq -e --arg name "$REQUIRE_LABEL" 'index($name)' >/dev/null <<<"$labels"; then
|
|
130
|
-
none "issue #$issue does not carry $REQUIRE_LABEL (has: $(jq -r 'join(", ")' <<<"$labels"))"
|
|
113
|
+
# An issue that is already closed is not the work this pull request is for: the feature
|
|
114
|
+
# landed some other way, and the label on a closed issue says nothing. Comments, labels
|
|
115
|
+
# and the acceptance criteria then come from the pull request itself. A pull request is
|
|
116
|
+
# never refused for lacking an issue: the merge train hands the gate every pull request
|
|
117
|
+
# that can move, and a gate that declines one without a trace is handed it again.
|
|
118
|
+
if [ -n "$issue" ]; then
|
|
119
|
+
issue_state="$(gh issue view "$issue" --repo "$REPO" --json state --jq '.state' 2>/dev/null || echo UNKNOWN)"
|
|
120
|
+
if [ "$issue_state" != OPEN ]; then
|
|
121
|
+
echo "Issue #$issue is $issue_state; gating the pull request on its own description."
|
|
122
|
+
issue=""
|
|
131
123
|
fi
|
|
132
124
|
fi
|
|
133
125
|
|
|
@@ -140,6 +132,31 @@ runs:
|
|
|
140
132
|
# both stalled on a human that way while CI was legitimately red and remediable.
|
|
141
133
|
ci_conclusion="$CONCLUSION"
|
|
142
134
|
ci_run_id="$CI_RUN_ID"
|
|
135
|
+
|
|
136
|
+
# A verdict from a CI run that judged an older head is not a verdict on this one. The
|
|
137
|
+
# router carries the run it saw, and the head can move before the gate starts: a repair
|
|
138
|
+
# pushed, a branch updated from main. Pliny-Bot #729 was blocked and sent to review on a
|
|
139
|
+
# red run of a commit that two later pushes had already fixed, and the gate's own report
|
|
140
|
+
# said so. When the run's commit is not the head, the carried verdict is dropped and read
|
|
141
|
+
# again from the head's own latest run; when that head has no finished run yet, there is
|
|
142
|
+
# nothing to judge, and the gate steps aside until CI finishes and the train hands the
|
|
143
|
+
# pull request back.
|
|
144
|
+
if [ -n "$ci_run_id" ]; then
|
|
145
|
+
run_sha="$(gh api "repos/$REPO/actions/runs/$ci_run_id" --jq '.head_sha' 2>/dev/null || true)"
|
|
146
|
+
current_sha="$(gh pr view "$PR_NUMBER" --repo "$REPO" --json headRefOid --jq '.headRefOid')"
|
|
147
|
+
if [ -n "$run_sha" ] && [ "$run_sha" != "$current_sha" ]; then
|
|
148
|
+
echo "::notice::CI run $ci_run_id judged ${run_sha:0:7}, but the head is ${current_sha:0:7}; reading the verdict again from the head."
|
|
149
|
+
ci_conclusion=""
|
|
150
|
+
ci_run_id=""
|
|
151
|
+
fresh="$(gh api "repos/$REPO/actions/runs?head_sha=$current_sha&per_page=100" --jq '[.workflow_runs[] | select((.name | ascii_downcase) == "app: ci" or (.name | ascii_downcase) == "ci") | select(.status == "completed")] | sort_by(.created_at) | last')"
|
|
152
|
+
if [ -z "$fresh" ] || [ "$fresh" = "null" ]; then
|
|
153
|
+
none "CI has not finished on the current head of PR #$PR_NUMBER; the train hands it back when it has"
|
|
154
|
+
fi
|
|
155
|
+
ci_conclusion="$(jq -r '.conclusion // "pending"' <<<"$fresh")"
|
|
156
|
+
ci_run_id="$(jq -r '.id' <<<"$fresh")"
|
|
157
|
+
fi
|
|
158
|
+
fi
|
|
159
|
+
|
|
143
160
|
if [ -z "$ci_conclusion" ] || [ -z "$ci_run_id" ]; then
|
|
144
161
|
head_sha="$(gh pr view "$PR_NUMBER" --repo "$REPO" --json headRefOid --jq '.headRefOid')"
|
|
145
162
|
# A run keeps the name it had when it ran, so a repository whose CI was renamed still
|
|
@@ -28,6 +28,10 @@ inputs:
|
|
|
28
28
|
description: How many times a blocked verdict, or a merge that did not happen, is retried after the base branch moves.
|
|
29
29
|
required: false
|
|
30
30
|
default: '3'
|
|
31
|
+
max-repairs:
|
|
32
|
+
description: How many pull requests may be in repair at once. Merges stay one at a time.
|
|
33
|
+
required: false
|
|
34
|
+
default: '1'
|
|
31
35
|
stuck-hours:
|
|
32
36
|
description: How long the train waits behind a CI run that never finishes before it passes that pull request by.
|
|
33
37
|
required: false
|
|
@@ -60,6 +64,7 @@ runs:
|
|
|
60
64
|
MAX_ATTEMPTS: ${{ inputs.max-gate-attempts }}
|
|
61
65
|
MAX_RETRIES: ${{ inputs.max-retries }}
|
|
62
66
|
STUCK_HOURS: ${{ inputs.stuck-hours }}
|
|
67
|
+
MAX_REPAIRS: ${{ inputs.max-repairs }}
|
|
63
68
|
DRY_RUN: ${{ inputs.dry-run }}
|
|
64
69
|
HINT_RUN_ID: ${{ inputs.ci-run-id }}
|
|
65
70
|
HINT_CONCLUSION: ${{ inputs.ci-conclusion }}
|
|
@@ -1,19 +1,20 @@
|
|
|
1
1
|
#!/usr/bin/env bash
|
|
2
2
|
# Managed by @plainconceptsplatform/workflows. Source: loops/actions/merge-train/merge-train.sh. Update with `workflows update --force`; consumer edits may be overwritten.
|
|
3
|
-
# Move the merge train
|
|
3
|
+
# Move the merge train: look at every open pull request, oldest first, and do for each the one
|
|
4
|
+
# thing it needs, with merges one at a time and repairs in parallel.
|
|
4
5
|
#
|
|
5
6
|
# The belt this replaces dispatched the gate for every eligible pull request at once. Each gate
|
|
6
7
|
# then resolved its branch against a default branch the gate ahead of it was about to move, so a
|
|
7
8
|
# night of ten pull requests became ten repairs, ten CI runs and ten more gates after every merge,
|
|
8
|
-
# and a verdict that parked a pull request held it however stale it went.
|
|
9
|
-
# pull request in flight, lowest number first. Everything else waits, untouched, until the one in
|
|
10
|
-
# front of it has merged or been parked for a person.
|
|
9
|
+
# and a verdict that parked a pull request held it however stale it went.
|
|
11
10
|
#
|
|
12
|
-
#
|
|
13
|
-
#
|
|
14
|
-
#
|
|
15
|
-
#
|
|
11
|
+
# What races is a merge, because it moves the default branch under everyone else. A repair (a gate
|
|
12
|
+
# handed red CI) pushes to its own branch and can never merge, so it races nothing. So:
|
|
13
|
+
# merge one at a time, the oldest ready pull request first; nothing behind a pull request that
|
|
14
|
+
# is waiting on its CI, or behind a gate that may merge, merges or is brought up to date
|
|
15
|
+
# repair a red or conflicting pull request goes to the gate at once, up to MAX_REPAIRS together
|
|
16
16
|
# approve its CI is waiting for approval and a trusted bot opened it
|
|
17
|
+
# update it is green but behind the default branch: merge the default branch in, CI reruns
|
|
17
18
|
# A pull request a person has to decide on, or that carries the hold label, is skipped, so it never
|
|
18
19
|
# blocks the line. Ticks come from every merge, every CI completion, every finished gate, and the
|
|
19
20
|
# hourly schedule as the fallback.
|
|
@@ -27,11 +28,20 @@ set -euo pipefail
|
|
|
27
28
|
# HOLD true when the pull request carries the hold label
|
|
28
29
|
# REVIEW_DECISION GitHub's review decision (CHANGES_REQUESTED holds it for its author)
|
|
29
30
|
# PARK none, or held when a standing gate verdict covers this head
|
|
30
|
-
# ATTEMPTS failed gate attempts on this head; MAX_ATTEMPTS is the budget
|
|
31
|
+
# ATTEMPTS failed gate attempts on this head; MAX_ATTEMPTS is the budget. Counts the
|
|
32
|
+
# larger of the attempt comments and the gate runs, so a gate that fails
|
|
33
|
+
# without a comment still spends it
|
|
34
|
+
# SILENT_GATE true when a gate already ran on this head and left no verdict and no attempt
|
|
35
|
+
# comment: it declined, or died before it could say so. Handing it the same
|
|
36
|
+
# pull request again reproduces that, a hot loop with every tick
|
|
31
37
|
# MERGEABLE MERGEABLE, CONFLICTING or UNKNOWN
|
|
32
38
|
# CI_STATUS none, or the latest CI run's status (queued, in_progress, completed, ...)
|
|
33
39
|
# CI_CONCLUSION that run's conclusion when it completed
|
|
34
40
|
# CI_STUCK true when that run has been pending longer than the train waits
|
|
41
|
+
# SAME_AS_MAIN true when every check that failed on the head also fails on the base branch
|
|
42
|
+
# tip: the failure is the base's, and repairing the pull request spends its
|
|
43
|
+
# attempts on code that is not at fault (Pliny-Bot, 2026-10-17: a red main
|
|
44
|
+
# cost every open pull request its repairs for seven hours)
|
|
35
45
|
# BEHIND_BY commits on the base branch the head does not have
|
|
36
46
|
# TRUSTED_AUTHOR true when a trusted bot opened it (only those runs are approved)
|
|
37
47
|
train_move() {
|
|
@@ -45,6 +55,8 @@ train_move() {
|
|
|
45
55
|
move=skip; reason="a gate verdict on this head is waiting for a person"
|
|
46
56
|
elif [ "${ATTEMPTS:-0}" -ge "${MAX_ATTEMPTS:-6}" ]; then
|
|
47
57
|
move=skip; reason="its gate attempt budget on this head is spent"
|
|
58
|
+
elif [ "${SILENT_GATE:-false}" = true ]; then
|
|
59
|
+
move=skip; reason="its gate already ran on this head and left no verdict or attempt"
|
|
48
60
|
elif [ "${MERGEABLE:-UNKNOWN}" = CONFLICTING ]; then
|
|
49
61
|
# A conflicting pull request has no merge ref, so CI can never run on it: waiting for a
|
|
50
62
|
# verdict would wait forever. Resolving the conflict is the gate's job.
|
|
@@ -62,7 +74,13 @@ train_move() {
|
|
|
62
74
|
move=gate; reason="green and up to date"
|
|
63
75
|
fi
|
|
64
76
|
;;
|
|
65
|
-
failure)
|
|
77
|
+
failure)
|
|
78
|
+
if [ "${SAME_AS_MAIN:-false}" = true ]; then
|
|
79
|
+
move="wait"; reason="CI failed, but only on checks that fail on the base branch too; waiting for the base to be fixed"
|
|
80
|
+
else
|
|
81
|
+
move=gate; reason="CI failed"
|
|
82
|
+
fi
|
|
83
|
+
;;
|
|
66
84
|
action_required)
|
|
67
85
|
if [ "${TRUSTED_AUTHOR:-false}" = true ]; then
|
|
68
86
|
move=approve; reason="CI is waiting for approval"
|
|
@@ -126,6 +144,8 @@ TRUSTED_BOTS="${TRUSTED_BOTS:-}"
|
|
|
126
144
|
HOLD_LABEL="${HOLD_LABEL:-hold}"
|
|
127
145
|
MAX_ATTEMPTS="${MAX_ATTEMPTS:-6}"
|
|
128
146
|
MAX_RETRIES="${MAX_RETRIES:-3}"
|
|
147
|
+
MAX_REPAIRS="${MAX_REPAIRS:-1}"
|
|
148
|
+
case "$MAX_REPAIRS" in ''|*[!0-9]*|0) MAX_REPAIRS=1 ;; esac
|
|
129
149
|
STUCK_HOURS="${STUCK_HOURS:-3}"
|
|
130
150
|
DRY_RUN="${DRY_RUN:-false}"
|
|
131
151
|
SELF_RUN_ID="${GITHUB_RUN_ID:-0}"
|
|
@@ -168,18 +188,18 @@ can_ride() {
|
|
|
168
188
|
[ "${WRITE_ACCESS[$login]}" = true ]
|
|
169
189
|
}
|
|
170
190
|
|
|
171
|
-
#
|
|
172
|
-
#
|
|
173
|
-
#
|
|
174
|
-
|
|
191
|
+
# The gates on the belt, one title per line. A gate handed red CI repairs and pushes and cannot
|
|
192
|
+
# merge, so run-name gives it a " (repair)" suffix and the router queues it per pull request;
|
|
193
|
+
# every other gate may merge, and there is one of those at a time across the repository. The run
|
|
194
|
+
# this tick belongs to is excluded: the tick after a gate runs inside the gate's own router run.
|
|
195
|
+
live_gates() {
|
|
175
196
|
local status
|
|
176
197
|
for status in queued in_progress waiting pending requested; do
|
|
177
|
-
gh api "repos/$REPO/actions/workflows/work-router.yml/runs?status=$status&per_page=100"
|
|
178
|
-
--jq ".workflow_runs[]
|
|
198
|
+
gh api "repos/$REPO/actions/workflows/work-router.yml/runs?status=$status&per_page=100" --jq ".workflow_runs[]
|
|
179
199
|
| select(.id != $SELF_RUN_ID)
|
|
180
200
|
| select(.display_title | startswith(\"$GATE_TITLE_PREFIX\"))
|
|
181
|
-
| .display_title" 2>/dev/null
|
|
182
|
-
done |
|
|
201
|
+
| .display_title" 2>/dev/null
|
|
202
|
+
done | sort -u
|
|
183
203
|
}
|
|
184
204
|
|
|
185
205
|
# GitHub works mergeability out on demand: the first read of a pull request nobody has opened
|
|
@@ -220,6 +240,14 @@ update_branch() {
|
|
|
220
240
|
gh api "repos/$REPO/pulls/$pr/update-branch" --method PUT -f expected_head_sha="$head_sha" >/dev/null
|
|
221
241
|
}
|
|
222
242
|
|
|
243
|
+
# How many gate runs were started for a pull request since a moment, whatever became of them. A
|
|
244
|
+
# run is titled by the router's run-name; this is the one count that does not depend on the gate
|
|
245
|
+
# having written anything.
|
|
246
|
+
gate_run_count() {
|
|
247
|
+
local pr="$1" since="$2"
|
|
248
|
+
gh api --paginate "repos/$REPO/actions/workflows/work-router.yml/runs?event=workflow_dispatch&per_page=100" --jq "[.workflow_runs[] | select(.display_title == \"$GATE_TITLE_PREFIX$pr\") | select(.created_at > \"$since\")] | length" 2>/dev/null | awk '{ n += $1 } END { print n + 0 }'
|
|
249
|
+
}
|
|
250
|
+
|
|
223
251
|
attempt_count() {
|
|
224
252
|
local number="$1" since="$2"
|
|
225
253
|
gh api --paginate "repos/$REPO/issues/$number/comments?per_page=100" \
|
|
@@ -241,7 +269,7 @@ dispatch_gate() {
|
|
|
241
269
|
# The next tick waits behind this one and must see the gate it started, or it starts a second.
|
|
242
270
|
for _ in $(seq 1 12); do
|
|
243
271
|
if gh api "repos/$REPO/actions/workflows/work-router.yml/runs?event=workflow_dispatch&created=>=$since&per_page=20" \
|
|
244
|
-
--jq ".workflow_runs[] | select(.display_title
|
|
272
|
+
--jq ".workflow_runs[] | select(.display_title | startswith(\"$GATE_TITLE_PREFIX$pr\")) | .id" 2>/dev/null | grep -q .; then
|
|
245
273
|
return 0
|
|
246
274
|
fi
|
|
247
275
|
sleep 5
|
|
@@ -252,11 +280,21 @@ dispatch_gate() {
|
|
|
252
280
|
summary "### Merge train"
|
|
253
281
|
summary ""
|
|
254
282
|
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
283
|
+
live_titles="$(live_gates)"
|
|
284
|
+
live_repairs=$(printf '%s
|
|
285
|
+
' "$live_titles" | grep -c ' (repair)$' || true)
|
|
286
|
+
live_merge=$(printf '%s
|
|
287
|
+
' "$live_titles" | grep -v ' (repair)$' | grep -c . || true)
|
|
288
|
+
live_for_pr() { printf '%s
|
|
289
|
+
' "$live_titles" | grep -qE "PR #$1( \(repair\))?$"; }
|
|
290
|
+
echo "On the belt: ${live_merge} merging gate(s), ${live_repairs} repair(s). Up to ${MAX_REPAIRS} repair(s) at once."
|
|
291
|
+
summary "On the belt: ${live_merge} merging gate(s), ${live_repairs} repair(s); up to ${MAX_REPAIRS} repair(s) at once."
|
|
292
|
+
summary ""
|
|
293
|
+
|
|
294
|
+
# Once a pull request ahead is waiting on its CI, or a gate that may merge is on the belt, nothing
|
|
295
|
+
# behind it may merge or be brought up to date: the oldest merges first, and one at a time.
|
|
296
|
+
merge_blocked=false
|
|
297
|
+
[ "$live_merge" -eq 0 ] || merge_blocked=true
|
|
260
298
|
|
|
261
299
|
base_head_date=""
|
|
262
300
|
base_head_date_for() {
|
|
@@ -332,6 +370,7 @@ for line in "${candidates[@]}"; do
|
|
|
332
370
|
RETRIES="$retries" MAX_RETRIES="$MAX_RETRIES" STACK_ROOT="$stack_root" train_park)
|
|
333
371
|
|
|
334
372
|
attempts=0
|
|
373
|
+
silent_gate=false
|
|
335
374
|
if [ "$park" = none ]; then
|
|
336
375
|
horizon="${head_committed:-1970-01-01T00:00:00Z}"
|
|
337
376
|
# A retried verdict starts a fresh budget from the verdict it retries, not from the head.
|
|
@@ -340,6 +379,12 @@ for line in "${candidates[@]}"; do
|
|
|
340
379
|
if [ -n "$linked_issue" ]; then
|
|
341
380
|
attempts=$((attempts + $(attempt_count "$linked_issue" "$horizon")))
|
|
342
381
|
fi
|
|
382
|
+
gate_runs=$(gate_run_count "$pr" "$horizon")
|
|
383
|
+
# Comments say what the gate decided. Runs say it was asked. A run on this head that left no
|
|
384
|
+
# attempt comment and no verdict is a gate that did nothing, and the next tick must not ask
|
|
385
|
+
# again: that is the loop.
|
|
386
|
+
if [ "$gate_runs" -gt 0 ] && [ "$attempts" -eq 0 ]; then silent_gate=true; fi
|
|
387
|
+
[ "$gate_runs" -le "$attempts" ] || attempts="$gate_runs"
|
|
343
388
|
fi
|
|
344
389
|
|
|
345
390
|
ci=$(gh api "repos/$REPO/actions/runs?head_sha=$head_sha&per_page=100" \
|
|
@@ -368,36 +413,71 @@ for line in "${candidates[@]}"; do
|
|
|
368
413
|
fi
|
|
369
414
|
fi
|
|
370
415
|
|
|
416
|
+
# What failed on the head, compared with what is failing on the base branch tip. Names are
|
|
417
|
+
# compared after the last " / ": a check that a reusable workflow runs is "CI / backend" on the
|
|
418
|
+
# base branch and "backend" on a pull request. Only when every failed check is also failing on
|
|
419
|
+
# the base is the pull request left alone.
|
|
420
|
+
same_as_main=false
|
|
421
|
+
if [ "$ci_status" = completed ] && [ "$ci_conclusion" = failure ] && [ "$park" = none ]; then
|
|
422
|
+
failed_here=$(gh api "repos/$REPO/commits/$head_sha/check-runs?per_page=100" \
|
|
423
|
+
--jq '[.check_runs[] | select(.conclusion == "failure") | (.name | sub("^.* / "; ""))] | unique | .[]' 2>/dev/null || true)
|
|
424
|
+
failed_base=$(gh api "repos/$REPO/commits/$base_ref/check-runs?per_page=100" \
|
|
425
|
+
--jq '[.check_runs[] | select(.conclusion == "failure") | (.name | sub("^.* / "; ""))] | unique | .[]' 2>/dev/null || true)
|
|
426
|
+
if [ -n "$failed_here" ] && [ -n "$failed_base" ] &&
|
|
427
|
+
[ -z "$(comm -23 <(sort <<<"$failed_here") <(sort <<<"$failed_base"))" ]; then
|
|
428
|
+
same_as_main=true
|
|
429
|
+
fi
|
|
430
|
+
fi
|
|
431
|
+
|
|
371
432
|
trusted_author=false
|
|
372
433
|
is_trusted_bot "$author" && trusted_author=true
|
|
373
434
|
|
|
374
435
|
decision=$(HOLD="$hold" REVIEW_DECISION="$review_decision" PARK="$park" \
|
|
375
|
-
ATTEMPTS="$attempts" MAX_ATTEMPTS="$MAX_ATTEMPTS" MERGEABLE="$mergeable" \
|
|
376
|
-
CI_STATUS="$ci_status" CI_CONCLUSION="$ci_conclusion" CI_STUCK="$ci_stuck" \
|
|
436
|
+
ATTEMPTS="$attempts" MAX_ATTEMPTS="$MAX_ATTEMPTS" SILENT_GATE="$silent_gate" MERGEABLE="$mergeable" \
|
|
437
|
+
CI_STATUS="$ci_status" CI_CONCLUSION="$ci_conclusion" CI_STUCK="$ci_stuck" SAME_AS_MAIN="$same_as_main" \
|
|
377
438
|
BEHIND_BY="${behind_by:-0}" TRUSTED_AUTHOR="$trusted_author" train_move)
|
|
378
439
|
move=$(sed -n 's/^move=//p' <<<"$decision")
|
|
379
440
|
reason=$(sed -n 's/^reason=//p' <<<"$decision")
|
|
380
441
|
|
|
442
|
+
# A gate is already on this pull request, whatever kind: offering it another would run the same
|
|
443
|
+
# work twice. It does not hold the line up for the repair kind; a merging gate already does.
|
|
444
|
+
if [ "$move" != skip ] && live_for_pr "$pr"; then
|
|
445
|
+
move=skip
|
|
446
|
+
reason="a gate is already on it"
|
|
447
|
+
fi
|
|
448
|
+
|
|
381
449
|
echo "#$pr ($author): $move, $reason."
|
|
382
450
|
summary "| #$pr | $author | $move | $reason |"
|
|
383
451
|
|
|
384
452
|
case "$move" in
|
|
385
453
|
skip) continue ;;
|
|
386
|
-
wait)
|
|
454
|
+
wait)
|
|
455
|
+
# Its CI is running: nothing behind it may merge or be brought up to date first, but the
|
|
456
|
+
# pull requests behind it that need repair do not wait for anyone.
|
|
457
|
+
merge_blocked=true
|
|
458
|
+
continue
|
|
459
|
+
;;
|
|
387
460
|
approve)
|
|
388
461
|
act "Approving the CI runs waiting on #$pr." approve_runs "$head_ref"
|
|
389
|
-
|
|
462
|
+
merge_blocked=true
|
|
463
|
+
continue
|
|
390
464
|
;;
|
|
391
465
|
update)
|
|
466
|
+
if [ "$merge_blocked" = true ]; then
|
|
467
|
+
echo " #$pr waits for the pull request ahead of it before it is brought up to date."
|
|
468
|
+
continue
|
|
469
|
+
fi
|
|
392
470
|
# Merge, never rebase: the gate's pushes are fast-forward only, and so are a person's.
|
|
393
471
|
if act "Merging $base_ref into #$pr so CI judges it against what it will land on." \
|
|
394
472
|
update_branch "$pr" "$head_sha"; then
|
|
395
|
-
|
|
473
|
+
merge_blocked=true
|
|
474
|
+
continue
|
|
396
475
|
fi
|
|
397
476
|
echo "::warning::GitHub could not update #$pr from $base_ref; handing it to the gate."
|
|
398
477
|
act "Dispatching the merge gate for #$pr to bring it up to date." \
|
|
399
478
|
dispatch_gate "$pr" success "${ci_run_id:-}" "$attempts"
|
|
400
|
-
|
|
479
|
+
merge_blocked=true
|
|
480
|
+
continue
|
|
401
481
|
;;
|
|
402
482
|
gate)
|
|
403
483
|
conclusion="${ci_conclusion:-}"
|
|
@@ -412,13 +492,30 @@ for line in "${candidates[@]}"; do
|
|
|
412
492
|
conclusion=$(cut -f2 <<<"$last")
|
|
413
493
|
[ -n "$conclusion" ] || conclusion=failure
|
|
414
494
|
fi
|
|
495
|
+
if [ "$conclusion" = failure ]; then
|
|
496
|
+
# A repair: it pushes and can never merge, so it races nothing. Up to MAX_REPAIRS at once.
|
|
497
|
+
if [ "$live_repairs" -ge "$MAX_REPAIRS" ]; then
|
|
498
|
+
echo " #$pr needs a repair; ${live_repairs} already in progress, the limit is ${MAX_REPAIRS}."
|
|
499
|
+
continue
|
|
500
|
+
fi
|
|
501
|
+
act "Dispatching a repair gate for #$pr (CI failed, attempt $((attempts + 1)) of $MAX_ATTEMPTS)." \
|
|
502
|
+
dispatch_gate "$pr" failure "${ci_run_id:-}" "$attempts"
|
|
503
|
+
live_repairs=$((live_repairs + 1))
|
|
504
|
+
continue
|
|
505
|
+
fi
|
|
506
|
+
# CI is green on the verdict: this gate may merge, so it goes only when nothing is ahead of it.
|
|
507
|
+
if [ "$merge_blocked" = true ]; then
|
|
508
|
+
echo " #$pr is ready for the gate; it waits for the pull request ahead of it."
|
|
509
|
+
continue
|
|
510
|
+
fi
|
|
415
511
|
act "Dispatching the merge gate for #$pr (CI $conclusion, attempt $((attempts + 1)) of $MAX_ATTEMPTS)." \
|
|
416
512
|
dispatch_gate "$pr" "$conclusion" "${ci_run_id:-}" "$attempts"
|
|
417
|
-
|
|
513
|
+
merge_blocked=true
|
|
514
|
+
continue
|
|
418
515
|
;;
|
|
419
516
|
esac
|
|
420
517
|
done
|
|
421
518
|
|
|
422
|
-
echo "
|
|
519
|
+
echo "That is every open pull request this tick could move."
|
|
423
520
|
summary ""
|
|
424
|
-
summary "
|
|
521
|
+
summary "Every open pull request was looked at; those that could move have."
|
|
@@ -24,7 +24,20 @@ runs:
|
|
|
24
24
|
env:
|
|
25
25
|
GH_TOKEN: ${{ inputs.token }}
|
|
26
26
|
run: |
|
|
27
|
-
|
|
27
|
+
set -euo pipefail
|
|
28
|
+
# The runners here outlive their jobs, and `git config --global` outlives the job with
|
|
29
|
+
# them. This step used to add one `url.<token>.insteadOf` rule per run and never remove it,
|
|
30
|
+
# so a runner that had pushed before held several rules for the same prefix, each carrying
|
|
31
|
+
# a token revoked since, and git picks among equal-length matches arbitrarily. The same
|
|
32
|
+
# gate pushed for one pull request and was refused for the next: "Invalid username or
|
|
33
|
+
# token", then "Target branch does not exist" because ls-remote exited 128 (Pliny-Bot #716
|
|
34
|
+
# failed twice that way while #717 pushed in between). Drop every such rule wherever it
|
|
35
|
+
# lives, and set this run's rule in the checkout only.
|
|
36
|
+
for scope in --global --local; do
|
|
37
|
+
git config "$scope" --get-regexp '^url\..*x-access-token.*\.insteadof$' 2>/dev/null |
|
|
38
|
+
while read -r key _; do git config "$scope" --unset-all "$key" || true; done || true
|
|
39
|
+
done
|
|
40
|
+
git config --local url."https://x-access-token:${GH_TOKEN}@github.com/".insteadOf "https://github.com/"
|
|
28
41
|
- name: Read pull request branch
|
|
29
42
|
id: request
|
|
30
43
|
shell: bash
|