@plainconceptsplatform/workflows 1.6.10 → 1.6.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/loops/actions/housekeeping/action.yml +15 -0
- package/loops/actions/identify-gate-subject/action.yml +25 -0
- package/loops/actions/merge-train/action.yml +7 -0
- package/loops/actions/merge-train/merge-train.sh +56 -2
- package/loops/actions/report-workflow-errors/action.yml +92 -23
- package/loops/actions/report-workflow-errors/report-policy.cjs +110 -0
- package/loops/actions/verify-route-matrix/verify-route-matrix.sh +122 -0
- package/loops/docs/diagrams.md +8 -4
- package/loops/templates/agentics/agentics-error-report.yml +2 -0
- package/loops/workflows/agent-apply-review.md +2 -0
- package/loops/workflows/agent-implement.md +55 -8
- package/package.json +1 -1
|
@@ -499,6 +499,21 @@ runs:
|
|
|
499
499
|
}
|
|
500
500
|
}
|
|
501
501
|
|
|
502
|
+
// The merge train wakes when a CI run finishes. When a gate it was waiting on ends without
|
|
503
|
+
// one, or is cancelled, nothing calls it again and every pull request sits until the next
|
|
504
|
+
// CI event: Pliny-Bot had four green pull requests waiting on that for the better part of an
|
|
505
|
+
// hour. The hourly sweep is the heartbeat. A dispatch of the router by the workflow token is
|
|
506
|
+
// allowed to start a run, and a tick with nothing to do costs one short job.
|
|
507
|
+
const openPulls = pulls.filter((pr) => !pr.draft);
|
|
508
|
+
if (openPulls.length > 0) {
|
|
509
|
+
await act(`dispatching a merge train tick: ${openPulls.length} pull request(s) are open`, () =>
|
|
510
|
+
github.rest.actions.createWorkflowDispatch({
|
|
511
|
+
...context.repo, workflow_id: 'work-router.yml',
|
|
512
|
+
ref: context.payload.repository?.default_branch ?? 'main',
|
|
513
|
+
inputs: { operation: 'merge-train' },
|
|
514
|
+
}));
|
|
515
|
+
}
|
|
516
|
+
|
|
502
517
|
await core.summary
|
|
503
518
|
.addHeading('Housekeeping', 2)
|
|
504
519
|
.addList([
|
|
@@ -132,6 +132,31 @@ runs:
|
|
|
132
132
|
# both stalled on a human that way while CI was legitimately red and remediable.
|
|
133
133
|
ci_conclusion="$CONCLUSION"
|
|
134
134
|
ci_run_id="$CI_RUN_ID"
|
|
135
|
+
|
|
136
|
+
# A verdict from a CI run that judged an older head is not a verdict on this one. The
|
|
137
|
+
# router carries the run it saw, and the head can move before the gate starts: a repair
|
|
138
|
+
# pushed, a branch updated from main. Pliny-Bot #729 was blocked and sent to review on a
|
|
139
|
+
# red run of a commit that two later pushes had already fixed, and the gate's own report
|
|
140
|
+
# said so. When the run's commit is not the head, the carried verdict is dropped and read
|
|
141
|
+
# again from the head's own latest run; when that head has no finished run yet, there is
|
|
142
|
+
# nothing to judge, and the gate steps aside until CI finishes and the train hands the
|
|
143
|
+
# pull request back.
|
|
144
|
+
if [ -n "$ci_run_id" ]; then
|
|
145
|
+
run_sha="$(gh api "repos/$REPO/actions/runs/$ci_run_id" --jq '.head_sha' 2>/dev/null || true)"
|
|
146
|
+
current_sha="$(gh pr view "$PR_NUMBER" --repo "$REPO" --json headRefOid --jq '.headRefOid')"
|
|
147
|
+
if [ -n "$run_sha" ] && [ "$run_sha" != "$current_sha" ]; then
|
|
148
|
+
echo "::notice::CI run $ci_run_id judged ${run_sha:0:7}, but the head is ${current_sha:0:7}; reading the verdict again from the head."
|
|
149
|
+
ci_conclusion=""
|
|
150
|
+
ci_run_id=""
|
|
151
|
+
fresh="$(gh api "repos/$REPO/actions/runs?head_sha=$current_sha&per_page=100" --jq '[.workflow_runs[] | select((.name | ascii_downcase) == "app: ci" or (.name | ascii_downcase) == "ci") | select(.status == "completed")] | sort_by(.created_at) | last')"
|
|
152
|
+
if [ -z "$fresh" ] || [ "$fresh" = "null" ]; then
|
|
153
|
+
none "CI has not finished on the current head of PR #$PR_NUMBER; the train hands it back when it has"
|
|
154
|
+
fi
|
|
155
|
+
ci_conclusion="$(jq -r '.conclusion // "pending"' <<<"$fresh")"
|
|
156
|
+
ci_run_id="$(jq -r '.id' <<<"$fresh")"
|
|
157
|
+
fi
|
|
158
|
+
fi
|
|
159
|
+
|
|
135
160
|
if [ -z "$ci_conclusion" ] || [ -z "$ci_run_id" ]; then
|
|
136
161
|
head_sha="$(gh pr view "$PR_NUMBER" --repo "$REPO" --json headRefOid --jq '.headRefOid')"
|
|
137
162
|
# A run keeps the name it had when it ran, so a repository whose CI was renamed still
|
|
@@ -32,6 +32,12 @@ inputs:
|
|
|
32
32
|
description: How many pull requests may be in repair at once. Merges stay one at a time.
|
|
33
33
|
required: false
|
|
34
34
|
default: '1'
|
|
35
|
+
main-red-wait-hours:
|
|
36
|
+
description: >-
|
|
37
|
+
How long the train waits for a base branch whose checks fail the same way as a pull request's
|
|
38
|
+
before it stops waiting and repairs the pull request anyway.
|
|
39
|
+
required: false
|
|
40
|
+
default: '2'
|
|
35
41
|
stuck-hours:
|
|
36
42
|
description: How long the train waits behind a CI run that never finishes before it passes that pull request by.
|
|
37
43
|
required: false
|
|
@@ -64,6 +70,7 @@ runs:
|
|
|
64
70
|
MAX_ATTEMPTS: ${{ inputs.max-gate-attempts }}
|
|
65
71
|
MAX_RETRIES: ${{ inputs.max-retries }}
|
|
66
72
|
STUCK_HOURS: ${{ inputs.stuck-hours }}
|
|
73
|
+
MAIN_RED_WAIT_HOURS: ${{ inputs.main-red-wait-hours }}
|
|
67
74
|
MAX_REPAIRS: ${{ inputs.max-repairs }}
|
|
68
75
|
DRY_RUN: ${{ inputs.dry-run }}
|
|
69
76
|
HINT_RUN_ID: ${{ inputs.ci-run-id }}
|
|
@@ -38,6 +38,10 @@ set -euo pipefail
|
|
|
38
38
|
# CI_STATUS none, or the latest CI run's status (queued, in_progress, completed, ...)
|
|
39
39
|
# CI_CONCLUSION that run's conclusion when it completed
|
|
40
40
|
# CI_STUCK true when that run has been pending longer than the train waits
|
|
41
|
+
# SAME_AS_MAIN true when every check that failed on the head also fails on the base branch
|
|
42
|
+
# tip: the failure is the base's, and repairing the pull request spends its
|
|
43
|
+
# attempts on code that is not at fault (Pliny-Bot, 2026-10-17: a red main
|
|
44
|
+
# cost every open pull request its repairs for seven hours)
|
|
41
45
|
# BEHIND_BY commits on the base branch the head does not have
|
|
42
46
|
# TRUSTED_AUTHOR true when a trusted bot opened it (only those runs are approved)
|
|
43
47
|
train_move() {
|
|
@@ -70,7 +74,13 @@ train_move() {
|
|
|
70
74
|
move=gate; reason="green and up to date"
|
|
71
75
|
fi
|
|
72
76
|
;;
|
|
73
|
-
failure)
|
|
77
|
+
failure)
|
|
78
|
+
if [ "${SAME_AS_MAIN:-false}" = true ]; then
|
|
79
|
+
move="wait"; reason="CI failed, but only on checks that fail on the base branch too; waiting for the base to be fixed"
|
|
80
|
+
else
|
|
81
|
+
move=gate; reason="CI failed"
|
|
82
|
+
fi
|
|
83
|
+
;;
|
|
74
84
|
action_required)
|
|
75
85
|
if [ "${TRUSTED_AUTHOR:-false}" = true ]; then
|
|
76
86
|
move=approve; reason="CI is waiting for approval"
|
|
@@ -136,6 +146,7 @@ MAX_ATTEMPTS="${MAX_ATTEMPTS:-6}"
|
|
|
136
146
|
MAX_RETRIES="${MAX_RETRIES:-3}"
|
|
137
147
|
MAX_REPAIRS="${MAX_REPAIRS:-1}"
|
|
138
148
|
case "$MAX_REPAIRS" in ''|*[!0-9]*|0) MAX_REPAIRS=1 ;; esac
|
|
149
|
+
MAIN_RED_WAIT_HOURS="${MAIN_RED_WAIT_HOURS:-2}"
|
|
139
150
|
STUCK_HOURS="${STUCK_HOURS:-3}"
|
|
140
151
|
DRY_RUN="${DRY_RUN:-false}"
|
|
141
152
|
SELF_RUN_ID="${GITHUB_RUN_ID:-0}"
|
|
@@ -403,16 +414,59 @@ for line in "${candidates[@]}"; do
|
|
|
403
414
|
fi
|
|
404
415
|
fi
|
|
405
416
|
|
|
417
|
+
# What failed on the head, compared with what is failing on the base branch tip. Names are
|
|
418
|
+
# compared after the last " / ": a check that a reusable workflow runs is "CI / backend" on the
|
|
419
|
+
# base branch and "backend" on a pull request. Only when every failed check is also failing on
|
|
420
|
+
# the base is the pull request left alone.
|
|
421
|
+
same_as_main=false
|
|
422
|
+
if [ "$ci_status" = completed ] && [ "$ci_conclusion" = failure ] && [ "$park" = none ]; then
|
|
423
|
+
failed_here=$(gh api "repos/$REPO/commits/$head_sha/check-runs?per_page=100" \
|
|
424
|
+
--jq '[.check_runs[] | select(.conclusion == "failure") | (.name | sub("^.* / "; ""))] | unique | .[]' 2>/dev/null || true)
|
|
425
|
+
failed_base=$(gh api "repos/$REPO/commits/$base_ref/check-runs?per_page=100" \
|
|
426
|
+
--jq '[.check_runs[] | select(.conclusion == "failure") | (.name | sub("^.* / "; ""))] | unique | .[]' 2>/dev/null || true)
|
|
427
|
+
# How long the base has been red: the oldest failing check on its tip. The wait is bounded,
|
|
428
|
+
# because waiting for a base nobody fixes starves every pull request behind it. Pliny-Bot's
|
|
429
|
+
# main was red from 19:30 and the train waited until morning, ten hours, saying nothing.
|
|
430
|
+
red_since=$(gh api "repos/$REPO/commits/$base_ref/check-runs?per_page=100" \
|
|
431
|
+
--jq '[.check_runs[] | select(.conclusion == "failure") | .completed_at] | min // empty' 2>/dev/null || true)
|
|
432
|
+
red_for_hours=0
|
|
433
|
+
if [ -n "$red_since" ]; then
|
|
434
|
+
red_for_hours=$(( (now_epoch - $(iso_epoch "$red_since")) / 3600 ))
|
|
435
|
+
fi
|
|
436
|
+
if [ -n "$failed_here" ] && [ -n "$failed_base" ] &&
|
|
437
|
+
[ -z "$(comm -23 <(sort <<<"$failed_here") <(sort <<<"$failed_base"))" ]; then
|
|
438
|
+
if [ "$red_for_hours" -lt "${MAIN_RED_WAIT_HOURS:-2}" ]; then
|
|
439
|
+
same_as_main=true
|
|
440
|
+
else
|
|
441
|
+
echo "::warning::$base_ref has been red for ${red_for_hours}h on the same checks as #$pr; the train no longer waits for it and repairs #$pr."
|
|
442
|
+
fi
|
|
443
|
+
fi
|
|
444
|
+
fi
|
|
445
|
+
|
|
406
446
|
trusted_author=false
|
|
407
447
|
is_trusted_bot "$author" && trusted_author=true
|
|
408
448
|
|
|
409
449
|
decision=$(HOLD="$hold" REVIEW_DECISION="$review_decision" PARK="$park" \
|
|
410
450
|
ATTEMPTS="$attempts" MAX_ATTEMPTS="$MAX_ATTEMPTS" SILENT_GATE="$silent_gate" MERGEABLE="$mergeable" \
|
|
411
|
-
CI_STATUS="$ci_status" CI_CONCLUSION="$ci_conclusion" CI_STUCK="$ci_stuck" \
|
|
451
|
+
CI_STATUS="$ci_status" CI_CONCLUSION="$ci_conclusion" CI_STUCK="$ci_stuck" SAME_AS_MAIN="$same_as_main" \
|
|
412
452
|
BEHIND_BY="${behind_by:-0}" TRUSTED_AUTHOR="$trusted_author" train_move)
|
|
413
453
|
move=$(sed -n 's/^move=//p' <<<"$decision")
|
|
414
454
|
reason=$(sed -n 's/^reason=//p' <<<"$decision")
|
|
415
455
|
|
|
456
|
+
# A repair gate dispatched for red CI is moot once CI on the head is green: its job was to make
|
|
457
|
+
# that run pass. Left alone it keeps the pull request marked as having a gate, so the train
|
|
458
|
+
# skipped four green Pliny-Bot pull requests with "a gate is already on it" until it was cancelled
|
|
459
|
+
# by hand. Cancel it and let the pull request move.
|
|
460
|
+
if { [ "$move" = gate ] || [ "$move" = update ]; } && [ "$ci_status" = completed ] && [ "$ci_conclusion" = success ] &&
|
|
461
|
+
printf '%s\n' "$live_titles" | grep -qE "PR #${pr} \(repair\)$"; then
|
|
462
|
+
for run_id in $(gh api "repos/$REPO/actions/workflows/work-router.yml/runs?per_page=100" \
|
|
463
|
+
--jq ".workflow_runs[] | select(.status != \"completed\" and (.display_title | endswith(\"PR #${pr} (repair)\"))) | .id" 2>/dev/null); do
|
|
464
|
+
act "Cancelling the repair gate $run_id on #$pr: its CI is green." \
|
|
465
|
+
gh api -X POST "repos/$REPO/actions/runs/$run_id/cancel" --silent
|
|
466
|
+
done
|
|
467
|
+
live_titles=$(printf '%s\n' "$live_titles" | grep -vE "PR #${pr} \(repair\)$" || true)
|
|
468
|
+
fi
|
|
469
|
+
|
|
416
470
|
# A gate is already on this pull request, whatever kind: offering it another would run the same
|
|
417
471
|
# work twice. It does not hold the line up for the repair kind; a merging gate already does.
|
|
418
472
|
if [ "$move" != skip ] && live_for_pr "$pr"; then
|
|
@@ -23,6 +23,20 @@ inputs:
|
|
|
23
23
|
description: How far back to look for failures.
|
|
24
24
|
required: false
|
|
25
25
|
default: '24'
|
|
26
|
+
min-occurrences:
|
|
27
|
+
description: >-
|
|
28
|
+
A signature seen fewer times than this inside the window does not open a new issue. One
|
|
29
|
+
occurrence is a blip; an issue that is already open is always appended to.
|
|
30
|
+
required: false
|
|
31
|
+
default: '2'
|
|
32
|
+
max-findings:
|
|
33
|
+
description: The most issues one run opens, the most frequent first. The rest stay in the job summary.
|
|
34
|
+
required: false
|
|
35
|
+
default: '5'
|
|
36
|
+
quiet-days:
|
|
37
|
+
description: A signature whose issue was closed within this many days stays quiet, whatever the reason it was closed.
|
|
38
|
+
required: false
|
|
39
|
+
default: '30'
|
|
26
40
|
dry-run:
|
|
27
41
|
description: Compute and print the report, file nothing.
|
|
28
42
|
required: false
|
|
@@ -43,6 +57,10 @@ runs:
|
|
|
43
57
|
UPSTREAM_TOKEN: ${{ inputs.upstream-token }}
|
|
44
58
|
UPSTREAM_REPO: ${{ inputs.upstream-repo }}
|
|
45
59
|
LOOKBACK_HOURS: ${{ inputs.lookback-hours }}
|
|
60
|
+
MIN_OCCURRENCES: ${{ inputs.min-occurrences }}
|
|
61
|
+
MAX_FINDINGS: ${{ inputs.max-findings }}
|
|
62
|
+
QUIET_DAYS: ${{ inputs.quiet-days }}
|
|
63
|
+
POLICY_PATH: ${{ github.action_path }}/report-policy.cjs
|
|
46
64
|
DRY_RUN: ${{ inputs.dry-run }}
|
|
47
65
|
with:
|
|
48
66
|
github-token: ${{ inputs.token }}
|
|
@@ -56,6 +74,10 @@ runs:
|
|
|
56
74
|
// is checked against a leak scanner and the job fails rather than files if the
|
|
57
75
|
// scanner trips.
|
|
58
76
|
|
|
77
|
+
// What to file and what to leave as a number lives in report-policy.cjs, apart from
|
|
78
|
+
// what this script reads and writes, so it can be run on its own.
|
|
79
|
+
const policy = require(process.env.POLICY_PATH);
|
|
80
|
+
|
|
59
81
|
const dryRun = process.env.DRY_RUN === 'true';
|
|
60
82
|
const upstreamToken = process.env.UPSTREAM_TOKEN;
|
|
61
83
|
const [upstreamOwner, upstreamName] = process.env.UPSTREAM_REPO.split('/');
|
|
@@ -140,7 +162,7 @@ runs:
|
|
|
140
162
|
const header = require('node:fs')
|
|
141
163
|
.readFileSync('.github/workflows/work-router.yml', 'utf8')
|
|
142
164
|
.split('\n')[0];
|
|
143
|
-
packageVersion =
|
|
165
|
+
packageVersion = policy.parseVersion(header);
|
|
144
166
|
} catch {
|
|
145
167
|
core.info('could not read the installed package version from the router header');
|
|
146
168
|
}
|
|
@@ -153,15 +175,19 @@ runs:
|
|
|
153
175
|
// upstream, and every part of it is package-owned or an enumerated value.
|
|
154
176
|
//
|
|
155
177
|
// THE FINDING SHAPE IS THE PRIVACY BOUNDARY. bodyFor() renders a finding, so a field
|
|
156
|
-
// added here is a field sent to another repository. These
|
|
178
|
+
// added here is a field sent to another repository. These ten are all
|
|
157
179
|
// package-owned names or enumerated values; nothing derived from a log, a branch, a
|
|
158
180
|
// path or an issue belongs on this object. The route matrix asserts this exact list,
|
|
159
181
|
// so adding a ninth field fails the build rather than leaking on the next cron.
|
|
160
|
-
const FINDING_FIELDS = ['workflow', 'job', 'step', 'conclusion', 'patternId', 'count', 'runnerLabels', 'attempts'];
|
|
182
|
+
const FINDING_FIELDS = ['workflow', 'job', 'step', 'conclusion', 'patternId', 'count', 'runnerLabels', 'attempts', 'severity', 'variants'];
|
|
161
183
|
const findings = new Map();
|
|
162
184
|
let ownedRuns = 0;
|
|
163
185
|
let ownedFailures = 0;
|
|
164
186
|
let skippedForeign = 0;
|
|
187
|
+
// Things that happened and are worth a number in the job summary, but that nobody can
|
|
188
|
+
// fix by editing a workflow, so they never open an issue: a cancelled job with no step
|
|
189
|
+
// that failed, a log that could not be read, and the fleet's own transient causes.
|
|
190
|
+
const quiet = { cancelledNoStep: 0, unreadable: 0, infra: new Map() };
|
|
165
191
|
let queueWaitTotal = 0;
|
|
166
192
|
let queueWaitCount = 0;
|
|
167
193
|
let queueWaitWorst = 0;
|
|
@@ -180,15 +206,7 @@ runs:
|
|
|
180
206
|
if (['queued', 'waiting', 'pending'].includes(run.status) &&
|
|
181
207
|
Date.now() - new Date(run.created_at).getTime() > STUCK_AFTER_MS) {
|
|
182
208
|
stuckInQueue += 1;
|
|
183
|
-
|
|
184
|
-
const seen = findings.get(signature) ?? {
|
|
185
|
-
workflow: file, job: '(whole run)', step: '(never started)',
|
|
186
|
-
conclusion: 'queued', patternId: 'runner-never-assigned', count: 0,
|
|
187
|
-
runnerLabels: new Set(), attempts: new Set(),
|
|
188
|
-
};
|
|
189
|
-
seen.count += 1;
|
|
190
|
-
seen.attempts.add(run.run_attempt ?? 1);
|
|
191
|
-
findings.set(signature, seen);
|
|
209
|
+
quiet.infra.set('runner-never-assigned', (quiet.infra.get('runner-never-assigned') ?? 0) + 1);
|
|
192
210
|
continue;
|
|
193
211
|
}
|
|
194
212
|
|
|
@@ -217,7 +235,24 @@ runs:
|
|
|
217
235
|
// package's own words. A `run-name` is not: it interpolates issue titles.
|
|
218
236
|
const stepName = step?.name ?? '(no failing step recorded)';
|
|
219
237
|
|
|
238
|
+
// A cancelled job that never reached a failing step is somebody or something
|
|
239
|
+
// stopping work, not a defect: a superseded run, a job cancelled behind a failed
|
|
240
|
+
// one. It was 189 of 320 reports and every one said nothing.
|
|
241
|
+
if (job.conclusion === 'cancelled' && !step) { quiet.cancelledNoStep += 1; continue; }
|
|
242
|
+
|
|
243
|
+
// GitHub writes why a runner went away into the job's annotations ("The runner has
|
|
244
|
+
// received a shutdown signal", exit code 143), not into the log, so they are read
|
|
245
|
+
// with it. Only the id of a catalogue entry that matches ever leaves.
|
|
246
|
+
let annotationText = '';
|
|
247
|
+
try {
|
|
248
|
+
const notes = await github.paginate(github.rest.checks.listAnnotations, {
|
|
249
|
+
...context.repo, check_run_id: job.id, per_page: 50,
|
|
250
|
+
});
|
|
251
|
+
annotationText = notes.map((note) => note.message ?? '').join('\n');
|
|
252
|
+
} catch { /* annotations are an extra; the log can still classify */ }
|
|
253
|
+
|
|
220
254
|
let patternId = 'unrecognised';
|
|
255
|
+
let logRead = true;
|
|
221
256
|
try {
|
|
222
257
|
const { data: log } = await github.rest.actions.downloadJobLogsForWorkflowRun({
|
|
223
258
|
...context.repo, job_id: job.id,
|
|
@@ -225,19 +260,32 @@ runs:
|
|
|
225
260
|
const text = typeof log === 'string' ? log : Buffer.from(log).toString('utf8');
|
|
226
261
|
// The tail is where the failure is, and reading less of the log is less to get
|
|
227
262
|
// wrong. Only the matched entry's id crosses into the report.
|
|
228
|
-
const tail = text.slice(-20000)
|
|
263
|
+
const tail = `${annotationText}\n${text.slice(-20000)}`;
|
|
229
264
|
patternId = CATALOGUE.find((entry) => entry.test.test(tail))?.id ?? 'unrecognised';
|
|
230
265
|
} catch {
|
|
231
|
-
|
|
266
|
+
logRead = false;
|
|
267
|
+
patternId = CATALOGUE.find((entry) => entry.test.test(annotationText))?.id ?? 'log-unavailable';
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
// A log nobody could read says nothing, so it is counted and not filed.
|
|
271
|
+
if (patternId === 'log-unavailable') { quiet.unreadable += 1; continue; }
|
|
272
|
+
if (!logRead && policy.severityOf(patternId) === 'infra') {
|
|
273
|
+
quiet.infra.set(patternId, (quiet.infra.get(patternId) ?? 0) + 1);
|
|
274
|
+
continue;
|
|
232
275
|
}
|
|
233
276
|
|
|
234
|
-
|
|
277
|
+
// One cause is one finding, whichever job tripped on it.
|
|
278
|
+
const signature = policy.signatureOf({
|
|
279
|
+
workflow: file, step: stepName, conclusion: job.conclusion, patternId,
|
|
280
|
+
});
|
|
235
281
|
const seen = findings.get(signature) ?? {
|
|
236
282
|
workflow: file, job: job.name, step: stepName,
|
|
237
|
-
conclusion: job.conclusion, patternId, count: 0,
|
|
283
|
+
conclusion: job.conclusion, patternId: patternId, count: 0,
|
|
238
284
|
runnerLabels: new Set(), attempts: new Set(),
|
|
285
|
+
severity: policy.severityOf(patternId), variants: new Set(),
|
|
239
286
|
};
|
|
240
287
|
seen.count += 1;
|
|
288
|
+
seen.variants.add(`${job.name}|${stepName}`);
|
|
241
289
|
seen.attempts.add(run.run_attempt ?? 1);
|
|
242
290
|
// A pool label is the one value on a finding that the CONSUMER writes, and the
|
|
243
291
|
// installer deliberately preserves theirs across updates (preserveRunnerPool in
|
|
@@ -261,6 +309,13 @@ runs:
|
|
|
261
309
|
const ranked = [...findings.values()].sort((a, b) => b.count - a.count);
|
|
262
310
|
const hours = process.env.LOOKBACK_HOURS;
|
|
263
311
|
|
|
312
|
+
// What becomes an issue. Infrastructure causes are a rate, a blip is not worth a
|
|
313
|
+
// person's time, and a bad night opens a handful of issues rather than forty.
|
|
314
|
+
const selection = policy.select(ranked, {
|
|
315
|
+
minOccurrences: Number(process.env.MIN_OCCURRENCES),
|
|
316
|
+
maxFindings: Number(process.env.MAX_FINDINGS),
|
|
317
|
+
});
|
|
318
|
+
|
|
264
319
|
// Enforce the boundary at run time as well as in the build. A field that reached a
|
|
265
320
|
// finding by some path the static check did not see stops the job here rather than
|
|
266
321
|
// being rendered into an issue in another repository.
|
|
@@ -273,7 +328,7 @@ runs:
|
|
|
273
328
|
}
|
|
274
329
|
|
|
275
330
|
const bodyFor = (finding) => [
|
|
276
|
-
`<!-- pcp-error-report: ${
|
|
331
|
+
`<!-- pcp-error-report: ${policy.markerKey(finding)} -->`,
|
|
277
332
|
'',
|
|
278
333
|
'Filed automatically by the `report-workflow-errors` job in a repository that installs',
|
|
279
334
|
'this package. It carries no information about that repository: only the names this',
|
|
@@ -286,6 +341,8 @@ runs:
|
|
|
286
341
|
`| Failing step | \`${finding.step}\` |`,
|
|
287
342
|
`| Conclusion | \`${finding.conclusion}\` |`,
|
|
288
343
|
`| Classification | \`${finding.patternId}\` |`,
|
|
344
|
+
`| Severity | \`${finding.severity}\` |`,
|
|
345
|
+
`| Job and step combinations with this cause | ${finding.variants.size} |`,
|
|
289
346
|
`| Runner labels | ${[...finding.runnerLabels].map((l) => `\`${l}\``).join(', ') || 'not reported'} |`,
|
|
290
347
|
`| Occurrences in the last ${hours}h | ${finding.count} |`,
|
|
291
348
|
`| Re-run attempts involved | ${[...finding.attempts].sort().join(', ')} |`,
|
|
@@ -298,8 +355,9 @@ runs:
|
|
|
298
355
|
`- ${new Date().toISOString().slice(0, 10)}: ${finding.count} in ${hours}h, on \`${packageVersion}\``,
|
|
299
356
|
].join('\n');
|
|
300
357
|
|
|
301
|
-
const titleFor = (finding) =>
|
|
302
|
-
|
|
358
|
+
const titleFor = (finding) => (finding.patternId === 'unrecognised'
|
|
359
|
+
? `[${finding.severity}] ${finding.workflow}: ${finding.patternId} in "${finding.step}"`
|
|
360
|
+
: `[${finding.severity}] ${finding.workflow}: ${finding.patternId}`);
|
|
303
361
|
|
|
304
362
|
// The scanner. A body that trips it is never filed, and the job goes red so the
|
|
305
363
|
// pattern that produced it gets fixed rather than quietly dropped every night.
|
|
@@ -325,6 +383,10 @@ runs:
|
|
|
325
383
|
'This report does not cover itself: a failure in this job shows up in this repository only.',
|
|
326
384
|
`Runs still queued after 45 minutes, so no runner ever took them: ${stuckInQueue}`,
|
|
327
385
|
`Distinct failure signatures: ${ranked.length}`,
|
|
386
|
+
`Filed or appended: ${selection.file.length}; held back: ${selection.held.length}`,
|
|
387
|
+
`Cancelled jobs with no failing step, not reported: ${quiet.cancelledNoStep}`,
|
|
388
|
+
`Jobs whose log could not be read, not reported: ${quiet.unreadable}`,
|
|
389
|
+
`Infrastructure causes, counted and not reported: ${[...quiet.infra].map(([id, n]) => `${id} ${n}`).join(', ') || 'none'}`,
|
|
328
390
|
queueWaitCount > 0
|
|
329
391
|
? `Runner queue wait: ${Math.round(queueWaitTotal / queueWaitCount)}s mean, ${queueWaitWorst}s worst`
|
|
330
392
|
: 'Runner queue wait: no jobs started in the window',
|
|
@@ -345,7 +407,7 @@ runs:
|
|
|
345
407
|
// full picture rather than one night at a time.
|
|
346
408
|
const reports = [];
|
|
347
409
|
let leaked = 0;
|
|
348
|
-
for (const finding of
|
|
410
|
+
for (const finding of selection.file) {
|
|
349
411
|
const title = titleFor(finding);
|
|
350
412
|
const body = bodyFor(finding);
|
|
351
413
|
if (!scan(`${title}\n${body}`, `The report for ${finding.patternId}`)) { leaked += 1; continue; }
|
|
@@ -373,10 +435,17 @@ runs:
|
|
|
373
435
|
let existing;
|
|
374
436
|
try {
|
|
375
437
|
const { data } = await upstream.rest.search.issuesAndPullRequests({
|
|
376
|
-
q: `repo:${upstreamOwner}/${upstreamName} is:issue
|
|
377
|
-
per_page:
|
|
438
|
+
q: `repo:${upstreamOwner}/${upstreamName} is:issue in:body "${marker}"`,
|
|
439
|
+
per_page: 10,
|
|
378
440
|
});
|
|
379
|
-
|
|
441
|
+
const matches = data.items.filter((item) => (item.body ?? '').includes(marker));
|
|
442
|
+
existing = matches.find((item) => item.state === 'open');
|
|
443
|
+
// Closing an issue is the answer to it. Without this every bulk close reopened the
|
|
444
|
+
// whole set the next morning: 150 of 320 reports re-filed a signature seen before.
|
|
445
|
+
if (!existing && matches.some((item) => policy.isQuiet(item.closed_at, Date.now(), Number(process.env.QUIET_DAYS)))) {
|
|
446
|
+
core.info(`left alone, an issue for it was closed within ${process.env.QUIET_DAYS} days: ${title}`);
|
|
447
|
+
continue;
|
|
448
|
+
}
|
|
380
449
|
} catch (error) {
|
|
381
450
|
core.warning(`could not search upstream for an existing report: HTTP ${error.status ?? '?'}`);
|
|
382
451
|
}
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
// Managed by @plainconceptsplatform/workflows. Source: loops/actions/report-workflow-errors/report-policy.cjs. Update with workflows update --force; consumer edits may be overwritten.
|
|
2
|
+
//
|
|
3
|
+
// What the error report decides, kept apart from what it reads and files so it can be run on its
|
|
4
|
+
// own. Pure: no network, no clock, nothing read from the repository. Everything it sees is a
|
|
5
|
+
// finding the report already built from package-owned names and enumerated values, so nothing
|
|
6
|
+
// here can widen what leaves a private repository.
|
|
7
|
+
//
|
|
8
|
+
// The report used to open one issue per job, step and classification, for every failure it saw.
|
|
9
|
+
// Read back over a month, 320 issues: 59% were cancelled jobs, 80% carried no diagnosis, 14% were
|
|
10
|
+
// a runner shut down under a job, 47% re-filed a signature seen before, every one was closed as
|
|
11
|
+
// not planned and none had a comment. What follows is the filter that would have kept about a
|
|
12
|
+
// hundred of them out of that pile and left the rest as a count in the job summary.
|
|
13
|
+
|
|
14
|
+
'use strict';
|
|
15
|
+
|
|
16
|
+
// Causes that say something about the fleet, not about this package. They are counted in the job
|
|
17
|
+
// summary, where a trend is visible, and never become an issue: nobody can fix a runner that was
|
|
18
|
+
// taken away mid-job by editing a workflow. A job that hit its timeout is NOT here, because that
|
|
19
|
+
// can be the package's own work taking too long.
|
|
20
|
+
const INFRA = new Set([
|
|
21
|
+
'runner-never-assigned',
|
|
22
|
+
'runner-lost-mid-job',
|
|
23
|
+
'gh-api-rate-limit',
|
|
24
|
+
'model-quota',
|
|
25
|
+
'model-gateway-unreachable',
|
|
26
|
+
'out-of-disk',
|
|
27
|
+
'out-of-memory',
|
|
28
|
+
]);
|
|
29
|
+
|
|
30
|
+
// Causes that are a defect in something this package ships.
|
|
31
|
+
const PACKAGE_BUG = new Set([
|
|
32
|
+
'jq-error',
|
|
33
|
+
'yaml-or-expression-invalid',
|
|
34
|
+
'lock-out-of-date',
|
|
35
|
+
'action-not-found',
|
|
36
|
+
'agent-produced-no-output',
|
|
37
|
+
'gh-api-403',
|
|
38
|
+
'gh-api-404',
|
|
39
|
+
'gh-api-409-conflict',
|
|
40
|
+
]);
|
|
41
|
+
|
|
42
|
+
/** infra, package-bug, or unknown (a cause nobody classified yet, worth a look when it repeats). */
|
|
43
|
+
function severityOf(patternId) {
|
|
44
|
+
if (INFRA.has(patternId)) return 'infra';
|
|
45
|
+
if (PACKAGE_BUG.has(patternId)) return 'package-bug';
|
|
46
|
+
return 'unknown';
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* One cause is one issue. A specific cause is the same defect whichever job of the router tripped
|
|
51
|
+
* on it, so the job and the step are not part of its identity: conclude, safe_outputs and reserve
|
|
52
|
+
* failing on one 403 are one finding, not three. A cause nobody could name is told apart by the
|
|
53
|
+
* step it failed in, because that is all there is to go on.
|
|
54
|
+
*/
|
|
55
|
+
function signatureOf({ workflow, step, conclusion, patternId }) {
|
|
56
|
+
if (patternId === 'unrecognised') return [workflow, step, conclusion, patternId].join('|');
|
|
57
|
+
return [workflow, conclusion, patternId].join('|');
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** The text the upstream search matches on, so a repeat lands on the issue that already exists. */
|
|
61
|
+
function markerKey({ workflow, step, conclusion, patternId }) {
|
|
62
|
+
return signatureOf({ workflow, step, conclusion, patternId });
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Which findings become issues, and why the others do not.
|
|
67
|
+
* - infra causes are held: they are a rate, not a bug.
|
|
68
|
+
* - a finding seen fewer than `minOccurrences` times in the window is held: once is a blip.
|
|
69
|
+
* - at most `maxFindings` are filed per run, the most frequent first, so a bad night opens a
|
|
70
|
+
* handful of issues and not forty.
|
|
71
|
+
*/
|
|
72
|
+
function select(findings, { minOccurrences = 2, maxFindings = 5 } = {}) {
|
|
73
|
+
const ranked = [...findings].sort((a, b) => b.count - a.count);
|
|
74
|
+
const file = [];
|
|
75
|
+
const held = [];
|
|
76
|
+
for (const finding of ranked) {
|
|
77
|
+
if (finding.severity === 'infra') {
|
|
78
|
+
held.push({ finding, reason: 'infrastructure cause, counted not filed' });
|
|
79
|
+
} else if (finding.count < minOccurrences) {
|
|
80
|
+
held.push({ finding, reason: `seen ${finding.count} time(s), under the ${minOccurrences} needed to file` });
|
|
81
|
+
} else if (file.length >= maxFindings) {
|
|
82
|
+
held.push({ finding, reason: `over the ${maxFindings} filed per run` });
|
|
83
|
+
} else {
|
|
84
|
+
file.push(finding);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
return { file, held };
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* A signature closed recently stays quiet. Closing an issue is the answer to it; the report used
|
|
92
|
+
* to search open issues only, so every bulk close reopened the whole set the next morning.
|
|
93
|
+
*/
|
|
94
|
+
function isQuiet(closedAt, now, quietDays = 30) {
|
|
95
|
+
if (!closedAt) return false;
|
|
96
|
+
const closed = new Date(closedAt).getTime();
|
|
97
|
+
if (Number.isNaN(closed)) return false;
|
|
98
|
+
return now - closed < quietDays * 24 * 3600 * 1000;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* The package version out of the ownership header. The character class used to swallow the
|
|
103
|
+
* full stop that ends the sentence, so every report said `1.2.3.`.
|
|
104
|
+
*/
|
|
105
|
+
function parseVersion(header) {
|
|
106
|
+
const match = /@plainconceptsplatform\/workflows@([0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.]*[0-9A-Za-z])?)/.exec(header ?? '');
|
|
107
|
+
return match ? match[1] : 'unknown';
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
module.exports = { INFRA, PACKAGE_BUG, severityOf, signatureOf, markerKey, select, isQuiet, parseVersion };
|
|
@@ -1788,6 +1788,24 @@ fi
|
|
|
1788
1788
|
# leaving a verdict or an attempt is a loop: the train cannot tell a gate that declined from a gate
|
|
1789
1789
|
# that has not run. Refusals are for what the train never hands over (a fork, a stranger, a pull
|
|
1790
1790
|
# request that is not open), and the train filters those itself.
|
|
1791
|
+
# The shape of the tests reaches the agent only through these two prompts (check-diff measures size,
|
|
1792
|
+
# never shape), so losing the sentence silently brings the pyramid back.
|
|
1793
|
+
if worker_installed implement; then
|
|
1794
|
+
if grep -qF 'Shape what you test like a trophy' "$IMPLEMENT_WORKER_MD"; then
|
|
1795
|
+
PASS=$((PASS + 1))
|
|
1796
|
+
else
|
|
1797
|
+
FAIL=$((FAIL + 1))
|
|
1798
|
+
echo "FAIL: the implement prompt must state the testing trophy shape" >&2
|
|
1799
|
+
fi
|
|
1800
|
+
fi
|
|
1801
|
+
if worker_installed apply-review; then
|
|
1802
|
+
if grep -qF 'put it at the integration layer' "${WORKFLOWS_DIR}/agent-apply-review.md"; then
|
|
1803
|
+
PASS=$((PASS + 1))
|
|
1804
|
+
else
|
|
1805
|
+
FAIL=$((FAIL + 1))
|
|
1806
|
+
echo "FAIL: the apply-review prompt must say new tests go at the integration layer" >&2
|
|
1807
|
+
fi
|
|
1808
|
+
fi
|
|
1791
1809
|
if worker_installed merge-gate; then
|
|
1792
1810
|
# check-diff exiting 1 is the `fail` the code_rules job reports. Under GitHub's `bash -e` a bare
|
|
1793
1811
|
# assignment from it ends the step, so the report is never recorded and the gate burns its
|
|
@@ -1979,6 +1997,16 @@ if worker_installed merge-gate; then
|
|
|
1979
1997
|
assert "running CI is waited on" wait "$(move MERGEABLE=MERGEABLE CI_STATUS=in_progress)"
|
|
1980
1998
|
assert "CI stuck past the limit is passed by" skip "$(move MERGEABLE=MERGEABLE CI_STATUS=queued CI_STUCK=true)"
|
|
1981
1999
|
assert "red CI goes to the gate" gate "$(move MERGEABLE=MERGEABLE CI_STATUS=completed CI_CONCLUSION=failure)"
|
|
2000
|
+
# A red base branch cost every open Pliny-Bot pull request its repairs for seven hours: the
|
|
2001
|
+
# failures were main's, and the gate spent attempts on code that was not at fault.
|
|
2002
|
+
assert "a failure that is main's too is waited out, not repaired" wait "$(move MERGEABLE=MERGEABLE CI_STATUS=completed CI_CONCLUSION=failure SAME_AS_MAIN=true)"
|
|
2003
|
+
assert "a failure that is not main's still goes to the gate" gate "$(move MERGEABLE=MERGEABLE CI_STATUS=completed CI_CONCLUSION=failure SAME_AS_MAIN=false)"
|
|
2004
|
+
assert "a conflict is still the gate's even when main is red" gate "$(move MERGEABLE=CONFLICTING CI_STATUS=completed CI_CONCLUSION=failure SAME_AS_MAIN=true)"
|
|
2005
|
+
# The wait for a red base is bounded. Pliny-Bot's main was red from 19:30 and the train waited
|
|
2006
|
+
# on it until morning, saying nothing, while five pull requests sat with their repairs unspent.
|
|
2007
|
+
grep -q 'MAIN_RED_WAIT_HOURS' "$TRAIN_SH" && grep -q 'red_for_hours' "$TRAIN_SH" &&
|
|
2008
|
+
grep -q 'main-red-wait-hours' "${HERE}/../merge-train/action.yml" ||
|
|
2009
|
+
{ FAIL=$((FAIL + 1)); echo "FAIL: the merge train waits on a red base branch with no time limit" >&2; }
|
|
1982
2010
|
assert "green but behind is brought up to date first" update "$(move MERGEABLE=MERGEABLE CI_STATUS=completed CI_CONCLUSION=success BEHIND_BY=2)"
|
|
1983
2011
|
assert "green and current goes to the gate" gate "$(move MERGEABLE=MERGEABLE CI_STATUS=completed CI_CONCLUSION=success BEHIND_BY=0)"
|
|
1984
2012
|
assert "no CI run is passed by" skip "$(move MERGEABLE=MERGEABLE CI_STATUS=none)"
|
|
@@ -2248,6 +2276,55 @@ if [ -f "$ERROR_REPORT_YML" ]; then
|
|
|
2248
2276
|
# above cannot see stops the job instead of being rendered upstream.
|
|
2249
2277
|
er 'not in the reportable field list' 'does not check the finding shape at run time'
|
|
2250
2278
|
|
|
2279
|
+
# 3b. What the report files and what it leaves as a number. Read back over a month, 320 issues:
|
|
2280
|
+
# 59% were cancelled jobs, 80% carried no diagnosis, 47% re-filed a signature seen before, and
|
|
2281
|
+
# every one was closed as not planned. The decision lives in report-policy.cjs so it can be run
|
|
2282
|
+
# here, and the action has to actually use it.
|
|
2283
|
+
POLICY_CJS="${HERE}/../report-workflow-errors/report-policy.cjs"
|
|
2284
|
+
if [ -f "$POLICY_CJS" ]; then
|
|
2285
|
+
policy_js='
|
|
2286
|
+
const p = require(process.argv[1]);
|
|
2287
|
+
const bad = [];
|
|
2288
|
+
const ok = (cond, what) => { if (!cond) bad.push(what); };
|
|
2289
|
+
ok(p.severityOf("runner-lost-mid-job") === "infra", "a runner lost mid-job is infrastructure");
|
|
2290
|
+
ok(p.severityOf("jq-error") === "package-bug", "a jq error is a package bug");
|
|
2291
|
+
ok(p.severityOf("unrecognised") === "unknown", "an unrecognised cause is unknown");
|
|
2292
|
+
const a = { workflow: "w.yml", step: "one", conclusion: "failure", patternId: "gh-api-403" };
|
|
2293
|
+
const b = { ...a, step: "two" };
|
|
2294
|
+
ok(p.signatureOf(a) === p.signatureOf(b), "one specific cause is one signature whatever the step");
|
|
2295
|
+
const u = { ...a, patternId: "unrecognised" };
|
|
2296
|
+
ok(p.signatureOf(u) !== p.signatureOf({ ...u, step: "two" }), "an unrecognised cause is told apart by its step");
|
|
2297
|
+
const f = (patternId, count, severity) => ({ patternId, count, severity });
|
|
2298
|
+
const r = p.select([f("a", 5, "package-bug"), f("b", 9, "infra"), f("c", 1, "unknown"), f("d", 3, "unknown")], { minOccurrences: 2, maxFindings: 5 });
|
|
2299
|
+
ok(r.file.map((x) => x.patternId).join() === "a,d", "infrastructure and one-off findings are held, the rest filed most frequent first");
|
|
2300
|
+
ok(r.held.length === 2, "two findings held");
|
|
2301
|
+
const many = Array.from({ length: 9 }, (_, i) => f("p" + i, 10 - i, "unknown"));
|
|
2302
|
+
ok(p.select(many, { minOccurrences: 2, maxFindings: 5 }).file.length === 5, "a bad night files at most five");
|
|
2303
|
+
const now = Date.parse("2026-10-17T00:00:00Z");
|
|
2304
|
+
ok(p.isQuiet("2026-10-10T00:00:00Z", now, 30) === true, "an issue closed a week ago keeps its signature quiet");
|
|
2305
|
+
ok(p.isQuiet("2026-08-01T00:00:00Z", now, 30) === false, "an issue closed long ago does not");
|
|
2306
|
+
ok(p.isQuiet(null, now, 30) === false, "no closing date is not quiet");
|
|
2307
|
+
ok(p.parseVersion("# Managed by @plainconceptsplatform/workflows@1.2.3. Source: x") === "1.2.3", "the version has no trailing full stop");
|
|
2308
|
+
ok(p.parseVersion("nothing here") === "unknown", "no version reads as unknown");
|
|
2309
|
+
if (bad.length) { console.error(bad.join("\n")); process.exit(1); }
|
|
2310
|
+
'
|
|
2311
|
+
if node -e "$policy_js" "$POLICY_CJS" 2>/tmp/policy-err; then
|
|
2312
|
+
PASS=$((PASS + 1))
|
|
2313
|
+
else
|
|
2314
|
+
ER_OK=0
|
|
2315
|
+
echo "FAIL: report-policy.cjs decides wrongly:" >&2
|
|
2316
|
+
sed 's/^/ /' /tmp/policy-err >&2
|
|
2317
|
+
fi
|
|
2318
|
+
else
|
|
2319
|
+
ER_OK=0
|
|
2320
|
+
echo "FAIL: report-workflow-errors has no report-policy.cjs" >&2
|
|
2321
|
+
fi
|
|
2322
|
+
er 'policy\.select\(ranked' 'does not filter what it files through the policy'
|
|
2323
|
+
er 'quiet\.cancelledNoStep \+= 1; continue;' 'files cancelled jobs that never reached a failing step'
|
|
2324
|
+
er "patternId === 'log-unavailable'\) \{ quiet\.unreadable \+= 1; continue; \}" 'files jobs whose log could not be read'
|
|
2325
|
+
er 'policy\.isQuiet\(' 'does not keep a recently closed signature quiet'
|
|
2326
|
+
er 'listAnnotations' 'does not read the job annotations, where the runner shutdown is written'
|
|
2327
|
+
|
|
2251
2328
|
# 4. No model. A model asked to summarise a failure paraphrases whatever the log held, which
|
|
2252
2329
|
# is the one thing that must not cross the boundary. This job stays deterministic.
|
|
2253
2330
|
if grep -qiE '(engine:|opencode|safe-outputs|OPENAI_API_KEY)' "$ERROR_REPORT_YML"; then
|
|
@@ -2578,6 +2655,46 @@ if worker_installed refine; then
|
|
|
2578
2655
|
PASS=$((PASS + 1))
|
|
2579
2656
|
fi
|
|
2580
2657
|
fi
|
|
2658
|
+
# The implement prompt used to say the safe-output tools are called as `safeoutputs/<tool>(...)`, an
|
|
2659
|
+
# MCP function call. The sandbox mounts them as a shell program on PATH instead, and the agent works
|
|
2660
|
+
# through an `execute` tool whose `tools` object does not hold them, so it searched there, found
|
|
2661
|
+
# nothing and ended Pliny-Bot #694 after seventy-eight minutes and sixteen commits with no pull
|
|
2662
|
+
# request. The prompt must show the shell form the refine prompt already uses.
|
|
2663
|
+
if worker_installed implement; then
|
|
2664
|
+
IMPLEMENT_SO_MD="${WORKFLOWS_DIR}/agent-implement.md"
|
|
2665
|
+
if grep -qF "safeoutputs create_pull_request ." "$IMPLEMENT_SO_MD" && grep -qF "safeoutputs report_incomplete ." "$IMPLEMENT_SO_MD" && ! grep -qF "safeoutputs/create_pull_request(" "$IMPLEMENT_SO_MD"; then
|
|
2666
|
+
PASS=$((PASS + 1))
|
|
2667
|
+
else
|
|
2668
|
+
FAIL=$((FAIL + 1))
|
|
2669
|
+
echo "FAIL: the implement prompt does not show the safe-output tools as shell commands on PATH; an agent that looks for them in the execute tool ends the run with nothing" >&2
|
|
2670
|
+
fi
|
|
2671
|
+
fi
|
|
2672
|
+
# The implement run is not interactive, so a turn that ends in text ends the run. Pliny-Bot #750 moved
|
|
2673
|
+
# `pnpm verify` to the background, said it would be resumed with the result and stopped: eight commits,
|
|
2674
|
+
# no pull request. The prompt must forbid backgrounding and waiting, and name the remedy.
|
|
2675
|
+
if worker_installed implement; then
|
|
2676
|
+
IMPLEMENT_FG_MD="${WORKFLOWS_DIR}/agent-implement.md"
|
|
2677
|
+
if grep -qF "Run every command in the foreground and wait for it in the same turn" "$IMPLEMENT_FG_MD" && grep -qF "never end a turn to wait for it" "$IMPLEMENT_FG_MD" && grep -qF "timeout" "$IMPLEMENT_FG_MD"; then
|
|
2678
|
+
PASS=$((PASS + 1))
|
|
2679
|
+
else
|
|
2680
|
+
FAIL=$((FAIL + 1))
|
|
2681
|
+
echo "FAIL: the implement prompt does not forbid running commands in the background and ending the turn to wait for them" >&2
|
|
2682
|
+
fi
|
|
2683
|
+
fi
|
|
2684
|
+
# The merge train wakes on a finished CI run only. Pliny-Bot had four green pull requests wait an hour
|
|
2685
|
+
# because a repair gate dispatched for red CI kept them marked as having a gate, and nothing called the
|
|
2686
|
+
# train again once it was cancelled. The hourly sweep is now a heartbeat, and a repair gate whose CI has
|
|
2687
|
+
# turned green is cancelled by the train itself.
|
|
2688
|
+
HK_YML="${HERE}/../housekeeping/action.yml"
|
|
2689
|
+
TRAIN_SH_HB="${HERE}/../merge-train/merge-train.sh"
|
|
2690
|
+
if [ -f "$HK_YML" ] && [ -f "$TRAIN_SH_HB" ]; then
|
|
2691
|
+
if grep -qF "operation: 'merge-train'" "$HK_YML" && grep -qF "createWorkflowDispatch" "$HK_YML" && grep -qF "its CI is green" "$TRAIN_SH_HB" && grep -qF 'live_titles=$(printf' "$TRAIN_SH_HB"; then
|
|
2692
|
+
PASS=$((PASS + 1))
|
|
2693
|
+
else
|
|
2694
|
+
FAIL=$((FAIL + 1))
|
|
2695
|
+
echo "FAIL: the merge train has no heartbeat from housekeeping, or does not cancel a repair gate whose CI turned green" >&2
|
|
2696
|
+
fi
|
|
2697
|
+
fi
|
|
2581
2698
|
# The implement prompt tells the agent it is already on its branch and to skip the pc-plan-goal
|
|
2582
2699
|
# branching phase. That is only true if a step before the agent made the branch, and nothing in the
|
|
2583
2700
|
# sandbox can do it later: no git credentials, and the prompt forbids the commands. Without the
|
|
@@ -2758,6 +2875,11 @@ if worker_installed merge-gate; then
|
|
|
2758
2875
|
GATE_RUN_ID_OK=0
|
|
2759
2876
|
echo "FAIL: identify-gate-subject resolves the CI run only when the conclusion is unknown; a router that passes both gets an empty run ID" >&2
|
|
2760
2877
|
fi
|
|
2878
|
+
# A verdict from a run that judged an older head is not a verdict on this one (Pliny-Bot #729).
|
|
2879
|
+
grep -Fq '"$run_sha" != "$current_sha"' "$GATE_SUBJECT_ACTION" ||
|
|
2880
|
+
{ GATE_RUN_ID_OK=0; echo "FAIL: identify-gate-subject trusts the carried CI run even when it judged an older head" >&2; }
|
|
2881
|
+
grep -Fq 'select(.status == "completed")' "$GATE_SUBJECT_ACTION" ||
|
|
2882
|
+
{ GATE_RUN_ID_OK=0; echo "FAIL: identify-gate-subject re-reads a stale verdict without requiring a finished run on the head" >&2; }
|
|
2761
2883
|
fi
|
|
2762
2884
|
# Repeating a report the validator refused reproduces it, and every repeat is a fresh agent run
|
|
2763
2885
|
# holding the repo-wide merge-belt slot. The worker gives that failure a smaller budget than a
|
package/loops/docs/diagrams.md
CHANGED
|
@@ -709,7 +709,7 @@ the repository it runs in. Every consumer is private, so that is the whole desig
|
|
|
709
709
|
|
|
710
710
|
What crosses the boundary is a fixed, enumerable set of facts about workflows **this package ships**:
|
|
711
711
|
their file names, their job and step names, a conclusion, a runner label, a count, and the id of a
|
|
712
|
-
matched entry from a catalogue of sixteen known failure shapes. What never crosses it is free text
|
|
712
|
+
matched entry from a catalogue of sixteen known failure shapes, plus a severity. What never crosses it is free text
|
|
713
713
|
of any kind no log line, no branch name, no issue title, no path, no URL, no number that could be
|
|
714
714
|
looked up. Hence no model: a model asked to summarise a failure paraphrases whatever the log held.
|
|
715
715
|
|
|
@@ -720,8 +720,12 @@ flowchart TB
|
|
|
720
720
|
erOwned -.->|"no: the name could describe<br/>a product or a customer"| erCount("Counted, never inspected")
|
|
721
721
|
erOwned -->|yes| erState{"Failed, or queued<br/>past 45 minutes?"}
|
|
722
722
|
erState -.->|neither| erCount
|
|
723
|
-
erState -->|yes| erClassify("Read the log tail,<br/>match the catalogue,<br/>keep only the matched id")
|
|
724
|
-
erClassify -->
|
|
723
|
+
erState -->|yes| erClassify("Read the log tail and the job annotations,<br/>match the catalogue,<br/>keep only the matched id")
|
|
724
|
+
erClassify --> erNoise{"Cancelled with no failing step,<br/>log unreadable, or an<br/>infrastructure cause?"}
|
|
725
|
+
erNoise -.->|yes| erCount
|
|
726
|
+
erNoise -->|no| erFilter{"Seen at least twice,<br/>within the cap, and not<br/>closed in the last 30 days?"}
|
|
727
|
+
erFilter -.->|no| erCount
|
|
728
|
+
erFilter -->|yes| erBody("Build the report from<br/>package-owned names,<br/>a conclusion and a count")
|
|
725
729
|
erBody --> erScan{"Leak scanner:<br/>repo or owner name, a URL,<br/>an email, a path, a token,<br/>an issue number, a ref?"}
|
|
726
730
|
erScan -.->|"anything matches"| erWithhold(("Withheld<br/>nothing filed, run goes red"))
|
|
727
731
|
erScan -->|clean| erToken{"Upstream token<br/>available?"}
|
|
@@ -737,7 +741,7 @@ flowchart TB
|
|
|
737
741
|
classDef idle fill:#f4f4f6,stroke:#5b5b66,stroke-width:2px,color:#2b2b33
|
|
738
742
|
class erCron start
|
|
739
743
|
class erRuns,erClassify,erBody,erFile,erCount action
|
|
740
|
-
class erOwned,erState,erScan,erToken decision
|
|
744
|
+
class erOwned,erState,erNoise,erFilter,erScan,erToken decision
|
|
741
745
|
class erWithhold failure
|
|
742
746
|
class erDone success
|
|
743
747
|
class erSummary idle
|
|
@@ -69,6 +69,8 @@ jobs:
|
|
|
69
69
|
# upstream with a separate token and writes nothing here.
|
|
70
70
|
contents: read
|
|
71
71
|
actions: read
|
|
72
|
+
# The annotations GitHub writes on a job say why a runner went away; the log does not.
|
|
73
|
+
checks: read
|
|
72
74
|
steps:
|
|
73
75
|
- name: Checkout workflow actions
|
|
74
76
|
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
@@ -545,6 +545,8 @@ timeout-minutes: 90
|
|
|
545
545
|
`node .opencode/scripts/check-diff.mjs`.
|
|
546
546
|
It must exit 0: delete the comments and tests it lists. The merge gate runs it too and does not
|
|
547
547
|
merge while it fails.
|
|
548
|
+
When you have to add a test, put it at the integration layer through the public surface, not as a
|
|
549
|
+
unit test of internals; a bug fix gets one regression test.
|
|
548
550
|
|
|
549
551
|
7. Select one review outcome.
|
|
550
552
|
|
|
@@ -594,6 +594,7 @@ jobs:
|
|
|
594
594
|
body: |
|
|
595
595
|
${{ env.IMPLEMENT_MARKER }}
|
|
596
596
|
${{ env.INCOMPLETE_COMMENT }}
|
|
597
|
+
What ended: the agent job was **${{ needs.agent.result }}** after ${{ steps.decide.outputs.minutes }} minutes (-1 when that could not be read), and publishing its output was **${{ needs.safe_outputs.result }}**.
|
|
597
598
|
[View this workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})
|
|
598
599
|
agent:
|
|
599
600
|
needs: [eligibility]
|
|
@@ -780,12 +781,37 @@ timeout-minutes: 180
|
|
|
780
781
|
|
|
781
782
|
Never open a pull request that does not pass them.
|
|
782
783
|
|
|
784
|
+
**Run every command in the foreground and wait for it in the same turn. Never move a command
|
|
785
|
+
to the background and never end a turn to wait for it.** This run is not interactive: nothing
|
|
786
|
+
resumes you. A turn that ends with text and no tool call ends the whole run, and the work is
|
|
787
|
+
discarded. Pliny-Bot #750 started `pnpm verify` in the background, wrote "I'll be resumed with
|
|
788
|
+
its result" and stopped: the run ended green with eight commits and no pull request, and #694
|
|
789
|
+
lost seventy-eight minutes the same way. The bash tool cuts a command at 120 seconds unless you
|
|
790
|
+
pass a `timeout` (milliseconds, up to 600000). Give a long command that timeout, and when it is
|
|
791
|
+
still too long, run it in parts (lint, typecheck, web tests, web build, dotnet build, dotnet test)
|
|
792
|
+
so every part finishes inside one call. The only valid way to end the run is the safe-output
|
|
793
|
+
call in step 6.
|
|
794
|
+
|
|
795
|
+
**Two things the unit tests do not catch, and CI will.** When you add a dependency, take the
|
|
796
|
+
latest released version and run the repository's audit for it (`pnpm audit --audit-level high`
|
|
797
|
+
for the web, `dotnet list package --vulnerable` for the backend): Pliny-Bot #752 added a PDF
|
|
798
|
+
library at a version with two critical and eight high advisories and the dependency scan
|
|
799
|
+
blocked it. When you register a new service, interface or handler, run at least one test that
|
|
800
|
+
boots the whole application, not only unit tests: Pliny-Bot #753 registered an interface for
|
|
801
|
+
one engine only, the application could not start under the others and every integration test
|
|
802
|
+
failed.
|
|
803
|
+
|
|
783
804
|
When `.opencode/scripts/check-diff.mjs` exists, also run
|
|
784
805
|
`node .opencode/scripts/check-diff.mjs`
|
|
785
806
|
on both paths. It must exit 0. It holds the comments and tests you add to the size of the
|
|
786
807
|
change and makes you clean the files you touched; fix every line it lists. The merge gate
|
|
787
808
|
runs it again and does not merge while it fails.
|
|
788
809
|
|
|
810
|
+
Shape what you test like a trophy, not a pyramid: put new tests at the integration or
|
|
811
|
+
acceptance layer, through the public surface of the code you changed, and write a plain
|
|
812
|
+
unit test only for logic with real branching. A bug fix gets one regression test. Do not
|
|
813
|
+
write tests of constants, configured defaults, mocks of your own code or markup attributes.
|
|
814
|
+
|
|
789
815
|
Anything you change to satisfy `check-diff` is a change: run the verification commands
|
|
790
816
|
again afterwards, and open the pull request only from a tree where both pass. Do not delete
|
|
791
817
|
a test that covers a branch you added to fit the budget; the complexity gate in CI measures
|
|
@@ -797,16 +823,37 @@ timeout-minutes: 180
|
|
|
797
823
|
code already written.
|
|
798
824
|
|
|
799
825
|
6. Finish by calling **exactly one** safe-output tool. A run that calls none is a wasted run:
|
|
800
|
-
the workflow reports a failure and everything you just did is discarded.
|
|
801
|
-
|
|
826
|
+
the workflow reports a failure and everything you just did is discarded. Pliny-Bot #694 spent
|
|
827
|
+
seventy-eight minutes and sixteen commits on a finished change and ended with nothing: the
|
|
828
|
+
agent searched for the tool in the `tools` object of the `execute` tool, did not find it
|
|
829
|
+
there, and concluded it did not exist.
|
|
830
|
+
|
|
831
|
+
**The safe-output tools are shell commands on your PATH, not entries in `tools`.** The sandbox
|
|
832
|
+
mounts the `safeoutputs` server as a command-line program. It is not under `tools.*`, `search`
|
|
833
|
+
does not return it, and an empty result from either says nothing about it. Run it with the
|
|
834
|
+
bash tool, and run `command -v safeoutputs` there first if you are unsure it is present.
|
|
835
|
+
|
|
836
|
+
**Pipe the JSON on stdin, put every parameter inside the JSON, and pass a lone `.` and no
|
|
837
|
+
flags.** The bridge reads the payload from stdin only when the command line is the tool name
|
|
838
|
+
followed by `.`. A `--flag` on the line makes it skip stdin and send just the flags, drop the
|
|
839
|
+
body and still report success. Correct:
|
|
840
|
+
|
|
841
|
+
`printf '%s' "$JSON" | safeoutputs create_pull_request .` with
|
|
842
|
+
`{"title":"[bot] Fix X","body":"Closes #${{ inputs.issue-number }}\n\n...","branch":"${{ env.IMPLEMENT_BRANCH_PREFIX }}${{ inputs.issue-number }}"}`
|
|
843
|
+
|
|
844
|
+
`printf '%s' "$JSON" | safeoutputs report_incomplete .` with `{"reason":"..."}`
|
|
845
|
+
|
|
846
|
+
`printf '%s' "$JSON" | safeoutputs noop .` with `{"message":"..."}`
|
|
847
|
+
|
|
848
|
+
Wrong (the body is silently empty): `safeoutputs create_pull_request --title "..." .`
|
|
802
849
|
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
850
|
+
If the call fails, read the error and send the whole payload again once. If it fails twice,
|
|
851
|
+
call `report_incomplete` the same way with the reason; never end the run with finished work
|
|
852
|
+
and no call.
|
|
806
853
|
|
|
807
854
|
Choose exactly one:
|
|
808
855
|
|
|
809
|
-
- **`
|
|
856
|
+
- **`create_pull_request`** , the normal path. Propose a pull request with the
|
|
810
857
|
verified changes. Do not pass `base`: the workflow sets it to the branch this run started
|
|
811
858
|
on, which is the branch you are working from, so a stack child lands on its feature branch
|
|
812
859
|
without you doing anything. The `body` **must close exactly this issue and nothing else**
|
|
@@ -814,10 +861,10 @@ timeout-minutes: 180
|
|
|
814
861
|
the branch you are on, the one named in step 1; do not invent another name. You do not
|
|
815
862
|
need to check whether a pull request already exists for this issue: the router does
|
|
816
863
|
that before dispatching you and does not start this workflow when one does.
|
|
817
|
-
- **`
|
|
864
|
+
- **`report_incomplete`** , only when infrastructure or tooling prevents you
|
|
818
865
|
from completing the task, such as a pre-existing build failure you cannot fix. Provide a
|
|
819
866
|
specific `reason`.
|
|
820
|
-
- **`
|
|
867
|
+
- **`noop`** , only when the issue context shows the work is already done and no
|
|
821
868
|
changes are needed. Provide a `message` explaining what you found.
|
|
822
869
|
|
|
823
870
|
Do not manage labels or post comments , the conclude job handles that.
|