@plainconceptsplatform/workflows 1.6.11 → 1.6.21

package/loops/actions/housekeeping/action.yml

@@ -499,6 +499,21 @@ runs:
             }
           }
 
+          // The merge train wakes when a CI run finishes. When a gate it was waiting on ends without
+          // one, or is cancelled, nothing calls it again and every pull request sits until the next
+          // CI event: Pliny-Bot had four green pull requests waiting on that for the better part of an
+          // hour. The hourly sweep is the heartbeat. A dispatch of the router by the workflow token is
+          // allowed to start a run, and a tick with nothing to do costs one short job.
+          const openPulls = pulls.filter((pr) => !pr.draft);
+          if (openPulls.length > 0) {
+            await act(`dispatching a merge train tick: ${openPulls.length} pull request(s) are open`, () =>
+              github.rest.actions.createWorkflowDispatch({
+                ...context.repo, workflow_id: 'work-router.yml',
+                ref: context.payload.repository?.default_branch ?? 'main',
+                inputs: { operation: 'merge-train' },
+              }));
+          }
+
           await core.summary
             .addHeading('Housekeeping', 2)
             .addList([

package/loops/actions/merge-train/action.yml

@@ -32,6 +32,12 @@ inputs:
     description: How many pull requests may be in repair at once. Merges stay one at a time.
     required: false
     default: '1'
+  main-red-wait-hours:
+    description: >-
+      How long the train waits for a base branch whose checks fail the same way as a pull request's
+      before it stops waiting and repairs the pull request anyway.
+    required: false
+    default: '2'
   stuck-hours:
     description: How long the train waits behind a CI run that never finishes before it passes that pull request by.
     required: false
@@ -64,6 +70,7 @@ runs:
         MAX_ATTEMPTS: ${{ inputs.max-gate-attempts }}
         MAX_RETRIES: ${{ inputs.max-retries }}
         STUCK_HOURS: ${{ inputs.stuck-hours }}
+        MAIN_RED_WAIT_HOURS: ${{ inputs.main-red-wait-hours }}
         MAX_REPAIRS: ${{ inputs.max-repairs }}
         DRY_RUN: ${{ inputs.dry-run }}
         HINT_RUN_ID: ${{ inputs.ci-run-id }}

package/loops/actions/merge-train/merge-train.sh

@@ -146,6 +146,7 @@ MAX_ATTEMPTS="${MAX_ATTEMPTS:-6}"
 MAX_RETRIES="${MAX_RETRIES:-3}"
 MAX_REPAIRS="${MAX_REPAIRS:-1}"
 case "$MAX_REPAIRS" in ''|*[!0-9]*|0) MAX_REPAIRS=1 ;; esac
+MAIN_RED_WAIT_HOURS="${MAIN_RED_WAIT_HOURS:-2}"
 STUCK_HOURS="${STUCK_HOURS:-3}"
 DRY_RUN="${DRY_RUN:-false}"
 SELF_RUN_ID="${GITHUB_RUN_ID:-0}"
@@ -423,9 +424,22 @@ for line in "${candidates[@]}"; do
       --jq '[.check_runs[] | select(.conclusion == "failure") | (.name | sub("^.* / "; ""))] | unique | .[]' 2>/dev/null || true)
     failed_base=$(gh api "repos/$REPO/commits/$base_ref/check-runs?per_page=100" \
       --jq '[.check_runs[] | select(.conclusion == "failure") | (.name | sub("^.* / "; ""))] | unique | .[]' 2>/dev/null || true)
+    # How long the base has been red: the oldest failing check on its tip. The wait is bounded,
+    # because waiting for a base nobody fixes starves every pull request behind it. Pliny-Bot's
+    # main was red from 19:30 and the train waited until morning, ten hours, saying nothing.
+    red_since=$(gh api "repos/$REPO/commits/$base_ref/check-runs?per_page=100" \
+      --jq '[.check_runs[] | select(.conclusion == "failure") | .completed_at] | min // empty' 2>/dev/null || true)
+    red_for_hours=0
+    if [ -n "$red_since" ]; then
+      red_for_hours=$(( (now_epoch - $(iso_epoch "$red_since")) / 3600 ))
+    fi
     if [ -n "$failed_here" ] && [ -n "$failed_base" ] &&
       [ -z "$(comm -23 <(sort <<<"$failed_here") <(sort <<<"$failed_base"))" ]; then
-      same_as_main=true
+      if [ "$red_for_hours" -lt "${MAIN_RED_WAIT_HOURS:-2}" ]; then
+        same_as_main=true
+      else
+        echo "::warning::$base_ref has been red for ${red_for_hours}h on the same checks as #$pr; the train no longer waits for it and repairs #$pr."
+      fi
     fi
   fi
 
@@ -439,6 +453,20 @@ for line in "${candidates[@]}"; do
   move=$(sed -n 's/^move=//p' <<<"$decision")
   reason=$(sed -n 's/^reason=//p' <<<"$decision")
 
+  # A repair gate dispatched for red CI is moot once CI on the head is green: its job was to make
+  # that run pass. Left alone it keeps the pull request marked as having a gate, so the train
+  # skipped four green Pliny-Bot pull requests with "a gate is already on it" until it was cancelled
+  # by hand. Cancel it and let the pull request move.
+  if { [ "$move" = gate ] || [ "$move" = update ]; } && [ "$ci_status" = completed ] && [ "$ci_conclusion" = success ] &&
+    printf '%s\n' "$live_titles" | grep -qE "PR #${pr} \(repair\)$"; then
+    for run_id in $(gh api "repos/$REPO/actions/workflows/work-router.yml/runs?per_page=100" \
+      --jq ".workflow_runs[] | select(.status != \"completed\" and (.display_title | endswith(\"PR #${pr} (repair)\"))) | .id" 2>/dev/null); do
+      act "Cancelling the repair gate $run_id on #$pr: its CI is green." \
+        gh api -X POST "repos/$REPO/actions/runs/$run_id/cancel" --silent
+    done
+    live_titles=$(printf '%s\n' "$live_titles" | grep -vE "PR #${pr} \(repair\)$" || true)
+  fi
+
   # A gate is already on this pull request, whatever kind: offering it another would run the same
   # work twice. It does not hold the line up for the repair kind; a merging gate already does.
   if [ "$move" != skip ] && live_for_pr "$pr"; then

package/loops/actions/report-workflow-errors/action.yml

@@ -23,6 +23,20 @@ inputs:
     description: How far back to look for failures.
     required: false
     default: '24'
+  min-occurrences:
+    description: >-
+      A signature seen fewer times than this inside the window does not open a new issue. One
+      occurrence is a blip; an issue that is already open is always appended to.
+    required: false
+    default: '2'
+  max-findings:
+    description: The most issues one run opens, the most frequent first. The rest stay in the job summary.
+    required: false
+    default: '5'
+  quiet-days:
+    description: A signature whose issue was closed within this many days stays quiet, whatever the reason it was closed.
+    required: false
+    default: '30'
   dry-run:
     description: Compute and print the report, file nothing.
     required: false
@@ -43,6 +57,10 @@ runs:
         UPSTREAM_TOKEN: ${{ inputs.upstream-token }}
         UPSTREAM_REPO: ${{ inputs.upstream-repo }}
         LOOKBACK_HOURS: ${{ inputs.lookback-hours }}
+        MIN_OCCURRENCES: ${{ inputs.min-occurrences }}
+        MAX_FINDINGS: ${{ inputs.max-findings }}
+        QUIET_DAYS: ${{ inputs.quiet-days }}
+        POLICY_PATH: ${{ github.action_path }}/report-policy.cjs
         DRY_RUN: ${{ inputs.dry-run }}
       with:
         github-token: ${{ inputs.token }}
@@ -56,6 +74,10 @@ runs:
           // is checked against a leak scanner and the job fails rather than files if the
           // scanner trips.
 
+          // What to file and what to leave as a number lives in report-policy.cjs, apart from
+          // what this script reads and writes, so it can be run on its own.
+          const policy = require(process.env.POLICY_PATH);
+
           const dryRun = process.env.DRY_RUN === 'true';
           const upstreamToken = process.env.UPSTREAM_TOKEN;
           const [upstreamOwner, upstreamName] = process.env.UPSTREAM_REPO.split('/');
@@ -140,7 +162,7 @@ runs:
             const header = require('node:fs')
               .readFileSync('.github/workflows/work-router.yml', 'utf8')
               .split('\n')[0];
-            packageVersion = /@plainconceptsplatform\/workflows@([0-9][0-9A-Za-z.\-+]*)/.exec(header)?.[1] ?? 'unknown';
+            packageVersion = policy.parseVersion(header);
           } catch {
             core.info('could not read the installed package version from the router header');
           }
@@ -153,15 +175,19 @@ runs:
           // upstream, and every part of it is package-owned or an enumerated value.
           //
           // THE FINDING SHAPE IS THE PRIVACY BOUNDARY. bodyFor() renders a finding, so a field
-          // added here is a field sent to another repository. These eight are all
+          // added here is a field sent to another repository. These ten are all
           // package-owned names or enumerated values; nothing derived from a log, a branch, a
           // path or an issue belongs on this object. The route matrix asserts this exact list,
           // so adding a ninth field fails the build rather than leaking on the next cron.
-          const FINDING_FIELDS = ['workflow', 'job', 'step', 'conclusion', 'patternId', 'count', 'runnerLabels', 'attempts'];
+          const FINDING_FIELDS = ['workflow', 'job', 'step', 'conclusion', 'patternId', 'count', 'runnerLabels', 'attempts', 'severity', 'variants'];
           const findings = new Map();
           let ownedRuns = 0;
           let ownedFailures = 0;
           let skippedForeign = 0;
+          // Things that happened and are worth a number in the job summary, but that nobody can
+          // fix by editing a workflow, so they never open an issue: a cancelled job with no step
+          // that failed, a log that could not be read, and the fleet's own transient causes.
+          const quiet = { cancelledNoStep: 0, unreadable: 0, infra: new Map() };
           let queueWaitTotal = 0;
           let queueWaitCount = 0;
           let queueWaitWorst = 0;
@@ -180,15 +206,7 @@ runs:
             if (['queued', 'waiting', 'pending'].includes(run.status) &&
                 Date.now() - new Date(run.created_at).getTime() > STUCK_AFTER_MS) {
               stuckInQueue += 1;
-              const signature = `${file} · (whole run) · (never started) · queued · runner-never-assigned`;
-              const seen = findings.get(signature) ?? {
-                workflow: file, job: '(whole run)', step: '(never started)',
-                conclusion: 'queued', patternId: 'runner-never-assigned', count: 0,
-                runnerLabels: new Set(), attempts: new Set(),
-              };
-              seen.count += 1;
-              seen.attempts.add(run.run_attempt ?? 1);
-              findings.set(signature, seen);
+              quiet.infra.set('runner-never-assigned', (quiet.infra.get('runner-never-assigned') ?? 0) + 1);
               continue;
             }
 
@@ -217,7 +235,24 @@ runs:
               // package's own words. A `run-name` is not: it interpolates issue titles.
               const stepName = step?.name ?? '(no failing step recorded)';
 
+              // A cancelled job that never reached a failing step is somebody or something
+              // stopping work, not a defect: a superseded run, a job cancelled behind a failed
+              // one. It was 189 of 320 reports and every one said nothing.
+              if (job.conclusion === 'cancelled' && !step) { quiet.cancelledNoStep += 1; continue; }
+
+              // GitHub writes why a runner went away into the job's annotations ("The runner has
+              // received a shutdown signal", exit code 143), not into the log, so they are read
+              // with it. Only the id of a catalogue entry that matches ever leaves.
+              let annotationText = '';
+              try {
+                const notes = await github.paginate(github.rest.checks.listAnnotations, {
+                  ...context.repo, check_run_id: job.id, per_page: 50,
+                });
+                annotationText = notes.map((note) => note.message ?? '').join('\n');
+              } catch { /* annotations are an extra; the log can still classify */ }
+
               let patternId = 'unrecognised';
+              let logRead = true;
               try {
                 const { data: log } = await github.rest.actions.downloadJobLogsForWorkflowRun({
                   ...context.repo, job_id: job.id,
@@ -225,19 +260,32 @@ runs:
                 const text = typeof log === 'string' ? log : Buffer.from(log).toString('utf8');
                 // The tail is where the failure is, and reading less of the log is less to get
                 // wrong. Only the matched entry's id crosses into the report.
-                const tail = text.slice(-20000);
+                const tail = `${annotationText}\n${text.slice(-20000)}`;
                 patternId = CATALOGUE.find((entry) => entry.test.test(tail))?.id ?? 'unrecognised';
               } catch {
-                patternId = 'log-unavailable';
+                logRead = false;
+                patternId = CATALOGUE.find((entry) => entry.test.test(annotationText))?.id ?? 'log-unavailable';
+              }
+
+              // A log nobody could read says nothing, so it is counted and not filed.
+              if (patternId === 'log-unavailable') { quiet.unreadable += 1; continue; }
+              if (!logRead && policy.severityOf(patternId) === 'infra') {
+                quiet.infra.set(patternId, (quiet.infra.get(patternId) ?? 0) + 1);
+                continue;
               }
 
-              const signature = `${file} · ${job.name} · ${stepName} · ${job.conclusion} · ${patternId}`;
+              // One cause is one finding, whichever job tripped on it.
+              const signature = policy.signatureOf({
+                workflow: file, step: stepName, conclusion: job.conclusion, patternId,
+              });
               const seen = findings.get(signature) ?? {
                 workflow: file, job: job.name, step: stepName,
-                conclusion: job.conclusion, patternId, count: 0,
+                conclusion: job.conclusion, patternId: patternId, count: 0,
                 runnerLabels: new Set(), attempts: new Set(),
+                severity: policy.severityOf(patternId), variants: new Set(),
               };
               seen.count += 1;
+              seen.variants.add(`${job.name}|${stepName}`);
               seen.attempts.add(run.run_attempt ?? 1);
               // A pool label is the one value on a finding that the CONSUMER writes, and the
               // installer deliberately preserves theirs across updates (preserveRunnerPool in
@@ -261,6 +309,13 @@ runs:
           const ranked = [...findings.values()].sort((a, b) => b.count - a.count);
           const hours = process.env.LOOKBACK_HOURS;
 
+          // What becomes an issue. Infrastructure causes are a rate, a blip is not worth a
+          // person's time, and a bad night opens a handful of issues rather than forty.
+          const selection = policy.select(ranked, {
+            minOccurrences: Number(process.env.MIN_OCCURRENCES),
+            maxFindings: Number(process.env.MAX_FINDINGS),
+          });
+
           // Enforce the boundary at run time as well as in the build. A field that reached a
           // finding by some path the static check did not see stops the job here rather than
           // being rendered into an issue in another repository.
@@ -273,7 +328,7 @@ runs:
           }
 
           const bodyFor = (finding) => [
-            `<!-- pcp-error-report: ${finding.workflow}|${finding.job}|${finding.step}|${finding.patternId} -->`,
+            `<!-- pcp-error-report: ${policy.markerKey(finding)} -->`,
             '',
             'Filed automatically by the `report-workflow-errors` job in a repository that installs',
             'this package. It carries no information about that repository: only the names this',
@@ -286,6 +341,8 @@ runs:
             `| Failing step | \`${finding.step}\` |`,
             `| Conclusion | \`${finding.conclusion}\` |`,
             `| Classification | \`${finding.patternId}\` |`,
+            `| Severity | \`${finding.severity}\` |`,
+            `| Job and step combinations with this cause | ${finding.variants.size} |`,
             `| Runner labels | ${[...finding.runnerLabels].map((l) => `\`${l}\``).join(', ') || 'not reported'} |`,
             `| Occurrences in the last ${hours}h | ${finding.count} |`,
             `| Re-run attempts involved | ${[...finding.attempts].sort().join(', ')} |`,
@@ -298,8 +355,9 @@ runs:
             `- ${new Date().toISOString().slice(0, 10)}: ${finding.count} in ${hours}h, on \`${packageVersion}\``,
           ].join('\n');
 
-          const titleFor = (finding) =>
-            `${finding.workflow}: ${finding.patternId} in "${finding.step}"`;
+          const titleFor = (finding) => (finding.patternId === 'unrecognised'
+            ? `[${finding.severity}] ${finding.workflow}: ${finding.patternId} in "${finding.step}"`
+            : `[${finding.severity}] ${finding.workflow}: ${finding.patternId}`);
 
           // The scanner. A body that trips it is never filed, and the job goes red so the
           // pattern that produced it gets fixed rather than quietly dropped every night.
@@ -325,6 +383,10 @@ runs:
               'This report does not cover itself: a failure in this job shows up in this repository only.',
               `Runs still queued after 45 minutes, so no runner ever took them: ${stuckInQueue}`,
               `Distinct failure signatures: ${ranked.length}`,
+              `Filed or appended: ${selection.file.length}; held back: ${selection.held.length}`,
+              `Cancelled jobs with no failing step, not reported: ${quiet.cancelledNoStep}`,
+              `Jobs whose log could not be read, not reported: ${quiet.unreadable}`,
+              `Infrastructure causes, counted and not reported: ${[...quiet.infra].map(([id, n]) => `${id} ${n}`).join(', ') || 'none'}`,
               queueWaitCount > 0
                 ? `Runner queue wait: ${Math.round(queueWaitTotal / queueWaitCount)}s mean, ${queueWaitWorst}s worst`
                 : 'Runner queue wait: no jobs started in the window',
@@ -345,7 +407,7 @@ runs:
           // full picture rather than one night at a time.
           const reports = [];
           let leaked = 0;
-          for (const finding of ranked) {
+          for (const finding of selection.file) {
             const title = titleFor(finding);
             const body = bodyFor(finding);
             if (!scan(`${title}\n${body}`, `The report for ${finding.patternId}`)) { leaked += 1; continue; }
@@ -373,10 +435,17 @@ runs:
             let existing;
             try {
               const { data } = await upstream.rest.search.issuesAndPullRequests({
-                q: `repo:${upstreamOwner}/${upstreamName} is:issue is:open in:body "${marker}"`,
-                per_page: 5,
+                q: `repo:${upstreamOwner}/${upstreamName} is:issue in:body "${marker}"`,
+                per_page: 10,
               });
-              existing = data.items.find((item) => (item.body ?? '').includes(marker));
+              const matches = data.items.filter((item) => (item.body ?? '').includes(marker));
+              existing = matches.find((item) => item.state === 'open');
+              // Closing an issue is the answer to it. Without this every bulk close reopened the
+              // whole set the next morning: 150 of 320 reports re-filed a signature seen before.
+              if (!existing && matches.some((item) => policy.isQuiet(item.closed_at, Date.now(), Number(process.env.QUIET_DAYS)))) {
+                core.info(`left alone, an issue for it was closed within ${process.env.QUIET_DAYS} days: ${title}`);
+                continue;
+              }
             } catch (error) {
               core.warning(`could not search upstream for an existing report: HTTP ${error.status ?? '?'}`);
             }

package/loops/actions/report-workflow-errors/report-policy.cjs

@@ -0,0 +1,110 @@
+// Managed by @plainconceptsplatform/workflows. Source: loops/actions/report-workflow-errors/report-policy.cjs. Update with workflows update --force; consumer edits may be overwritten.
+//
+// What the error report decides, kept apart from what it reads and files so it can be run on its
+// own. Pure: no network, no clock, nothing read from the repository. Everything it sees is a
+// finding the report already built from package-owned names and enumerated values, so nothing
+// here can widen what leaves a private repository.
+//
+// The report used to open one issue per job, step and classification, for every failure it saw.
+// Read back over a month, 320 issues: 59% were cancelled jobs, 80% carried no diagnosis, 14% were
+// a runner shut down under a job, 47% re-filed a signature seen before, every one was closed as
+// not planned and none had a comment. What follows is the filter that would have kept about a
+// hundred of them out of that pile and left the rest as a count in the job summary.
+
+'use strict';
+
+// Causes that say something about the fleet, not about this package. They are counted in the job
+// summary, where a trend is visible, and never become an issue: nobody can fix a runner that was
+// taken away mid-job by editing a workflow. A job that hit its timeout is NOT here, because that
+// can be the package's own work taking too long.
+const INFRA = new Set([
+  'runner-never-assigned',
+  'runner-lost-mid-job',
+  'gh-api-rate-limit',
+  'model-quota',
+  'model-gateway-unreachable',
+  'out-of-disk',
+  'out-of-memory',
+]);
+
+// Causes that are a defect in something this package ships.
+const PACKAGE_BUG = new Set([
+  'jq-error',
+  'yaml-or-expression-invalid',
+  'lock-out-of-date',
+  'action-not-found',
+  'agent-produced-no-output',
+  'gh-api-403',
+  'gh-api-404',
+  'gh-api-409-conflict',
+]);
+
+/** infra, package-bug, or unknown (a cause nobody classified yet, worth a look when it repeats). */
+function severityOf(patternId) {
+  if (INFRA.has(patternId)) return 'infra';
+  if (PACKAGE_BUG.has(patternId)) return 'package-bug';
+  return 'unknown';
+}
+
+/**
+ * One cause is one issue. A specific cause is the same defect whichever job of the router tripped
+ * on it, so the job and the step are not part of its identity: conclude, safe_outputs and reserve
+ * failing on one 403 are one finding, not three. A cause nobody could name is told apart by the
+ * step it failed in, because that is all there is to go on.
+ */
+function signatureOf({ workflow, step, conclusion, patternId }) {
+  if (patternId === 'unrecognised') return [workflow, step, conclusion, patternId].join('|');
+  return [workflow, conclusion, patternId].join('|');
+}
+
+/** The text the upstream search matches on, so a repeat lands on the issue that already exists. */
+function markerKey({ workflow, step, conclusion, patternId }) {
+  return signatureOf({ workflow, step, conclusion, patternId });
+}
+
+/**
+ * Which findings become issues, and why the others do not.
+ *  - infra causes are held: they are a rate, not a bug.
+ *  - a finding seen fewer than `minOccurrences` times in the window is held: once is a blip.
+ *  - at most `maxFindings` are filed per run, the most frequent first, so a bad night opens a
+ *    handful of issues and not forty.
+ */
+function select(findings, { minOccurrences = 2, maxFindings = 5 } = {}) {
+  const ranked = [...findings].sort((a, b) => b.count - a.count);
+  const file = [];
+  const held = [];
+  for (const finding of ranked) {
+    if (finding.severity === 'infra') {
+      held.push({ finding, reason: 'infrastructure cause, counted not filed' });
+    } else if (finding.count < minOccurrences) {
+      held.push({ finding, reason: `seen ${finding.count} time(s), under the ${minOccurrences} needed to file` });
+    } else if (file.length >= maxFindings) {
+      held.push({ finding, reason: `over the ${maxFindings} filed per run` });
+    } else {
+      file.push(finding);
+    }
+  }
+  return { file, held };
+}
+
+/**
+ * A signature closed recently stays quiet. Closing an issue is the answer to it; the report used
+ * to search open issues only, so every bulk close reopened the whole set the next morning.
+ */
+function isQuiet(closedAt, now, quietDays = 30) {
+  if (!closedAt) return false;
+  const closed = new Date(closedAt).getTime();
+  if (Number.isNaN(closed)) return false;
+  return now - closed < quietDays * 24 * 3600 * 1000;
+}
+
+/**
+ * The package version out of the ownership header. The character class used to swallow the
+ * full stop that ends the sentence, so every report said `1.2.3.`.
+ */
+function parseVersion(header) {
+  const match = /@plainconceptsplatform\/workflows@([0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.]*[0-9A-Za-z])?)/.exec(header ?? '');
+  return match ? match[1] : 'unknown';
+}
+
+module.exports = { INFRA, PACKAGE_BUG, severityOf, signatureOf, markerKey, select, isQuiet, parseVersion };