@plainconceptsplatform/workflows 1.6.11 → 1.6.16

package/loops/actions/housekeeping/action.yml

@@ -499,6 +499,21 @@ runs:
             }
           }
 
+          // The merge train wakes when a CI run finishes. When a gate it was waiting on ends without
+          // one, or is cancelled, nothing calls it again and every pull request sits until the next
+          // CI event: Pliny-Bot had four green pull requests waiting on that for the better part of an
+          // hour. The hourly sweep is the heartbeat. A dispatch of the router by the workflow token is
+          // allowed to start a run, and a tick with nothing to do costs one short job.
+          const openPulls = pulls.filter((pr) => !pr.draft);
+          if (openPulls.length > 0) {
+            await act(`dispatching a merge train tick: ${openPulls.length} pull request(s) are open`, () =>
+              github.rest.actions.createWorkflowDispatch({
+                ...context.repo, workflow_id: 'work-router.yml',
+                ref: context.payload.repository?.default_branch ?? 'main',
+                inputs: { operation: 'merge-train' },
+              }));
+          }
+
           await core.summary
             .addHeading('Housekeeping', 2)
             .addList([

package/loops/actions/merge-train/action.yml

@@ -32,6 +32,12 @@ inputs:
     description: How many pull requests may be in repair at once. Merges stay one at a time.
     required: false
     default: '1'
+  main-red-wait-hours:
+    description: >-
+      How long the train waits for a base branch whose checks fail the same way as a pull request's
+      before it stops waiting and repairs the pull request anyway.
+    required: false
+    default: '2'
   stuck-hours:
     description: How long the train waits behind a CI run that never finishes before it passes that pull request by.
     required: false
@@ -64,6 +70,7 @@ runs:
         MAX_ATTEMPTS: ${{ inputs.max-gate-attempts }}
         MAX_RETRIES: ${{ inputs.max-retries }}
         STUCK_HOURS: ${{ inputs.stuck-hours }}
+        MAIN_RED_WAIT_HOURS: ${{ inputs.main-red-wait-hours }}
         MAX_REPAIRS: ${{ inputs.max-repairs }}
         DRY_RUN: ${{ inputs.dry-run }}
         HINT_RUN_ID: ${{ inputs.ci-run-id }}

package/loops/actions/merge-train/merge-train.sh

@@ -146,6 +146,7 @@ MAX_ATTEMPTS="${MAX_ATTEMPTS:-6}"
 MAX_RETRIES="${MAX_RETRIES:-3}"
 MAX_REPAIRS="${MAX_REPAIRS:-1}"
 case "$MAX_REPAIRS" in ''|*[!0-9]*|0) MAX_REPAIRS=1 ;; esac
+MAIN_RED_WAIT_HOURS="${MAIN_RED_WAIT_HOURS:-2}"
 STUCK_HOURS="${STUCK_HOURS:-3}"
 DRY_RUN="${DRY_RUN:-false}"
 SELF_RUN_ID="${GITHUB_RUN_ID:-0}"
@@ -423,9 +424,22 @@ for line in "${candidates[@]}"; do
       --jq '[.check_runs[] | select(.conclusion == "failure") | (.name | sub("^.* / "; ""))] | unique | .[]' 2>/dev/null || true)
     failed_base=$(gh api "repos/$REPO/commits/$base_ref/check-runs?per_page=100" \
       --jq '[.check_runs[] | select(.conclusion == "failure") | (.name | sub("^.* / "; ""))] | unique | .[]' 2>/dev/null || true)
+    # How long the base has been red: the oldest failing check on its tip. The wait is bounded,
+    # because waiting for a base nobody fixes starves every pull request behind it. Pliny-Bot's
+    # main was red from 19:30 and the train waited until morning, ten hours, saying nothing.
+    red_since=$(gh api "repos/$REPO/commits/$base_ref/check-runs?per_page=100" \
+      --jq '[.check_runs[] | select(.conclusion == "failure") | .completed_at] | min // empty' 2>/dev/null || true)
+    red_for_hours=0
+    if [ -n "$red_since" ]; then
+      red_for_hours=$(( (now_epoch - $(iso_epoch "$red_since")) / 3600 ))
+    fi
     if [ -n "$failed_here" ] && [ -n "$failed_base" ] &&
       [ -z "$(comm -23 <(sort <<<"$failed_here") <(sort <<<"$failed_base"))" ]; then
-      same_as_main=true
+      if [ "$red_for_hours" -lt "${MAIN_RED_WAIT_HOURS:-2}" ]; then
+        same_as_main=true
+      else
+        echo "::warning::$base_ref has been red for ${red_for_hours}h on the same checks as #$pr; the train no longer waits for it and repairs #$pr."
+      fi
     fi
   fi
 
@@ -439,6 +453,20 @@ for line in "${candidates[@]}"; do
   move=$(sed -n 's/^move=//p' <<<"$decision")
   reason=$(sed -n 's/^reason=//p' <<<"$decision")
 
+  # A repair gate dispatched for red CI is moot once CI on the head is green: its job was to make
+  # that run pass. Left alone it keeps the pull request marked as having a gate, so the train
+  # skipped four green Pliny-Bot pull requests with "a gate is already on it" until it was cancelled
+  # by hand. Cancel it and let the pull request move.
+  if { [ "$move" = gate ] || [ "$move" = update ]; } && [ "$ci_status" = completed ] && [ "$ci_conclusion" = success ] &&
+    printf '%s\n' "$live_titles" | grep -qE "PR #${pr} \(repair\)$"; then
+    for run_id in $(gh api "repos/$REPO/actions/workflows/work-router.yml/runs?per_page=100" \
+      --jq ".workflow_runs[] | select(.status != \"completed\" and (.display_title | endswith(\"PR #${pr} (repair)\"))) | .id" 2>/dev/null); do
+      act "Cancelling the repair gate $run_id on #$pr: its CI is green." \
+        gh api -X POST "repos/$REPO/actions/runs/$run_id/cancel" --silent
+    done
+    live_titles=$(printf '%s\n' "$live_titles" | grep -vE "PR #${pr} \(repair\)$" || true)
+  fi
+
   # A gate is already on this pull request, whatever kind: offering it another would run the same
   # work twice. It does not hold the line up for the repair kind; a merging gate already does.
   if [ "$move" != skip ] && live_for_pr "$pr"; then

package/loops/actions/report-workflow-errors/action.yml

@@ -23,6 +23,20 @@ inputs:
     description: How far back to look for failures.
     required: false
     default: '24'
+  min-occurrences:
+    description: >-
+      A signature seen fewer times than this inside the window does not open a new issue. One
+      occurrence is a blip; an issue that is already open is always appended to.
+    required: false
+    default: '2'
+  max-findings:
+    description: The most issues one run opens, the most frequent first. The rest stay in the job summary.
+    required: false
+    default: '5'
+  quiet-days:
+    description: A signature whose issue was closed within this many days stays quiet, whatever the reason it was closed.
+    required: false
+    default: '30'
   dry-run:
     description: Compute and print the report, file nothing.
     required: false
@@ -43,6 +57,10 @@ runs:
         UPSTREAM_TOKEN: ${{ inputs.upstream-token }}
         UPSTREAM_REPO: ${{ inputs.upstream-repo }}
         LOOKBACK_HOURS: ${{ inputs.lookback-hours }}
+        MIN_OCCURRENCES: ${{ inputs.min-occurrences }}
+        MAX_FINDINGS: ${{ inputs.max-findings }}
+        QUIET_DAYS: ${{ inputs.quiet-days }}
+        POLICY_PATH: ${{ github.action_path }}/report-policy.cjs
         DRY_RUN: ${{ inputs.dry-run }}
       with:
         github-token: ${{ inputs.token }}
@@ -56,6 +74,10 @@ runs:
           // is checked against a leak scanner and the job fails rather than files if the
           // scanner trips.
 
+          // What to file and what to leave as a number lives in report-policy.cjs, apart from
+          // what this script reads and writes, so it can be run on its own.
+          const policy = require(process.env.POLICY_PATH);
+
           const dryRun = process.env.DRY_RUN === 'true';
           const upstreamToken = process.env.UPSTREAM_TOKEN;
           const [upstreamOwner, upstreamName] = process.env.UPSTREAM_REPO.split('/');
@@ -140,7 +162,7 @@ runs:
             const header = require('node:fs')
               .readFileSync('.github/workflows/work-router.yml', 'utf8')
               .split('\n')[0];
-            packageVersion = /@plainconceptsplatform\/workflows@([0-9][0-9A-Za-z.\-+]*)/.exec(header)?.[1] ?? 'unknown';
+            packageVersion = policy.parseVersion(header);
           } catch {
             core.info('could not read the installed package version from the router header');
           }
@@ -153,15 +175,19 @@ runs:
           // upstream, and every part of it is package-owned or an enumerated value.
           //
           // THE FINDING SHAPE IS THE PRIVACY BOUNDARY. bodyFor() renders a finding, so a field
-          // added here is a field sent to another repository. These eight are all
+          // added here is a field sent to another repository. These ten are all
           // package-owned names or enumerated values; nothing derived from a log, a branch, a
           // path or an issue belongs on this object. The route matrix asserts this exact list,
           // so adding a ninth field fails the build rather than leaking on the next cron.
-          const FINDING_FIELDS = ['workflow', 'job', 'step', 'conclusion', 'patternId', 'count', 'runnerLabels', 'attempts'];
+          const FINDING_FIELDS = ['workflow', 'job', 'step', 'conclusion', 'patternId', 'count', 'runnerLabels', 'attempts', 'severity', 'variants'];
           const findings = new Map();
           let ownedRuns = 0;
           let ownedFailures = 0;
           let skippedForeign = 0;
+          // Things that happened and are worth a number in the job summary, but that nobody can
+          // fix by editing a workflow, so they never open an issue: a cancelled job with no step
+          // that failed, a log that could not be read, and the fleet's own transient causes.
+          const quiet = { cancelledNoStep: 0, unreadable: 0, infra: new Map() };
           let queueWaitTotal = 0;
           let queueWaitCount = 0;
           let queueWaitWorst = 0;
@@ -180,15 +206,7 @@ runs:
             if (['queued', 'waiting', 'pending'].includes(run.status) &&
                 Date.now() - new Date(run.created_at).getTime() > STUCK_AFTER_MS) {
               stuckInQueue += 1;
-              const signature = `${file} · (whole run) · (never started) · queued · runner-never-assigned`;
-              const seen = findings.get(signature) ?? {
-                workflow: file, job: '(whole run)', step: '(never started)',
-                conclusion: 'queued', patternId: 'runner-never-assigned', count: 0,
-                runnerLabels: new Set(), attempts: new Set(),
-              };
-              seen.count += 1;
-              seen.attempts.add(run.run_attempt ?? 1);
-              findings.set(signature, seen);
+              quiet.infra.set('runner-never-assigned', (quiet.infra.get('runner-never-assigned') ?? 0) + 1);
               continue;
             }
 
@@ -217,7 +235,24 @@ runs:
               // package's own words. A `run-name` is not: it interpolates issue titles.
               const stepName = step?.name ?? '(no failing step recorded)';
 
+              // A cancelled job that never reached a failing step is somebody or something
+              // stopping work, not a defect: a superseded run, a job cancelled behind a failed
+              // one. It was 189 of 320 reports and every one said nothing.
+              if (job.conclusion === 'cancelled' && !step) { quiet.cancelledNoStep += 1; continue; }
+
+              // GitHub writes why a runner went away into the job's annotations ("The runner has
+              // received a shutdown signal", exit code 143), not into the log, so they are read
+              // with it. Only the id of a catalogue entry that matches ever leaves.
+              let annotationText = '';
+              try {
+                const notes = await github.paginate(github.rest.checks.listAnnotations, {
+                  ...context.repo, check_run_id: job.id, per_page: 50,
+                });
+                annotationText = notes.map((note) => note.message ?? '').join('\n');
+              } catch { /* annotations are an extra; the log can still classify */ }
+
               let patternId = 'unrecognised';
+              let logRead = true;
               try {
                 const { data: log } = await github.rest.actions.downloadJobLogsForWorkflowRun({
                   ...context.repo, job_id: job.id,
@@ -225,19 +260,32 @@ runs:
                 const text = typeof log === 'string' ? log : Buffer.from(log).toString('utf8');
                 // The tail is where the failure is, and reading less of the log is less to get
                 // wrong. Only the matched entry's id crosses into the report.
-                const tail = text.slice(-20000);
+                const tail = `${annotationText}\n${text.slice(-20000)}`;
                 patternId = CATALOGUE.find((entry) => entry.test.test(tail))?.id ?? 'unrecognised';
               } catch {
-                patternId = 'log-unavailable';
+                logRead = false;
+                patternId = CATALOGUE.find((entry) => entry.test.test(annotationText))?.id ?? 'log-unavailable';
+              }
+
+              // A log nobody could read says nothing, so it is counted and not filed.
+              if (patternId === 'log-unavailable') { quiet.unreadable += 1; continue; }
+              if (!logRead && policy.severityOf(patternId) === 'infra') {
+                quiet.infra.set(patternId, (quiet.infra.get(patternId) ?? 0) + 1);
+                continue;
               }
 
-              const signature = `${file} · ${job.name} · ${stepName} · ${job.conclusion} · ${patternId}`;
+              // One cause is one finding, whichever job tripped on it.
+              const signature = policy.signatureOf({
+                workflow: file, step: stepName, conclusion: job.conclusion, patternId,
+              });
               const seen = findings.get(signature) ?? {
                 workflow: file, job: job.name, step: stepName,
-                conclusion: job.conclusion, patternId, count: 0,
+                conclusion: job.conclusion, patternId: patternId, count: 0,
                 runnerLabels: new Set(), attempts: new Set(),
+                severity: policy.severityOf(patternId), variants: new Set(),
               };
               seen.count += 1;
+              seen.variants.add(`${job.name}|${stepName}`);
               seen.attempts.add(run.run_attempt ?? 1);
               // A pool label is the one value on a finding that the CONSUMER writes, and the
               // installer deliberately preserves theirs across updates (preserveRunnerPool in
@@ -261,6 +309,13 @@ runs:
           const ranked = [...findings.values()].sort((a, b) => b.count - a.count);
           const hours = process.env.LOOKBACK_HOURS;
 
+          // What becomes an issue. Infrastructure causes are a rate, a blip is not worth a
+          // person's time, and a bad night opens a handful of issues rather than forty.
+          const selection = policy.select(ranked, {
+            minOccurrences: Number(process.env.MIN_OCCURRENCES),
+            maxFindings: Number(process.env.MAX_FINDINGS),
+          });
+
           // Enforce the boundary at run time as well as in the build. A field that reached a
           // finding by some path the static check did not see stops the job here rather than
           // being rendered into an issue in another repository.
@@ -273,7 +328,7 @@ runs:
           }
 
           const bodyFor = (finding) => [
-            `<!-- pcp-error-report: ${finding.workflow}|${finding.job}|${finding.step}|${finding.patternId} -->`,
+            `<!-- pcp-error-report: ${policy.markerKey(finding)} -->`,
             '',
             'Filed automatically by the `report-workflow-errors` job in a repository that installs',
             'this package. It carries no information about that repository: only the names this',
@@ -286,6 +341,8 @@ runs:
             `| Failing step | \`${finding.step}\` |`,
             `| Conclusion | \`${finding.conclusion}\` |`,
             `| Classification | \`${finding.patternId}\` |`,
+            `| Severity | \`${finding.severity}\` |`,
+            `| Job and step combinations with this cause | ${finding.variants.size} |`,
             `| Runner labels | ${[...finding.runnerLabels].map((l) => `\`${l}\``).join(', ') || 'not reported'} |`,
             `| Occurrences in the last ${hours}h | ${finding.count} |`,
             `| Re-run attempts involved | ${[...finding.attempts].sort().join(', ')} |`,
@@ -298,8 +355,9 @@ runs:
             `- ${new Date().toISOString().slice(0, 10)}: ${finding.count} in ${hours}h, on \`${packageVersion}\``,
           ].join('\n');
 
-          const titleFor = (finding) =>
-            `${finding.workflow}: ${finding.patternId} in "${finding.step}"`;
+          const titleFor = (finding) => (finding.patternId === 'unrecognised'
+            ? `[${finding.severity}] ${finding.workflow}: ${finding.patternId} in "${finding.step}"`
+            : `[${finding.severity}] ${finding.workflow}: ${finding.patternId}`);
 
           // The scanner. A body that trips it is never filed, and the job goes red so the
           // pattern that produced it gets fixed rather than quietly dropped every night.
@@ -325,6 +383,10 @@ runs:
               'This report does not cover itself: a failure in this job shows up in this repository only.',
               `Runs still queued after 45 minutes, so no runner ever took them: ${stuckInQueue}`,
               `Distinct failure signatures: ${ranked.length}`,
+              `Filed or appended: ${selection.file.length}; held back: ${selection.held.length}`,
+              `Cancelled jobs with no failing step, not reported: ${quiet.cancelledNoStep}`,
+              `Jobs whose log could not be read, not reported: ${quiet.unreadable}`,
+              `Infrastructure causes, counted and not reported: ${[...quiet.infra].map(([id, n]) => `${id} ${n}`).join(', ') || 'none'}`,
               queueWaitCount > 0
                 ? `Runner queue wait: ${Math.round(queueWaitTotal / queueWaitCount)}s mean, ${queueWaitWorst}s worst`
                 : 'Runner queue wait: no jobs started in the window',
@@ -345,7 +407,7 @@ runs:
           // full picture rather than one night at a time.
           const reports = [];
           let leaked = 0;
-          for (const finding of ranked) {
+          for (const finding of selection.file) {
             const title = titleFor(finding);
             const body = bodyFor(finding);
             if (!scan(`${title}\n${body}`, `The report for ${finding.patternId}`)) { leaked += 1; continue; }
@@ -373,10 +435,17 @@ runs:
             let existing;
             try {
               const { data } = await upstream.rest.search.issuesAndPullRequests({
-                q: `repo:${upstreamOwner}/${upstreamName} is:issue is:open in:body "${marker}"`,
-                per_page: 5,
+                q: `repo:${upstreamOwner}/${upstreamName} is:issue in:body "${marker}"`,
+                per_page: 10,
               });
-              existing = data.items.find((item) => (item.body ?? '').includes(marker));
+              const matches = data.items.filter((item) => (item.body ?? '').includes(marker));
+              existing = matches.find((item) => item.state === 'open');
+              // Closing an issue is the answer to it. Without this every bulk close reopened the
+              // whole set the next morning: 150 of 320 reports re-filed a signature seen before.
+              if (!existing && matches.some((item) => policy.isQuiet(item.closed_at, Date.now(), Number(process.env.QUIET_DAYS)))) {
+                core.info(`left alone, an issue for it was closed within ${process.env.QUIET_DAYS} days: ${title}`);
+                continue;
+              }
             } catch (error) {
               core.warning(`could not search upstream for an existing report: HTTP ${error.status ?? '?'}`);
             }

package/loops/actions/report-workflow-errors/report-policy.cjs

@@ -0,0 +1,110 @@
+// Managed by @plainconceptsplatform/workflows. Source: loops/actions/report-workflow-errors/report-policy.cjs. Update with workflows update --force; consumer edits may be overwritten.
+//
+// What the error report decides, kept apart from what it reads and files so it can be run on its
+// own. Pure: no network, no clock, nothing read from the repository. Everything it sees is a
+// finding the report already built from package-owned names and enumerated values, so nothing
+// here can widen what leaves a private repository.
+//
+// The report used to open one issue per job, step and classification, for every failure it saw.
+// Read back over a month, 320 issues: 59% were cancelled jobs, 80% carried no diagnosis, 14% were
+// a runner shut down under a job, 47% re-filed a signature seen before, every one was closed as
+// not planned and none had a comment. What follows is the filter that would have kept about a
+// hundred of them out of that pile and left the rest as a count in the job summary.
+
+'use strict';
+
+// Causes that say something about the fleet, not about this package. They are counted in the job
+// summary, where a trend is visible, and never become an issue: nobody can fix a runner that was
+// taken away mid-job by editing a workflow. A job that hit its timeout is NOT here, because that
+// can be the package's own work taking too long.
+const INFRA = new Set([
+  'runner-never-assigned',
+  'runner-lost-mid-job',
+  'gh-api-rate-limit',
+  'model-quota',
+  'model-gateway-unreachable',
+  'out-of-disk',
+  'out-of-memory',
+]);
+
+// Causes that are a defect in something this package ships.
+const PACKAGE_BUG = new Set([
+  'jq-error',
+  'yaml-or-expression-invalid',
+  'lock-out-of-date',
+  'action-not-found',
+  'agent-produced-no-output',
+  'gh-api-403',
+  'gh-api-404',
+  'gh-api-409-conflict',
+]);
+
+/** infra, package-bug, or unknown (a cause nobody classified yet, worth a look when it repeats). */
+function severityOf(patternId) {
+  if (INFRA.has(patternId)) return 'infra';
+  if (PACKAGE_BUG.has(patternId)) return 'package-bug';
+  return 'unknown';
+}
+
+/**
+ * One cause is one issue. A specific cause is the same defect whichever job of the router tripped
+ * on it, so the job and the step are not part of its identity: conclude, safe_outputs and reserve
+ * failing on one 403 are one finding, not three. A cause nobody could name is told apart by the
+ * step it failed in, because that is all there is to go on.
+ */
+function signatureOf({ workflow, step, conclusion, patternId }) {
+  if (patternId === 'unrecognised') return [workflow, step, conclusion, patternId].join('|');
+  return [workflow, conclusion, patternId].join('|');
+}
+
+/** The text the upstream search matches on, so a repeat lands on the issue that already exists. */
+function markerKey({ workflow, step, conclusion, patternId }) {
+  return signatureOf({ workflow, step, conclusion, patternId });
+}
+
+/**
+ * Which findings become issues, and why the others do not.
+ *  - infra causes are held: they are a rate, not a bug.
+ *  - a finding seen fewer than `minOccurrences` times in the window is held: once is a blip.
+ *  - at most `maxFindings` are filed per run, the most frequent first, so a bad night opens a
+ *    handful of issues and not forty.
+ */
+function select(findings, { minOccurrences = 2, maxFindings = 5 } = {}) {
+  const ranked = [...findings].sort((a, b) => b.count - a.count);
+  const file = [];
+  const held = [];
+  for (const finding of ranked) {
+    if (finding.severity === 'infra') {
+      held.push({ finding, reason: 'infrastructure cause, counted not filed' });
+    } else if (finding.count < minOccurrences) {
+      held.push({ finding, reason: `seen ${finding.count} time(s), under the ${minOccurrences} needed to file` });
+    } else if (file.length >= maxFindings) {
+      held.push({ finding, reason: `over the ${maxFindings} filed per run` });
+    } else {
+      file.push(finding);
+    }
+  }
+  return { file, held };
+}
+
+/**
+ * A signature closed recently stays quiet. Closing an issue is the answer to it; the report used
+ * to search open issues only, so every bulk close reopened the whole set the next morning.
+ */
+function isQuiet(closedAt, now, quietDays = 30) {
+  if (!closedAt) return false;
+  const closed = new Date(closedAt).getTime();
+  if (Number.isNaN(closed)) return false;
+  return now - closed < quietDays * 24 * 3600 * 1000;
+}
+
+/**
+ * The package version out of the ownership header. The character class used to swallow the
+ * full stop that ends the sentence, so every report said `1.2.3.`.
+ */
+function parseVersion(header) {
+  const match = /@plainconceptsplatform\/workflows@([0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.]*[0-9A-Za-z])?)/.exec(header ?? '');
+  return match ? match[1] : 'unknown';
+}
+
+module.exports = { INFRA, PACKAGE_BUG, severityOf, signatureOf, markerKey, select, isQuiet, parseVersion };

package/loops/actions/verify-route-matrix/verify-route-matrix.sh

@@ -1788,6 +1788,24 @@ fi
 # leaving a verdict or an attempt is a loop: the train cannot tell a gate that declined from a gate
 # that has not run. Refusals are for what the train never hands over (a fork, a stranger, a pull
 # request that is not open), and the train filters those itself.
+# The shape of the tests reaches the agent only through these two prompts (check-diff measures size,
+# never shape), so losing the sentence silently brings the pyramid back.
+if worker_installed implement; then
+  if grep -qF 'Shape what you test like a trophy' "$IMPLEMENT_WORKER_MD"; then
+    PASS=$((PASS + 1))
+  else
+    FAIL=$((FAIL + 1))
+    echo "FAIL: the implement prompt must state the testing trophy shape" >&2
+  fi
+fi
+if worker_installed apply-review; then
+  if grep -qF 'put it at the integration layer' "${WORKFLOWS_DIR}/agent-apply-review.md"; then
+    PASS=$((PASS + 1))
+  else
+    FAIL=$((FAIL + 1))
+    echo "FAIL: the apply-review prompt must say new tests go at the integration layer" >&2
+  fi
+fi
 if worker_installed merge-gate; then
   # check-diff exiting 1 is the `fail` the code_rules job reports. Under GitHub's `bash -e` a bare
   # assignment from it ends the step, so the report is never recorded and the gate burns its
@@ -1984,6 +2002,11 @@ if worker_installed merge-gate; then
     assert "a failure that is main's too is waited out, not repaired" wait "$(move MERGEABLE=MERGEABLE CI_STATUS=completed CI_CONCLUSION=failure SAME_AS_MAIN=true)"
     assert "a failure that is not main's still goes to the gate" gate "$(move MERGEABLE=MERGEABLE CI_STATUS=completed CI_CONCLUSION=failure SAME_AS_MAIN=false)"
     assert "a conflict is still the gate's even when main is red" gate "$(move MERGEABLE=CONFLICTING CI_STATUS=completed CI_CONCLUSION=failure SAME_AS_MAIN=true)"
+    # The wait for a red base is bounded. Pliny-Bot's main was red from 19:30 and the train waited
+    # on it until morning, saying nothing, while five pull requests sat with their repairs unspent.
+    grep -q 'MAIN_RED_WAIT_HOURS' "$TRAIN_SH" && grep -q 'red_for_hours' "$TRAIN_SH" &&
+      grep -q 'main-red-wait-hours' "${HERE}/../merge-train/action.yml" ||
+      { FAIL=$((FAIL + 1)); echo "FAIL: the merge train waits on a red base branch with no time limit" >&2; }
     assert "green but behind is brought up to date first" update "$(move MERGEABLE=MERGEABLE CI_STATUS=completed CI_CONCLUSION=success BEHIND_BY=2)"
     assert "green and current goes to the gate" gate "$(move MERGEABLE=MERGEABLE CI_STATUS=completed CI_CONCLUSION=success BEHIND_BY=0)"
     assert "no CI run is passed by" skip "$(move MERGEABLE=MERGEABLE CI_STATUS=none)"
@@ -2253,6 +2276,55 @@ if [ -f "$ERROR_REPORT_YML" ]; then
   # above cannot see stops the job instead of being rendered upstream.
   er 'not in the reportable field list' 'does not check the finding shape at run time'
 
+  # 3b. What the report files and what it leaves as a number. Read back over a month, 320 issues:
+  # 59% were cancelled jobs, 80% carried no diagnosis, 47% re-filed a signature seen before, and
+  # every one was closed as not planned. The decision lives in report-policy.cjs so it can be run
+  # here, and the action has to actually use it.
+  POLICY_CJS="${HERE}/../report-workflow-errors/report-policy.cjs"
+  if [ -f "$POLICY_CJS" ]; then
+    policy_js='
+      const p = require(process.argv[1]);
+      const bad = [];
+      const ok = (cond, what) => { if (!cond) bad.push(what); };
+      ok(p.severityOf("runner-lost-mid-job") === "infra", "a runner lost mid-job is infrastructure");
+      ok(p.severityOf("jq-error") === "package-bug", "a jq error is a package bug");
+      ok(p.severityOf("unrecognised") === "unknown", "an unrecognised cause is unknown");
+      const a = { workflow: "w.yml", step: "one", conclusion: "failure", patternId: "gh-api-403" };
+      const b = { ...a, step: "two" };
+      ok(p.signatureOf(a) === p.signatureOf(b), "one specific cause is one signature whatever the step");
+      const u = { ...a, patternId: "unrecognised" };
+      ok(p.signatureOf(u) !== p.signatureOf({ ...u, step: "two" }), "an unrecognised cause is told apart by its step");
+      const f = (patternId, count, severity) => ({ patternId, count, severity });
+      const r = p.select([f("a", 5, "package-bug"), f("b", 9, "infra"), f("c", 1, "unknown"), f("d", 3, "unknown")], { minOccurrences: 2, maxFindings: 5 });
+      ok(r.file.map((x) => x.patternId).join() === "a,d", "infrastructure and one-off findings are held, the rest filed most frequent first");
+      ok(r.held.length === 2, "two findings held");
+      const many = Array.from({ length: 9 }, (_, i) => f("p" + i, 10 - i, "unknown"));
+      ok(p.select(many, { minOccurrences: 2, maxFindings: 5 }).file.length === 5, "a bad night files at most five");
+      const now = Date.parse("2026-10-17T00:00:00Z");
+      ok(p.isQuiet("2026-10-10T00:00:00Z", now, 30) === true, "an issue closed a week ago keeps its signature quiet");
+      ok(p.isQuiet("2026-08-01T00:00:00Z", now, 30) === false, "an issue closed long ago does not");
+      ok(p.isQuiet(null, now, 30) === false, "no closing date is not quiet");
+      ok(p.parseVersion("# Managed by @plainconceptsplatform/workflows@1.2.3. Source: x") === "1.2.3", "the version has no trailing full stop");
+      ok(p.parseVersion("nothing here") === "unknown", "no version reads as unknown");
+      if (bad.length) { console.error(bad.join("\n")); process.exit(1); }
+    '
+    if node -e "$policy_js" "$POLICY_CJS" 2>/tmp/policy-err; then
+      PASS=$((PASS + 1))
+    else
+      ER_OK=0
+      echo "FAIL: report-policy.cjs decides wrongly:" >&2
+      sed 's/^/  /' /tmp/policy-err >&2
+    fi
+  else
+    ER_OK=0
+    echo "FAIL: report-workflow-errors has no report-policy.cjs" >&2
+  fi
+  er 'policy\.select\(ranked' 'does not filter what it files through the policy'
+  er 'quiet\.cancelledNoStep \+= 1; continue;' 'files cancelled jobs that never reached a failing step'
+  er "patternId === 'log-unavailable'\) \{ quiet\.unreadable \+= 1; continue; \}" 'files jobs whose log could not be read'
+  er 'policy\.isQuiet\(' 'does not keep a recently closed signature quiet'
+  er 'listAnnotations' 'does not read the job annotations, where the runner shutdown is written'
+
   # 4. No model. A model asked to summarise a failure paraphrases whatever the log held, which
   # is the one thing that must not cross the boundary. This job stays deterministic.
   if grep -qiE '(engine:|opencode|safe-outputs|OPENAI_API_KEY)' "$ERROR_REPORT_YML"; then
@@ -2583,6 +2655,46 @@ if worker_installed refine; then
     PASS=$((PASS + 1))
   fi
 fi
+# The implement prompt used to say the safe-output tools are called as `safeoutputs/<tool>(...)`, an
+# MCP function call. The sandbox mounts them as a shell program on PATH instead, and the agent works
+# through an `execute` tool whose `tools` object does not hold them, so it searched there, found
+# nothing and ended Pliny-Bot #694 after seventy-eight minutes and sixteen commits with no pull
+# request. The prompt must show the shell form the refine prompt already uses.
+if worker_installed implement; then
+  IMPLEMENT_SO_MD="${WORKFLOWS_DIR}/agent-implement.md"
+  if grep -qF "safeoutputs create_pull_request ." "$IMPLEMENT_SO_MD"     && grep -qF "safeoutputs report_incomplete ." "$IMPLEMENT_SO_MD"     && ! grep -qF "safeoutputs/create_pull_request(" "$IMPLEMENT_SO_MD"; then
+    PASS=$((PASS + 1))
+  else
+    FAIL=$((FAIL + 1))
+    echo "FAIL: the implement prompt does not show the safe-output tools as shell commands on PATH; an agent that looks for them in the execute tool ends the run with nothing" >&2
+  fi
+fi
+# The implement run is not interactive, so a turn that ends in text ends the run. Pliny-Bot #750 moved
+# `pnpm verify` to the background, said it would be resumed with the result and stopped: eight commits,
+# no pull request. The prompt must forbid backgrounding and waiting, and name the remedy.
+if worker_installed implement; then
+  IMPLEMENT_FG_MD="${WORKFLOWS_DIR}/agent-implement.md"
+  if grep -qF "Run every command in the foreground and wait for it in the same turn" "$IMPLEMENT_FG_MD"     && grep -qF "never end a turn to wait for it" "$IMPLEMENT_FG_MD"     && grep -qF "timeout" "$IMPLEMENT_FG_MD"; then
+    PASS=$((PASS + 1))
+  else
+    FAIL=$((FAIL + 1))
+    echo "FAIL: the implement prompt does not forbid running commands in the background and ending the turn to wait for them" >&2
+  fi
+fi
+# The merge train wakes on a finished CI run only. Pliny-Bot had four green pull requests wait an hour
+# because a repair gate dispatched for red CI kept them marked as having a gate, and nothing called the
+# train again once it was cancelled. The hourly sweep is now a heartbeat, and a repair gate whose CI has
+# turned green is cancelled by the train itself.
+HK_YML="${HERE}/../housekeeping/action.yml"
+TRAIN_SH_HB="${HERE}/../merge-train/merge-train.sh"
+if [ -f "$HK_YML" ] && [ -f "$TRAIN_SH_HB" ]; then
+  if grep -qF "operation: 'merge-train'" "$HK_YML"     && grep -qF "createWorkflowDispatch" "$HK_YML"     && grep -qF "its CI is green" "$TRAIN_SH_HB"     && grep -qF 'live_titles=$(printf' "$TRAIN_SH_HB"; then
+    PASS=$((PASS + 1))
+  else
+    FAIL=$((FAIL + 1))
+    echo "FAIL: the merge train has no heartbeat from housekeeping, or does not cancel a repair gate whose CI turned green" >&2
+  fi
+fi
 # The implement prompt tells the agent it is already on its branch and to skip the pc-plan-goal
 # branching phase. That is only true if a step before the agent made the branch, and nothing in the
 # sandbox can do it later: no git credentials, and the prompt forbids the commands. Without the

package/loops/docs/diagrams.md

@@ -709,7 +709,7 @@ the repository it runs in. Every consumer is private, so that is the whole desig
 
 What crosses the boundary is a fixed, enumerable set of facts about workflows **this package ships**:
 their file names, their job and step names, a conclusion, a runner label, a count, and the id of a
-matched entry from a catalogue of sixteen known failure shapes. What never crosses it is free text
+matched entry from a catalogue of sixteen known failure shapes, plus a severity. What never crosses it is free text
 of any kind no log line, no branch name, no issue title, no path, no URL, no number that could be
 looked up. Hence no model: a model asked to summarise a failure paraphrases whatever the log held.
 
@@ -720,8 +720,12 @@ flowchart TB
     erOwned -.->|"no: the name could describe<br/>a product or a customer"| erCount("Counted, never inspected")
     erOwned -->|yes| erState{"Failed, or queued<br/>past 45 minutes?"}
     erState -.->|neither| erCount
-    erState -->|yes| erClassify("Read the log tail,<br/>match the catalogue,<br/>keep only the matched id")
-    erClassify --> erBody("Build the report from<br/>package-owned names,<br/>a conclusion and a count")
+    erState -->|yes| erClassify("Read the log tail and the job annotations,<br/>match the catalogue,<br/>keep only the matched id")
+    erClassify --> erNoise{"Cancelled with no failing step,<br/>log unreadable, or an<br/>infrastructure cause?"}
+    erNoise -.->|yes| erCount
+    erNoise -->|no| erFilter{"Seen at least twice,<br/>within the cap, and not<br/>closed in the last 30 days?"}
+    erFilter -.->|no| erCount
+    erFilter -->|yes| erBody("Build the report from<br/>package-owned names,<br/>a conclusion and a count")
     erBody --> erScan{"Leak scanner:<br/>repo or owner name, a URL,<br/>an email, a path, a token,<br/>an issue number, a ref?"}
     erScan -.->|"anything matches"| erWithhold(("Withheld<br/>nothing filed, run goes red"))
     erScan -->|clean| erToken{"Upstream token<br/>available?"}
@@ -737,7 +741,7 @@ flowchart TB
     classDef idle fill:#f4f4f6,stroke:#5b5b66,stroke-width:2px,color:#2b2b33
     class erCron start
     class erRuns,erClassify,erBody,erFile,erCount action
-    class erOwned,erState,erScan,erToken decision
+    class erOwned,erState,erNoise,erFilter,erScan,erToken decision
     class erWithhold failure
     class erDone success
     class erSummary idle

package/loops/templates/agentics/agentics-error-report.yml

@@ -69,6 +69,8 @@ jobs:
       # upstream with a separate token and writes nothing here.
       contents: read
       actions: read
+      # The annotations GitHub writes on a job say why a runner went away; the log does not.
+      checks: read
     steps:
       - name: Checkout workflow actions
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

package/loops/workflows/agent-apply-review.md

@@ -545,6 +545,8 @@ timeout-minutes: 90
    `node .opencode/scripts/check-diff.mjs`.
    It must exit 0: delete the comments and tests it lists. The merge gate runs it too and does not
    merge while it fails.
+   When you have to add a test, put it at the integration layer through the public surface, not as a
+   unit test of internals; a bug fix gets one regression test.
 
 7. Select one review outcome.
 

package/loops/workflows/agent-implement.md

@@ -781,12 +781,37 @@ timeout-minutes: 180
 
      Never open a pull request that does not pass them.
 
+     **Run every command in the foreground and wait for it in the same turn. Never move a command
+     to the background and never end a turn to wait for it.** This run is not interactive: nothing
+     resumes you. A turn that ends with text and no tool call ends the whole run, and the work is
+     discarded. Pliny-Bot #750 started `pnpm verify` in the background, wrote "I'll be resumed with
+     its result" and stopped: the run ended green with eight commits and no pull request, and #694
+     lost seventy-eight minutes the same way. The bash tool cuts a command at 120 seconds unless you
+     pass a `timeout` (milliseconds, up to 600000). Give a long command that timeout, and when it is
+     still too long, run it in parts (lint, typecheck, web tests, web build, dotnet build, dotnet test)
+     so every part finishes inside one call. The only valid way to end the run is the safe-output
+     call in step 6.
+
+     **Two things the unit tests do not catch, and CI will.** When you add a dependency, take the
+     latest released version and run the repository's audit for it (`pnpm audit --audit-level high`
+     for the web, `dotnet list package --vulnerable` for the backend): Pliny-Bot #752 added a PDF
+     library at a version with two critical and eight high advisories and the dependency scan
+     blocked it. When you register a new service, interface or handler, run at least one test that
+     boots the whole application, not only unit tests: Pliny-Bot #753 registered an interface for
+     one engine only, the application could not start under the others and every integration test
+     failed.
+
      When `.opencode/scripts/check-diff.mjs` exists, also run
      `node .opencode/scripts/check-diff.mjs`
      on both paths. It must exit 0. It holds the comments and tests you add to the size of the
      change and makes you clean the files you touched; fix every line it lists. The merge gate
      runs it again and does not merge while it fails.
 
+     Shape what you test like a trophy, not a pyramid: put new tests at the integration or
+     acceptance layer, through the public surface of the code you changed, and write a plain
+     unit test only for logic with real branching. A bug fix gets one regression test. Do not
+     write tests of constants, configured defaults, mocks of your own code or markup attributes.
+
      Anything you change to satisfy `check-diff` is a change: run the verification commands
      again afterwards, and open the pull request only from a tree where both pass. Do not delete
      a test that covers a branch you added to fit the budget; the complexity gate in CI measures
@@ -798,16 +823,37 @@ timeout-minutes: 180
    code already written.
 
 6. Finish by calling **exactly one** safe-output tool. A run that calls none is a wasted run:
-   the workflow reports a failure and everything you just did is discarded. All safe-output
-   tools are on the `safeoutputs` MCP server, called as `safeoutputs/<tool>` , for example:
+   the workflow reports a failure and everything you just did is discarded. Pliny-Bot #694 spent
+   seventy-eight minutes and sixteen commits on a finished change and ended with nothing: the
+   agent searched for the tool in the `tools` object of the `execute` tool, did not find it
+   there, and concluded it did not exist.
+
+   **The safe-output tools are shell commands on your PATH, not entries in `tools`.** The sandbox
+   mounts the `safeoutputs` server as a command-line program. It is not under `tools.*`, `search`
+   does not return it, and an empty result from either says nothing about it. Run it with the
+   bash tool, and run `command -v safeoutputs` there first if you are unsure it is present.
+
+   **Pipe the JSON on stdin, put every parameter inside the JSON, and pass a lone `.` and no
+   flags.** The bridge reads the payload from stdin only when the command line is the tool name
+   followed by `.`. A `--flag` on the line makes it skip stdin and send just the flags, drop the
+   body and still report success. Correct:
+
+   `printf '%s' "$JSON" | safeoutputs create_pull_request .` with
+   `{"title":"[bot] Fix X","body":"Closes #${{ inputs.issue-number }}\n\n...","branch":"${{ env.IMPLEMENT_BRANCH_PREFIX }}${{ inputs.issue-number }}"}`
+
+   `printf '%s' "$JSON" | safeoutputs report_incomplete .` with `{"reason":"..."}`
+
+   `printf '%s' "$JSON" | safeoutputs noop .` with `{"message":"..."}`
+
+   Wrong (the body is silently empty): `safeoutputs create_pull_request --title "..." .`
 
-   ```
-   safeoutputs/create_pull_request(title="[bot] Fix X", body="Closes #${{ inputs.issue-number }}\n\n...", branch="${{ env.IMPLEMENT_BRANCH_PREFIX }}${{ inputs.issue-number }}")
-   ```
+   If the call fails, read the error and send the whole payload again once. If it fails twice,
+   call `report_incomplete` the same way with the reason; never end the run with finished work
+   and no call.
 
    Choose exactly one:
 
-   - **`safeoutputs/create_pull_request`** , the normal path. Propose a pull request with the
+   - **`create_pull_request`** , the normal path. Propose a pull request with the
      verified changes. Do not pass `base`: the workflow sets it to the branch this run started
      on, which is the branch you are working from, so a stack child lands on its feature branch
      without you doing anything. The `body` **must close exactly this issue and nothing else**
@@ -815,10 +861,10 @@ timeout-minutes: 180
      the branch you are on, the one named in step 1; do not invent another name. You do not
      need to check whether a pull request already exists for this issue: the router does
      that before dispatching you and does not start this workflow when one does.
-   - **`safeoutputs/report_incomplete`** , only when infrastructure or tooling prevents you
+   - **`report_incomplete`** , only when infrastructure or tooling prevents you
      from completing the task, such as a pre-existing build failure you cannot fix. Provide a
      specific `reason`.
-   - **`safeoutputs/noop`** , only when the issue context shows the work is already done and no
+   - **`noop`** , only when the issue context shows the work is already done and no
      changes are needed. Provide a `message` explaining what you found.
 
    Do not manage labels or post comments , the conclude job handles that.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@plainconceptsplatform/workflows",
-  "version": "1.6.11",
+  "version": "1.6.16",
   "description": "Install and update Platform GitHub agentic workflows.",
   "keywords": [
     "github-actions",