@norskvideo/ctl-dev-kit 0.2.100 → 0.2.101

package/conventions/bot-prs.yml

@@ -35,7 +35,15 @@
 #     actually read;
 #   - checks or a tier still running -> leave it for the next trigger; a tier
 #     that has not started an hour after checks finished never will (its
-#     workflow is broken, or does not run on PRs), so that is red too.
+#     workflow is broken, or does not run on PRs), so that is red too;
+#   - a tier integration.yml cancelled because the host was too contended to
+#     judge it (its job carries a `ctl-contention-uncertain` annotation) -> not
+#     yet judged, so neither red nor merged: re-run once a backoff has passed
+#     (vars.BOT_PRS_CONTENDED_BACKOFF_MINUTES, default 30, doubling per
+#     attempt), up to vars.BOT_PRS_CONTENDED_ATTEMPTS attempts (default 4);
+#     after that it is red as "still contended after N attempts: needs a human
+#     or a quieter runner", never as a failure. Any other cancelled check is
+#     red, and a cancelled tier is re-run up to its third attempt, as before.
 # The cell goes green again once nothing is red.
 #
 # Why the conflict rebuild: when the three bots fired within minutes of each
@@ -111,6 +119,14 @@ jobs:
           WRITE_TOKEN: ${{ secrets.CI_DISPATCH_TOKEN }}
           REPO: ${{ github.repository }}
           BASE: ${{ inputs.base || 'main' }}
+          # A run integration.yml cancelled because the host was too contended
+          # to judge it carries this annotation title on its job. Such a run is
+          # not yet judged: re-run with backoff, up to the attempt budget.
+          # Repo variables, so the verbatim copy needs no edit; unset or not a
+          # number takes the default (4 attempts, 30 minutes doubling).
+          CONTENDED_MARK: ctl-contention-uncertain
+          CONTENDED_ATTEMPTS: ${{ vars.BOT_PRS_CONTENDED_ATTEMPTS }}
+          CONTENDED_BACKOFF_MINUTES: ${{ vars.BOT_PRS_CONTENDED_BACKOFF_MINUTES }}
         run: |
           set -euo pipefail
           api="https://api.github.com/repos/$REPO"
@@ -181,9 +197,17 @@ jobs:
           # guides queue on one concurrency group per repo and GitHub keeps one
           # pending run per group, cancelling the older, so three bot PRs opening
           # within the hour can cancel a guides run with nothing wrong in it.
+          # CONTENDED (optional): the run ids contended_runs found. A cancelled
+          # tier among them was never judged: it is re-run once the backoff
+          # since its cancel has passed (CONTENDED_BACKOFF_MINUTES, doubling per
+          # attempt), up to CONTENDED_ATTEMPTS attempts, then reported as still
+          # contended. It never reads ok.
           tier_verdict() {
-            local required="$1" runs="$2" now="$3" f run status conclusion checks_age
+            local required="$1" runs="$2" now="$3" contended="${4:-}" f run id status conclusion checks_age
+            local attempt max="${CONTENDED_ATTEMPTS:-}" base="${CONTENDED_BACKOFF_MINUTES:-}" backoff age
             local -a waiting=() failed=() rerun=()
+            [[ "$max" =~ ^[1-9][0-9]*$ ]] || max=4
+            [[ "$base" =~ ^[0-9]+$ ]] || base=30
             checks_age="$(jq -r --argjson now "$now" '
               [.workflow_runs[] | select((.path | split("@")[0]) == ".github/workflows/checks.yml"
                 and .event == "pull_request" and .status == "completed")] | max_by(.id)
@@ -202,8 +226,22 @@ jobs:
               fi
               status="$(jq -r .status <<<"$run")"
               conclusion="$(jq -r '.conclusion // ""' <<<"$run")"
+              id="$(jq -r .id <<<"$run")"
+              attempt="$(jq -r '.run_attempt // 1' <<<"$run")"
               if [ "$status" != "completed" ]; then
                 waiting+=("${f%.yml} $status")
+              elif [ "$conclusion" = "cancelled" ] && grep -qxF "$id" <<<"$contended"; then
+                if [ "$attempt" -ge "$max" ]; then
+                  failed+=("${f%.yml} still contended after $attempt attempts: needs a human or a quieter runner")
+                  continue
+                fi
+                backoff=$((base * 60 * (1 << (attempt - 1))))
+                age=$((now - $(jq -r '.updated_at | fromdateiso8601' <<<"$run")))
+                if [ "$age" -ge "$backoff" ]; then
+                  rerun+=("$id")
+                else
+                  waiting+=("${f%.yml} not yet judged, the host was contended (attempt $attempt of $max); re-running in $(((backoff - age + 59) / 60))m")
+                fi
               elif [ "$conclusion" = "cancelled" ] && [ "$(jq -r '.run_attempt // 1' <<<"$run")" -lt 3 ]; then
                 rerun+=("$(jq -r .id <<<"$run")")
               elif [ "$conclusion" != "success" ]; then
@@ -221,6 +259,41 @@ jobs:
             fi
           }
 
+          # contended_runs CHECK_RUNS -> the workflow run ids (one per line) that
+          # integration.yml cancelled because the host was too contended to
+          # judge them: a CANCELLED job of the run carries the CONTENDED_MARK
+          # annotation. The marker on a job that failed (its cancel never
+          # landed) is not one: that run is red. Annotations it cannot read
+          # leave the run red, as every cancel was before.
+          contended_runs() {
+            local checks="$1" id notes
+            while IFS= read -r id; do
+              [ -n "$id" ] || continue
+              notes="$(gh_api "$api/check-runs/$id/annotations?per_page=100")"
+              if ! jq -e 'type == "array"' >/dev/null 2>&1 <<<"$notes"; then
+                echo "::warning::could not read the annotations of check run $id; a cancel there counts as red" >&2
+                continue
+              fi
+              jq -e --arg m "$CONTENDED_MARK" 'any(.[]; .title == $m)' >/dev/null <<<"$notes" || continue
+              jq -r --argjson id "$id" '.check_runs[] | select(.id == $id)
+                | (.details_url // "") | (capture("/actions/runs/(?<r>[0-9]+)").r // empty)' <<<"$checks"
+            done < <(jq -r '.check_runs[] | select(.conclusion == "cancelled" and (.output.annotations_count // 0) > 0) | .id' <<<"$checks") | sort -u
+          }
+
+          # failed_checks CHECK_RUNS CONTENDED -> the names of the checks that
+          # failed, comma separated. A cancelled job of a contended run (every
+          # one of them: GitHub cancels the run's other jobs with it) is left to
+          # tier_verdict, which re-runs it; any other cancel is red, as before.
+          failed_checks() {
+            jq -r --arg c "$2" '($c | split("\n") | map(select(. != ""))) as $contended
+              | [.check_runs[]
+                 | select(.conclusion | IN("failure","cancelled","timed_out","action_required","startup_failure"))
+                 | select(.conclusion != "cancelled"
+                     or (((.details_url // "") | (capture("/actions/runs/(?<r>[0-9]+)").r // "")) as $r
+                         | any($contended[]; . == $r) | not))
+                 | .name] | unique | join(", ")' <<<"$1"
+          }
+
           # The workflow that rebuilds each bot branch: bump.yml, for the retired
           # bots' branches too (their workflows went when bump.yml arrived, and
           # bump.yml supersedes their PRs for every component it moves).
@@ -276,7 +349,8 @@ jobs:
             fi
             runs="$(gh_api "$api/commits/$sha/check-runs?per_page=100")"
             expect "reading checks for #$n" '.check_runs | type == "array"' "$runs"
-            failed="$(jq -r '[.check_runs[] | select(.conclusion | IN("failure","cancelled","timed_out","action_required","startup_failure")) | .name] | unique | join(", ")' <<<"$runs")"
+            contended="$(contended_runs "$runs")"
+            failed="$(failed_checks "$runs" "$contended")"
             pending="$(jq -r '[.check_runs[] | select(.status != "completed")] | length' <<<"$runs")"
             passed="$(jq -r '[.check_runs[] | select(.conclusion == "success")] | length' <<<"$runs")"
             if [ -n "$failed" ]; then red+=("#$n $ref: failing $failed"); continue; fi
@@ -286,14 +360,18 @@ jobs:
             required="$(tiers_for "$sha")"
             wf_runs="$(gh_api "$api/actions/runs?head_sha=$sha&per_page=100")"
             expect "reading workflow runs for #$n" '.workflow_runs | type == "array"' "$wf_runs"
-            verdict="$(tier_verdict "$required" "$wf_runs" "$(date +%s)")"
+            verdict="$(tier_verdict "$required" "$wf_runs" "$(date +%s)" "$contended")"
             case "$verdict" in
               ok) ;;
               red\ *) red+=("#$n $ref: ${verdict#red }"); continue ;;
               rerun\ *)
                 for id in ${verdict#rerun }; do
                   code="$(gh_actions -o /dev/null -w '%{http_code}' -X POST "$api/actions/runs/$id/rerun")"
-                  echo "#$n $ref: a gating tier (run $id) was cancelled; re-running it (HTTP $code)"
+                  if grep -qxF "$id" <<<"$contended"; then
+                    echo "#$n $ref: a gating tier (run $id) was cancelled as uncertain, the host was contended; re-running it (HTTP $code)"
+                  else
+                    echo "#$n $ref: a gating tier (run $id) was cancelled; re-running it (HTTP $code)"
+                  fi
                 done
                 continue ;;
               *) echo "#$n $ref: ${verdict#wait }; next time"; continue ;;

package/conventions/integration-testing.md

@@ -140,9 +140,18 @@ the file present, the canonical workflow runs `bun run test:integration` through
   reporter flag.
 - When the suite fails and **every** failure overlapped contention, the run is
   **uncertain**: the job writes a `::warning::` and a job summary (the verdict,
-  each failure with its numbers, pressure per minute), then cancels its own run.
+  each failure with its numbers, pressure per minute), plus a
+  `::notice title=ctl-contention-uncertain::` marker, then cancels its own run.
   A cancelled run is not reported to the CI board, and bot-prs merges only a
   success. One failure on a quiet stretch keeps the run red, as before.
+- On a bot PR, bot-prs reads that marker off the cancelled job's annotations
+  and treats the run as **not yet judged**: it re-runs it once a backoff has
+  passed since the cancel (30 minutes, doubling per attempt), up to 4 attempts,
+  then reports it as "still contended after N attempts: needs a human or a
+  quieter runner", distinct from a failure. Repo variables
+  `BOT_PRS_CONTENDED_ATTEMPTS` and `BOT_PRS_CONTENDED_BACKOFF_MINUTES` override
+  the budget. A run cancelled for any other reason (a person, a concurrency
+  group) is treated exactly as before.
 - If the host is starved early in the suite, or for a long stretch anywhere, the
   suite is stopped there rather than run for 40 minutes to an uncertain end.
 - An uncertain attempt is rerun in the same job (`rerun.max`, default 1), after
@@ -186,10 +195,9 @@ real failure; 75 (EX_TEMPFAIL) when uncertain, which the workflow believes only
 alongside a `verdict.json` that says so.
 
 **Caveats.** A `ctl-candidate` dispatch that is cancelled is still scored red by
-release-collect (the gate is advisory). bot-prs reports a PR whose integration
-check was cancelled as needing a human rather than re-running it, because its
-check-run filter counts `cancelled` before the tier rerun logic sees it; the
-in-job rerun is what retries.
+release-collect (the gate is advisory). bot-prs re-runs an uncertain cancel
+only while the annotation is readable with the job token (`checks: read`); if
+it cannot read it, the cancel counts as red, as every cancel did before.
 
 ## Reaching an image product's control plane
 

package/conventions/integration.yml

@@ -518,12 +518,17 @@ jobs:
           # host was starved. Believed only with the verdict file that says so,
           # since a suite could exit 75 for its own reasons. Not a failure:
           # the last step of this job cancels the run instead, so it reads
-          # cancelled (not red) on the board and bot-prs does not merge it.
+          # cancelled (not red) on the board, and bot-prs re-runs it later
+          # rather than merging it or reporting it red.
           # Every other status, a real failure on a quiet box included, exits
-          # exactly as it did.
+          # exactly as it did. The %2C is a comma: workflow-command properties
+          # are comma separated, so a bare one cut the title short.
           verdict="$NORSK_CONTENTION_DIR/verdict.json"
           if [ "$rc" -eq 75 ] && [ "$(jq -r .status "$verdict" 2>/dev/null)" = uncertain ]; then
-            echo "::warning title=integration uncertain, the host was contended::$(jq -r .reason "$verdict"). This run will be cancelled rather than failed; the pressure timeline and per-test classification are in the job summary and the integration artifact."
+            echo "::warning title=integration uncertain%2C the host was contended::$(jq -r .reason "$verdict"). This run will be cancelled rather than failed; the pressure timeline and per-test classification are in the job summary and the integration artifact."
+            # The marker bot-prs reads off this job's annotations to tell this
+            # cancel from any other (bot-prs-contended.test.ts).
+            echo "::notice title=ctl-contention-uncertain::cancelled as uncertain, not failed: bot-prs re-runs it once the host may be quieter"
             echo "NORSK_CONTENTION_UNCERTAIN=1" >> "$GITHUB_ENV"
             exit 0
           fi
@@ -586,7 +591,8 @@ jobs:
       # step flagged this run uncertain: its failures were the starved host's,
       # not the product's, so it is cancelled rather than failed. A cancelled
       # run is not reported to the CI board (notify is !cancelled()), and
-      # bot-prs merges only a success, so a bump cannot land on it.
+      # bot-prs merges only a success, so a bump cannot land on it (it re-runs
+      # a run whose job carries the suite step's ctl-contention-uncertain notice).
       # Cancellation is asynchronous: the step holds the job until it lands,
       # and if it is refused or never lands the job FAILS, because an uncertain
       # run must never finish reading green. contention-wired.test.ts.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@norskvideo/ctl-dev-kit",
-  "version": "0.2.100",
+  "version": "0.2.101",
   "type": "module",
   "exports": {
     "./create-product": "./create-product/create-product.ts",