@norskvideo/ctl-dev-kit 0.2.99 → 0.2.101

package/conventions/bot-prs.yml

@@ -35,7 +35,15 @@
 #     actually read;
 #   - checks or a tier still running -> leave it for the next trigger; a tier
 #     that has not started an hour after checks finished never will (its
-#     workflow is broken, or does not run on PRs), so that is red too.
+#     workflow is broken, or does not run on PRs), so that is red too;
+#   - a tier integration.yml cancelled because the host was too contended to
+#     judge it (its job carries a `ctl-contention-uncertain` annotation) -> not
+#     yet judged, so neither red nor merged: re-run once a backoff has passed
+#     (vars.BOT_PRS_CONTENDED_BACKOFF_MINUTES, default 30, doubling per
+#     attempt), up to vars.BOT_PRS_CONTENDED_ATTEMPTS attempts (default 4);
+#     after that it is red as "still contended after N attempts: needs a human
+#     or a quieter runner", never as a failure. Any other cancelled check is
+#     red, and a cancelled tier is re-run up to its third attempt, as before.
 # The cell goes green again once nothing is red.
 #
 # Why the conflict rebuild: when the three bots fired within minutes of each
@@ -111,6 +119,14 @@ jobs:
           WRITE_TOKEN: ${{ secrets.CI_DISPATCH_TOKEN }}
           REPO: ${{ github.repository }}
           BASE: ${{ inputs.base || 'main' }}
+          # A run integration.yml cancelled because the host was too contended
+          # to judge it carries this annotation title on its job. Such a run is
+          # not yet judged: re-run with backoff, up to the attempt budget.
+          # Repo variables, so the verbatim copy needs no edit; unset or not a
+          # number takes the default (4 attempts, 30 minutes doubling).
+          CONTENDED_MARK: ctl-contention-uncertain
+          CONTENDED_ATTEMPTS: ${{ vars.BOT_PRS_CONTENDED_ATTEMPTS }}
+          CONTENDED_BACKOFF_MINUTES: ${{ vars.BOT_PRS_CONTENDED_BACKOFF_MINUTES }}
         run: |
           set -euo pipefail
           api="https://api.github.com/repos/$REPO"
@@ -181,9 +197,17 @@ jobs:
           # guides queue on one concurrency group per repo and GitHub keeps one
           # pending run per group, cancelling the older, so three bot PRs opening
           # within the hour can cancel a guides run with nothing wrong in it.
+          # CONTENDED (optional): the run ids contended_runs found. A cancelled
+          # tier among them was never judged: it is re-run once the backoff
+          # since its cancel has passed (CONTENDED_BACKOFF_MINUTES, doubling per
+          # attempt), up to CONTENDED_ATTEMPTS attempts, then reported as still
+          # contended. It never reads ok.
           tier_verdict() {
-            local required="$1" runs="$2" now="$3" f run status conclusion checks_age
+            local required="$1" runs="$2" now="$3" contended="${4:-}" f run id status conclusion checks_age
+            local attempt max="${CONTENDED_ATTEMPTS:-}" base="${CONTENDED_BACKOFF_MINUTES:-}" backoff age
             local -a waiting=() failed=() rerun=()
+            [[ "$max" =~ ^[1-9][0-9]*$ ]] || max=4
+            [[ "$base" =~ ^[0-9]+$ ]] || base=30
             checks_age="$(jq -r --argjson now "$now" '
               [.workflow_runs[] | select((.path | split("@")[0]) == ".github/workflows/checks.yml"
                 and .event == "pull_request" and .status == "completed")] | max_by(.id)
@@ -202,8 +226,22 @@ jobs:
               fi
               status="$(jq -r .status <<<"$run")"
               conclusion="$(jq -r '.conclusion // ""' <<<"$run")"
+              id="$(jq -r .id <<<"$run")"
+              attempt="$(jq -r '.run_attempt // 1' <<<"$run")"
               if [ "$status" != "completed" ]; then
                 waiting+=("${f%.yml} $status")
+              elif [ "$conclusion" = "cancelled" ] && grep -qxF "$id" <<<"$contended"; then
+                if [ "$attempt" -ge "$max" ]; then
+                  failed+=("${f%.yml} still contended after $attempt attempts: needs a human or a quieter runner")
+                  continue
+                fi
+                backoff=$((base * 60 * (1 << (attempt - 1))))
+                age=$((now - $(jq -r '.updated_at | fromdateiso8601' <<<"$run")))
+                if [ "$age" -ge "$backoff" ]; then
+                  rerun+=("$id")
+                else
+                  waiting+=("${f%.yml} not yet judged, the host was contended (attempt $attempt of $max); re-running in $(((backoff - age + 59) / 60))m")
+                fi
               elif [ "$conclusion" = "cancelled" ] && [ "$(jq -r '.run_attempt // 1' <<<"$run")" -lt 3 ]; then
                 rerun+=("$(jq -r .id <<<"$run")")
               elif [ "$conclusion" != "success" ]; then
@@ -221,6 +259,41 @@ jobs:
             fi
           }
 
+          # contended_runs CHECK_RUNS -> the workflow run ids (one per line) that
+          # integration.yml cancelled because the host was too contended to
+          # judge them: a CANCELLED job of the run carries the CONTENDED_MARK
+          # annotation. The marker on a job that failed (its cancel never
+          # landed) is not one: that run is red. Annotations it cannot read
+          # leave the run red, as every cancel was before.
+          contended_runs() {
+            local checks="$1" id notes
+            while IFS= read -r id; do
+              [ -n "$id" ] || continue
+              notes="$(gh_api "$api/check-runs/$id/annotations?per_page=100")"
+              if ! jq -e 'type == "array"' >/dev/null 2>&1 <<<"$notes"; then
+                echo "::warning::could not read the annotations of check run $id; a cancel there counts as red" >&2
+                continue
+              fi
+              jq -e --arg m "$CONTENDED_MARK" 'any(.[]; .title == $m)' >/dev/null <<<"$notes" || continue
+              jq -r --argjson id "$id" '.check_runs[] | select(.id == $id)
+                | (.details_url // "") | (capture("/actions/runs/(?<r>[0-9]+)").r // empty)' <<<"$checks"
+            done < <(jq -r '.check_runs[] | select(.conclusion == "cancelled" and (.output.annotations_count // 0) > 0) | .id' <<<"$checks") | sort -u
+          }
+
+          # failed_checks CHECK_RUNS CONTENDED -> the names of the checks that
+          # failed, comma separated. A cancelled job of a contended run (every
+          # one of them: GitHub cancels the run's other jobs with it) is left to
+          # tier_verdict, which re-runs it; any other cancel is red, as before.
+          failed_checks() {
+            jq -r --arg c "$2" '($c | split("\n") | map(select(. != ""))) as $contended
+              | [.check_runs[]
+                 | select(.conclusion | IN("failure","cancelled","timed_out","action_required","startup_failure"))
+                 | select(.conclusion != "cancelled"
+                     or (((.details_url // "") | (capture("/actions/runs/(?<r>[0-9]+)").r // "")) as $r
+                         | any($contended[]; . == $r) | not))
+                 | .name] | unique | join(", ")' <<<"$1"
+          }
+
           # The workflow that rebuilds each bot branch: bump.yml, for the retired
           # bots' branches too (their workflows went when bump.yml arrived, and
           # bump.yml supersedes their PRs for every component it moves).
@@ -276,7 +349,8 @@ jobs:
             fi
             runs="$(gh_api "$api/commits/$sha/check-runs?per_page=100")"
             expect "reading checks for #$n" '.check_runs | type == "array"' "$runs"
-            failed="$(jq -r '[.check_runs[] | select(.conclusion | IN("failure","cancelled","timed_out","action_required","startup_failure")) | .name] | unique | join(", ")' <<<"$runs")"
+            contended="$(contended_runs "$runs")"
+            failed="$(failed_checks "$runs" "$contended")"
             pending="$(jq -r '[.check_runs[] | select(.status != "completed")] | length' <<<"$runs")"
             passed="$(jq -r '[.check_runs[] | select(.conclusion == "success")] | length' <<<"$runs")"
             if [ -n "$failed" ]; then red+=("#$n $ref: failing $failed"); continue; fi
@@ -286,14 +360,18 @@ jobs:
             required="$(tiers_for "$sha")"
             wf_runs="$(gh_api "$api/actions/runs?head_sha=$sha&per_page=100")"
             expect "reading workflow runs for #$n" '.workflow_runs | type == "array"' "$wf_runs"
-            verdict="$(tier_verdict "$required" "$wf_runs" "$(date +%s)")"
+            verdict="$(tier_verdict "$required" "$wf_runs" "$(date +%s)" "$contended")"
             case "$verdict" in
               ok) ;;
               red\ *) red+=("#$n $ref: ${verdict#red }"); continue ;;
               rerun\ *)
                 for id in ${verdict#rerun }; do
                   code="$(gh_actions -o /dev/null -w '%{http_code}' -X POST "$api/actions/runs/$id/rerun")"
-                  echo "#$n $ref: a gating tier (run $id) was cancelled; re-running it (HTTP $code)"
+                  if grep -qxF "$id" <<<"$contended"; then
+                    echo "#$n $ref: a gating tier (run $id) was cancelled as uncertain, the host was contended; re-running it (HTTP $code)"
+                  else
+                    echo "#$n $ref: a gating tier (run $id) was cancelled; re-running it (HTTP $code)"
+                  fi
                 done
                 continue ;;
               *) echo "#$n $ref: ${verdict#wait }; next time"; continue ;;

package/conventions/contention-suite.sh

@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+# ctl-dev-kit convention: the integration suite under contention verdicts.
+#
+# integration.yml runs this, inside the dev shell and from the repo root, in
+# place of `bun run test:integration` when the repo has opted in with
+# scripts/integration/contention.json (thresholds; `{}` takes every default).
+# It runs the suite under @norskvideo/ctl-test-harness's ctl-contention, which
+# samples host pressure (Linux PSI) alongside it and classifies the result:
+#
+#   exit 0     the suite passed
+#   exit N     a real failure: the suite's own status, as before
+#   exit 75    uncertain: every failure overlapped host contention, or the
+#              suite was stopped early because the host was starved
+#
+# An uncertain attempt is rerun up to rerun.max times (default 1), once the
+# host has gone quiet or rerun.maxWaitSec has passed. Each attempt's timeline,
+# verdict and summary land in $NORSK_CONTENTION_DIR/attempt-<n>/, the last
+# verdict at $NORSK_CONTENTION_DIR/verdict.json for the workflow to read.
+#
+# Anything missing (a harness without ctl-contention, a config it cannot
+# read) runs the suite exactly as before, with a warning: opting in must
+# never be what turns a tier red.
+set -uo pipefail
+
+config=scripts/integration/contention.json
+bin=node_modules/.bin/ctl-contention
+dir="${NORSK_CONTENTION_DIR:-test-results/contention}"
+
+as_before() {
+  echo "::warning title=contention verdicts off::$1; the suite runs without them, exactly as before."
+  exec bun run test:integration
+}
+
+[ -x "$bin" ] || as_before "$config is present but this repo's @norskvideo/ctl-test-harness has no ctl-contention (bump it)"
+if ! max="$("$bin" config rerun.max --config "$config")"; then
+  as_before "ctl-contention could not read $config"
+fi
+
+# Between attempts: the containers the last attempt launched. Its teardown may
+# never have run (an abort kills the suite), and a held port or a leftover
+# engine would starve or break the rerun. The same two rules the job-end reap
+# steps use: instances the harness recorded, and anything mounting this job's
+# test-temp.
+reap_leftovers() {
+  command -v docker >/dev/null 2>&1 || return 0
+  local m="${NORSK_LAUNCHED_MANIFEST:-}" base="${NORSK_CTL_TEST_TMP:-}" id ids c src
+  if [ -n "$m" ] && [ -f "$m" ]; then
+    while IFS= read -r id; do
+      [ -n "$id" ] || continue
+      ids="$(docker ps -aq --filter "label=norsk-ctl.instance=$id" 2>/dev/null || true)"
+      [ -z "$ids" ] || docker rm -f $ids >/dev/null 2>&1 || true
+    done < <(sort -u "$m")
+  fi
+  [ -n "$base" ] || return 0
+  for c in $(docker ps -aq 2>/dev/null || true); do
+    src="$(docker inspect -f '{{range .Mounts}}{{println .Source}}{{end}}' "$c" 2>/dev/null)" || continue
+    if grep -q "^$base/" <<<"$src"; then docker rm -f "$c" >/dev/null 2>&1 || true; fi
+  done
+}
+
+attempt=1
+while :; do
+  out="$dir/attempt-$attempt"
+  "$bin" run --config "$config" --out "$out" -- bun run test:integration
+  rc=$?
+  [ -f "$out/verdict.json" ] && cp "$out/verdict.json" "$dir/verdict.json"
+  if [ -n "${GITHUB_STEP_SUMMARY:-}" ] && [ -f "$out/summary.md" ]; then
+    { echo "#### Attempt $attempt"; echo; cat "$out/summary.md"; echo; } >>"$GITHUB_STEP_SUMMARY"
+  fi
+  if [ "$rc" -ne 75 ] || [ "$attempt" -gt "$max" ]; then break; fi
+  echo "::warning title=integration attempt $attempt uncertain::the host was contended; waiting for it to go quiet, then rerunning (attempt $((attempt + 1)) of $((max + 1)))"
+  reap_leftovers
+  "$bin" wait-quiet --config "$config" || true
+  attempt=$((attempt + 1))
+done
+exit "$rc"

package/conventions/integration-testing.md

@@ -116,6 +116,89 @@ runner without the mount runs unserialised and says so in the log; a suite that
 has waited an hour runs anyway with a warning. A hand-owned `integration.yml`
 does not take the lock until it converges onto the canonical.
 
+### Contention verdicts: a starved host is not a red
+
+The lock stops two suites starving each other, but not everything else on the
+host. On test2, commentary has failed with browser main threads starved for
+6-80% of a test window, the engine 50 s behind realtime, a 0.55 s switch taking
+5-20 s, and jobs killed mid-run by SIGTERM. Those failures are the host's, not
+the product's. A product can opt in to having the tier say so.
+
+**Opting in.** Add `scripts/integration/contention.json` (and a
+`@norskvideo/ctl-test-harness` new enough to ship `ctl-contention`). `{}` takes
+every default; any key below overrides one, and a misspelt key is an error. With
+the file present, the canonical workflow runs `bun run test:integration` through
+`conventions/contention-suite.sh`, which runs it under the harness's
+`ctl-contention run`:
+
+- A sampler reads host pressure once a second for the whole suite: Linux PSI
+  (`/proc/pressure/{cpu,memory,io}`, host-wide, falling back to the cgroup v2
+  root's `*.pressure`), the load average and `MemAvailable`. The timeline is
+  `pressure.jsonl` in the job's `integration-attempt-N` artifact.
+- Each failing test's window is read off bun's own `(fail) ... [duration]` line
+  as it streams past, so the product's `test:integration` script needs no
+  reporter flag.
+- When the suite fails and **every** failure overlapped contention, the run is
+  **uncertain**: the job writes a `::warning::` and a job summary (the verdict,
+  each failure with its numbers, pressure per minute), plus a
+  `::notice title=ctl-contention-uncertain::` marker, then cancels its own run.
+  A cancelled run is not reported to the CI board, and bot-prs merges only a
+  success. One failure on a quiet stretch keeps the run red, as before.
+- On a bot PR, bot-prs reads that marker off the cancelled job's annotations
+  and treats the run as **not yet judged**: it re-runs it once a backoff has
+  passed since the cancel (30 minutes, doubling per attempt), up to 4 attempts,
+  then reports it as "still contended after N attempts: needs a human or a
+  quieter runner", distinct from a failure. Repo variables
+  `BOT_PRS_CONTENDED_ATTEMPTS` and `BOT_PRS_CONTENDED_BACKOFF_MINUTES` override
+  the budget. A run cancelled for any other reason (a person, a concurrency
+  group) is treated exactly as before.
+- If the host is starved early in the suite, or for a long stretch anywhere, the
+  suite is stopped there rather than run for 40 minutes to an uncertain end.
+- An uncertain attempt is rerun in the same job (`rerun.max`, default 1), after
+  waiting up to `rerun.maxWaitSec` for the host to go quiet.
+
+**When nothing changes.** No `contention.json`: the suite runs exactly as it did.
+No readable PSI (an old kernel, `psi=0`, a runner that cannot read it): the
+summary says "pressure unavailable" and every verdict is the suite's own. A
+harness without `ctl-contention`, or a config it cannot read: a warning, then the
+suite runs as before. Every job prints a "Probe whether this runner can read
+host pressure (PSI)" step whether or not the repo has opted in, so a runner's
+first log answers whether the signal exists there.
+
+**Defaults** (`classify`, judged on the failure's window widened by `marginSec`):
+
+| Key                        | Default | Why                                                                                                                    |
+| -------------------------- | ------- | ---------------------------------------------------------------------------------------------------------------------- |
+| `classify.cpuSomeMean`     | 0.25    | cpu `some` bounds any one thread's wait from above; passing runs saw at most ~5% browser starvation, so 25% is 5x that |
+| `classify.cpuSomePeak`     | 0.60    | over any `peakWindowSec` (10 s): a stall long enough to break a seconds-scale assertion that a long test's mean hides  |
+| `classify.memSomeMean`     | 0.10    | reclaim as a constant cost of running                                                                                  |
+| `classify.memFullPeak`     | 0.05    | half a second in any 10 in which nothing ran at all                                                                    |
+| `classify.minCoverage`     | 0.5     | a window the sampler barely saw is not judged (and so stays red)                                                       |
+| `classify.killWindowSec`   | 120     | a suite killed from outside is judged on its last two minutes                                                          |
+| `abort.cpuSome`            | 0.70    | mean over the rule's window; with `memSome` 0.30 and `memFull` 0.10                                                    |
+| `abort.earlyWindowSec`     | 60      | starved for a minute within the first `earlyWithinSec` (300): stop                                                     |
+| `abort.sustainedWindowSec` | 300     | starved for five minutes anywhere: stop                                                                                |
+| `rerun.max`                | 1       | reruns after an uncertain attempt; 0 turns them off                                                                    |
+
+Calling a real failure uncertain costs a rerun; calling a starved one red costs a
+human an investigation of nothing. The bars are set where the first is unlikely,
+and they are config so test2's recorded timelines can move them.
+
+**From a test.** `hostContendedDuring({ fromMs, toMs })` from
+`@norskvideo/ctl-test-harness/contention` answers for any window of the running
+suite (it reads the timeline the wrapper names in `NORSK_CONTENTION_TIMELINE`),
+with the numbers. Outside the wrapper it answers "unavailable", never
+"contended".
+
+**Exit statuses** of `ctl-contention run`: 0 pass; the suite's own status on a
+real failure; 75 (EX_TEMPFAIL) when uncertain, which the workflow believes only
+alongside a `verdict.json` that says so.
+
+**Caveats.** A `ctl-candidate` dispatch that is cancelled is still scored red by
+release-collect (the gate is advisory). bot-prs re-runs an uncertain cancel
+only while the annotation is readable with the job token (`checks: read`); if
+it cannot read it, the cancel counts as red, as every cancel did before.
+
 ## Reaching an image product's control plane
 
 `product add --image` loopback-publishes the control plane on a
@@ -255,7 +338,9 @@ NON-PRODUCTION` warning is expected in CI, not an error. The media engine's
 (`conventions/integration.yml`) and drift-gated like `smoke.yml`, with the same
 two per-repo lines: the `product:` dashboard key and the matrix `runner:` label
 array. Everything else a repo used to hand-edit into the workflow now lives in
-three optional hooks the workflow calls if they are executable:
+three optional hooks the workflow calls if they are executable (and one opt-in
+file, `scripts/integration/contention.json`, described under "Contention
+verdicts" below):
 
 | Hook                                   | When                              | What goes there                                                                           |
 | -------------------------------------- | --------------------------------- | ----------------------------------------------------------------------------------------- |

package/conventions/integration.yml

@@ -17,7 +17,7 @@
 # green, and ctl's (advisory) release gate, whose ctl-candidate dispatch runs it
 # and shows the result on the CI board.
 #
-# Per-repo variation lives in three optional hooks, not in this file:
+# Per-repo variation lives in four optional hooks, not in this file:
 #   scripts/integration/prepull-extra.sh  extra images the suite launches (a
 #                                         driver, a sink, a sidecar)
 #   scripts/integration/prepare.sh        built before the suite runs (a
@@ -26,6 +26,11 @@
 #   scripts/integration/verify.sh         after the suite; absent, a product
 #                                         with tests/demo.spec.ts gets
 #                                         `bun run demo -- check`
+#   scripts/integration/contention.json   opts the suite into contention verdicts
+#                                         (thresholds; {} takes the defaults): a
+#                                         failure the starved HOST caused cancels
+#                                         the run instead of failing it. See
+#                                         contention-suite.sh and the steps below.
 #
 # The suite is piped into tee so its log survives as an artifact, and the step
 # sets pipefail FIRST: the default `bash -e {0}` has none, so without it the
@@ -102,6 +107,11 @@ jobs:
         include:
           - runner: '["x64"]'
     runs-on: ${{ fromJSON(matrix.runner) }}
+    # actions: write is for one call only: the last step cancelling this run
+    # when the host was too contended to judge it.
+    permissions:
+      contents: read
+      actions: write
     steps:
       # Runs BEFORE checkout, and does two things: points the harness's temp base
       # out of the workspace for every later step in this job, and clears whatever
@@ -192,6 +202,38 @@ jobs:
             echo "NORSK_RUNNER_OWNS_DOCKER=true" >> "$GITHUB_ENV"
           fi
 
+      # Contention verdicts (scripts/integration/contention.json) read host
+      # pressure from Linux PSI, and whether a runner can is a property of its
+      # kernel (CONFIG_PSI, the psi= boot flag) and of its container (cgroup v2,
+      # what is masked). This says which, on every run and whether or not the
+      # repo has opted in, so the log answers "does the signal exist on this
+      # host?" before anything depends on it. It only reports.
+      - name: Probe whether this runner can read host pressure (PSI)
+        run: |
+          set -uo pipefail
+          echo "host=$(hostname) kernel=$(uname -r) cpus=$(nproc) of $(grep -c ^processor /proc/cpuinfo 2>/dev/null || echo '?')"
+          echo "loadavg: $(cat /proc/loadavg 2>/dev/null || echo unreadable)"
+          grep -E '^(MemTotal|MemAvailable):' /proc/meminfo 2>/dev/null || true
+          readable=""
+          for f in /proc/pressure/cpu /proc/pressure/memory /proc/pressure/io /sys/fs/cgroup/cpu.pressure /sys/fs/cgroup/memory.pressure; do
+            if out="$(cat "$f" 2>&1)" && grep -q '^some avg10=' <<<"$out"; then
+              echo "$f: readable"
+              sed 's/^/  /' <<<"$out"
+              readable="$readable $f"
+            else
+              echo "$f: NOT readable (${out:-empty})"
+            fi
+          done
+          if [ -f /sys/fs/cgroup/cgroup.controllers ]; then echo "cgroup v2 at /sys/fs/cgroup; this process: $(cat /proc/self/cgroup 2>/dev/null)"; else echo "no cgroup v2 at /sys/fs/cgroup"; fi
+          case "$readable" in
+            *"/proc/pressure/cpu"*"/proc/pressure/memory"*) verdict="host-wide PSI readable (/proc/pressure)" ;;
+            *"/sys/fs/cgroup/cpu.pressure"*"/sys/fs/cgroup/memory.pressure"*) verdict="only this cgroup's PSI readable: blind to the engine containers, which are host siblings" ;;
+            *) verdict="PSI unavailable: contention verdicts would change nothing here" ;;
+          esac
+          if [ -f scripts/integration/contention.json ]; then optin="on (scripts/integration/contention.json)"; else optin="off (no scripts/integration/contention.json)"; fi
+          echo "PSI on $(hostname): $verdict; contention verdicts $optin"
+          echo "PSI on \`$(hostname)\`: $verdict; contention verdicts $optin" >> "$GITHUB_STEP_SUMMARY"
+
       # Integration needs this host's docker to be able to create a network: the
       # harness makes norsk-net and a network per instance. On 2026-10-02
       # ashton-nuc's filter table had lost DOCKER-ISOLATION-STAGE-2, so every
@@ -409,6 +451,9 @@ jobs:
           # and the first artifact built that way carried ten of another product's
           # instances alongside its own three. Same shape the test-temp base uses.
           NORSK_CI_LOG_DIR: ${{ runner.temp }}/norsk-logs-${{ github.run_id }}-${{ github.run_attempt }}
+          # Where contention-suite.sh puts each attempt's pressure timeline,
+          # verdict and summary, keyed like the log dir for the same reason.
+          NORSK_CONTENTION_DIR: ${{ runner.temp }}/contention-${{ github.run_id }}-${{ github.run_attempt }}
         run: |
           set -euo pipefail
           suite() {
@@ -424,7 +469,11 @@ jobs:
                 echo "running scripts/integration/prepare.sh"
                 ./scripts/integration/prepare.sh
               fi
-              bun run test:integration
+              if [ -f scripts/integration/contention.json ]; then
+                bash node_modules/@norskvideo/ctl-dev-kit/conventions/contention-suite.sh
+              else
+                bun run test:integration
+              fi
               if [ -x scripts/integration/verify.sh ]; then
                 echo "running scripts/integration/verify.sh"
                 ./scripts/integration/verify.sh
@@ -460,7 +509,30 @@ jobs:
           else
             echo "no writable /srv/ci-scratch on this runner: the suite runs without the host lock"
           fi
+          set +e
           suite 9>&- 2>&1 | tee "$RUNNER_TEMP/integration.log"
+          rc=$?
+          set -e
+          # 75 is contention-suite.sh saying "uncertain": every failure
+          # overlapped host contention, or the suite was stopped because the
+          # host was starved. Believed only with the verdict file that says so,
+          # since a suite could exit 75 for its own reasons. Not a failure:
+          # the last step of this job cancels the run instead, so it reads
+          # cancelled (not red) on the board, and bot-prs re-runs it later
+          # rather than merging it or reporting it red.
+          # Every other status, a real failure on a quiet box included, exits
+          # exactly as it did. The %2C is a comma: workflow-command properties
+          # are comma separated, so a bare one cut the title short.
+          verdict="$NORSK_CONTENTION_DIR/verdict.json"
+          if [ "$rc" -eq 75 ] && [ "$(jq -r .status "$verdict" 2>/dev/null)" = uncertain ]; then
+            echo "::warning title=integration uncertain%2C the host was contended::$(jq -r .reason "$verdict"). This run will be cancelled rather than failed; the pressure timeline and per-test classification are in the job summary and the integration artifact."
+            # The marker bot-prs reads off this job's annotations to tell this
+            # cancel from any other (bot-prs-contended.test.ts).
+            echo "::notice title=ctl-contention-uncertain::cancelled as uncertain, not failed: bot-prs re-runs it once the host may be quieter"
+            echo "NORSK_CONTENTION_UNCERTAIN=1" >> "$GITHUB_ENV"
+            exit 0
+          fi
+          exit "$rc"
 
       # The log AND the harness evidence. Playwright and the harness write
       # screenshots, traces and captured container logs under test-results/;
@@ -475,6 +547,7 @@ jobs:
             ${{ runner.temp }}/integration.log
             test-results/
             ${{ runner.temp }}/norsk-logs-${{ github.run_id }}-${{ github.run_attempt }}
+            ${{ runner.temp }}/contention-${{ github.run_id }}-${{ github.run_attempt }}
           retention-days: 7
           if-no-files-found: warn
 
@@ -514,6 +587,31 @@ jobs:
             docker rm -f $s
           fi
 
+      # Last, so the evidence is uploaded and the host reaped first. The suite
+      # step flagged this run uncertain: its failures were the starved host's,
+      # not the product's, so it is cancelled rather than failed. A cancelled
+      # run is not reported to the CI board (notify is !cancelled()), and
+      # bot-prs merges only a success, so a bump cannot land on it (it re-runs
+      # a run whose job carries the suite step's ctl-contention-uncertain notice).
+      # Cancellation is asynchronous: the step holds the job until it lands,
+      # and if it is refused or never lands the job FAILS, because an uncertain
+      # run must never finish reading green. contention-wired.test.ts.
+      - name: Cancel this run, the host was too contended to judge it
+        if: ${{ always() && env.NORSK_CONTENTION_UNCERTAIN == '1' }}
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          set -uo pipefail
+          code="$(curl -sS -o /dev/null -w '%{http_code}' -X POST \
+            -H "Authorization: Bearer $GH_TOKEN" \
+            -H "Accept: application/vnd.github+json" \
+            "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/cancel")"
+          [ "$code" = "202" ] || { echo "::error::could not cancel this uncertain run (HTTP $code), so it fails instead: read the contention summary before treating it as a product failure"; exit 1; }
+          echo "cancel accepted; waiting for it to land"
+          sleep 300
+          echo "::error::still running five minutes after the cancel was accepted, so it fails instead: read the contention summary before treating it as a product failure"
+          exit 1
+
   # Report this pipeline's result to the aggregated product CI dashboard
   # (id3as/ci-workflows). !cancelled() so a real pass/fail reports, but a
   # cancelled/superseded run does NOT — a cancellation dispatched as failure

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@norskvideo/ctl-dev-kit",
-  "version": "0.2.99",
+  "version": "0.2.101",
   "type": "module",
   "exports": {
     "./create-product": "./create-product/create-product.ts",