@norskvideo/ctl-dev-kit 0.2.99 → 0.2.100

package/conventions/contention-suite.sh

@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+# ctl-dev-kit convention: the integration suite under contention verdicts.
+#
+# integration.yml runs this, inside the dev shell and from the repo root, in
+# place of `bun run test:integration` when the repo has opted in with
+# scripts/integration/contention.json (thresholds; `{}` takes every default).
+# It runs the suite under @norskvideo/ctl-test-harness's ctl-contention, which
+# samples host pressure (Linux PSI) alongside it and classifies the result:
+#
+#   exit 0     the suite passed
+#   exit N     a real failure: the suite's own status, as before
+#   exit 75    uncertain: every failure overlapped host contention, or the
+#              suite was stopped early because the host was starved
+#
+# An uncertain attempt is rerun up to rerun.max times (default 1), once the
+# host has gone quiet or rerun.maxWaitSec has passed. Each attempt's timeline,
+# verdict and summary land in $NORSK_CONTENTION_DIR/attempt-<n>/, the last
+# verdict at $NORSK_CONTENTION_DIR/verdict.json for the workflow to read.
+#
+# Anything missing (a harness without ctl-contention, a config it cannot
+# read) runs the suite exactly as before, with a warning: opting in must
+# never be what turns a tier red.
+set -uo pipefail
+
+config=scripts/integration/contention.json
+bin=node_modules/.bin/ctl-contention
+dir="${NORSK_CONTENTION_DIR:-test-results/contention}"
+
+as_before() {
+  echo "::warning title=contention verdicts off::$1; the suite runs without them, exactly as before."
+  exec bun run test:integration
+}
+
+[ -x "$bin" ] || as_before "$config is present but this repo's @norskvideo/ctl-test-harness has no ctl-contention (bump it)"
+if ! max="$("$bin" config rerun.max --config "$config")"; then
+  as_before "ctl-contention could not read $config"
+fi
+
+# Between attempts: the containers the last attempt launched. Its teardown may
+# never have run (an abort kills the suite), and a held port or a leftover
+# engine would starve or break the rerun. The same two rules the job-end reap
+# steps use: instances the harness recorded, and anything mounting this job's
+# test-temp.
+reap_leftovers() {
+  command -v docker >/dev/null 2>&1 || return 0
+  local m="${NORSK_LAUNCHED_MANIFEST:-}" base="${NORSK_CTL_TEST_TMP:-}" id ids c src
+  if [ -n "$m" ] && [ -f "$m" ]; then
+    while IFS= read -r id; do
+      [ -n "$id" ] || continue
+      ids="$(docker ps -aq --filter "label=norsk-ctl.instance=$id" 2>/dev/null || true)"
+      [ -z "$ids" ] || docker rm -f $ids >/dev/null 2>&1 || true
+    done < <(sort -u "$m")
+  fi
+  [ -n "$base" ] || return 0
+  for c in $(docker ps -aq 2>/dev/null || true); do
+    src="$(docker inspect -f '{{range .Mounts}}{{println .Source}}{{end}}' "$c" 2>/dev/null)" || continue
+    if grep -q "^$base/" <<<"$src"; then docker rm -f "$c" >/dev/null 2>&1 || true; fi
+  done
+}
+
+attempt=1
+while :; do
+  out="$dir/attempt-$attempt"
+  "$bin" run --config "$config" --out "$out" -- bun run test:integration
+  rc=$?
+  [ -f "$out/verdict.json" ] && cp "$out/verdict.json" "$dir/verdict.json"
+  if [ -n "${GITHUB_STEP_SUMMARY:-}" ] && [ -f "$out/summary.md" ]; then
+    { echo "#### Attempt $attempt"; echo; cat "$out/summary.md"; echo; } >>"$GITHUB_STEP_SUMMARY"
+  fi
+  if [ "$rc" -ne 75 ] || [ "$attempt" -gt "$max" ]; then break; fi
+  echo "::warning title=integration attempt $attempt uncertain::the host was contended; waiting for it to go quiet, then rerunning (attempt $((attempt + 1)) of $((max + 1)))"
+  reap_leftovers
+  "$bin" wait-quiet --config "$config" || true
+  attempt=$((attempt + 1))
+done
+exit "$rc"

package/conventions/integration-testing.md

@@ -116,6 +116,81 @@ runner without the mount runs unserialised and says so in the log; a suite that
 has waited an hour runs anyway with a warning. A hand-owned `integration.yml`
 does not take the lock until it converges onto the canonical.
 
+### Contention verdicts: a starved host is not a red
+
+The lock stops two suites starving each other, but not everything else on the
+host. On test2, commentary has failed with browser main threads starved for
+6-80% of a test window, the engine 50 s behind realtime, a 0.55 s switch taking
+5-20 s, and jobs killed mid-run by SIGTERM. Those failures are the host's, not
+the product's. A product can opt in to having the tier say so.
+
+**Opting in.** Add `scripts/integration/contention.json` (and a
+`@norskvideo/ctl-test-harness` new enough to ship `ctl-contention`). `{}` takes
+every default; any key below overrides one, and a misspelt key is an error. With
+the file present, the canonical workflow runs `bun run test:integration` through
+`conventions/contention-suite.sh`, which runs it under the harness's
+`ctl-contention run`:
+
+- A sampler reads host pressure once a second for the whole suite: Linux PSI
+  (`/proc/pressure/{cpu,memory,io}`, host-wide, falling back to the cgroup v2
+  root's `*.pressure`), the load average and `MemAvailable`. The timeline is
+  `pressure.jsonl` in the job's `integration-attempt-N` artifact.
+- Each failing test's window is read off bun's own `(fail) ... [duration]` line
+  as it streams past, so the product's `test:integration` script needs no
+  reporter flag.
+- When the suite fails and **every** failure overlapped contention, the run is
+  **uncertain**: the job writes a `::warning::` and a job summary (the verdict,
+  each failure with its numbers, pressure per minute), then cancels its own run.
+  A cancelled run is not reported to the CI board, and bot-prs merges only a
+  success. One failure on a quiet stretch keeps the run red, as before.
+- If the host is starved early in the suite, or for a long stretch anywhere, the
+  suite is stopped there rather than run for 40 minutes to an uncertain end.
+- An uncertain attempt is rerun in the same job (`rerun.max`, default 1), after
+  waiting up to `rerun.maxWaitSec` for the host to go quiet.
+
+**When nothing changes.** No `contention.json`: the suite runs exactly as it did.
+No readable PSI (an old kernel, `psi=0`, a runner that cannot read it): the
+summary says "pressure unavailable" and every verdict is the suite's own. A
+harness without `ctl-contention`, or a config it cannot read: a warning, then the
+suite runs as before. Every job prints a "Probe whether this runner can read
+host pressure (PSI)" step whether or not the repo has opted in, so a runner's
+first log answers whether the signal exists there.
+
+**Defaults** (`classify`, judged on the failure's window widened by `marginSec`):
+
+| Key                        | Default | Why                                                                                                                    |
+| -------------------------- | ------- | ---------------------------------------------------------------------------------------------------------------------- |
+| `classify.cpuSomeMean`     | 0.25    | cpu `some` bounds any one thread's wait from above; passing runs saw at most ~5% browser starvation, so 25% is 5x that |
+| `classify.cpuSomePeak`     | 0.60    | over any `peakWindowSec` (10 s): a stall long enough to break a seconds-scale assertion that a long test's mean hides  |
+| `classify.memSomeMean`     | 0.10    | reclaim as a constant cost of running                                                                                  |
+| `classify.memFullPeak`     | 0.05    | half a second in any 10 in which nothing ran at all                                                                    |
+| `classify.minCoverage`     | 0.5     | a window the sampler barely saw is not judged (and so stays red)                                                       |
+| `classify.killWindowSec`   | 120     | a suite killed from outside is judged on its last two minutes                                                          |
+| `abort.cpuSome`            | 0.70    | mean over the rule's window; with `memSome` 0.30 and `memFull` 0.10                                                    |
+| `abort.earlyWindowSec`     | 60      | starved for a minute within the first `earlyWithinSec` (300): stop                                                     |
+| `abort.sustainedWindowSec` | 300     | starved for five minutes anywhere: stop                                                                                |
+| `rerun.max`                | 1       | reruns after an uncertain attempt; 0 turns them off                                                                    |
+
+Calling a real failure uncertain costs a rerun; calling a starved one red costs a
+human an investigation of nothing. The bars are set where the first is unlikely,
+and they are config so test2's recorded timelines can move them.
+
+**From a test.** `hostContendedDuring({ fromMs, toMs })` from
+`@norskvideo/ctl-test-harness/contention` answers for any window of the running
+suite (it reads the timeline the wrapper names in `NORSK_CONTENTION_TIMELINE`),
+with the numbers. Outside the wrapper it answers "unavailable", never
+"contended".
+
+**Exit statuses** of `ctl-contention run`: 0 pass; the suite's own status on a
+real failure; 75 (EX_TEMPFAIL) when uncertain, which the workflow believes only
+alongside a `verdict.json` that says so.
+
+**Caveats.** A `ctl-candidate` dispatch that is cancelled is still scored red by
+release-collect (the gate is advisory). bot-prs reports a PR whose integration
+check was cancelled as needing a human rather than re-running it, because its
+check-run filter counts `cancelled` before the tier rerun logic sees it; the
+in-job rerun is what retries.
+
 ## Reaching an image product's control plane
 
 `product add --image` loopback-publishes the control plane on a
@@ -255,7 +330,9 @@ NON-PRODUCTION` warning is expected in CI, not an error. The media engine's
 (`conventions/integration.yml`) and drift-gated like `smoke.yml`, with the same
 two per-repo lines: the `product:` dashboard key and the matrix `runner:` label
 array. Everything else a repo used to hand-edit into the workflow now lives in
-three optional hooks the workflow calls if they are executable:
+three optional hooks the workflow calls if they are executable (and one opt-in
+file, `scripts/integration/contention.json`, described under "Contention
+verdicts" below):
 
 | Hook                                   | When                              | What goes there                                                                           |
 | -------------------------------------- | --------------------------------- | ----------------------------------------------------------------------------------------- |

package/conventions/integration.yml

@@ -17,7 +17,7 @@
 # green, and ctl's (advisory) release gate, whose ctl-candidate dispatch runs it
 # and shows the result on the CI board.
 #
-# Per-repo variation lives in three optional hooks, not in this file:
+# Per-repo variation lives in four optional hooks, not in this file:
 #   scripts/integration/prepull-extra.sh  extra images the suite launches (a
 #                                         driver, a sink, a sidecar)
 #   scripts/integration/prepare.sh        built before the suite runs (a
@@ -26,6 +26,11 @@
 #   scripts/integration/verify.sh         after the suite; absent, a product
 #                                         with tests/demo.spec.ts gets
 #                                         `bun run demo -- check`
+#   scripts/integration/contention.json   opts the suite into contention verdicts
+#                                         (thresholds; {} takes the defaults): a
+#                                         failure the starved HOST caused cancels
+#                                         the run instead of failing it. See
+#                                         contention-suite.sh and the steps below.
 #
 # The suite is piped into tee so its log survives as an artifact, and the step
 # sets pipefail FIRST: the default `bash -e {0}` has none, so without it the
@@ -102,6 +107,11 @@ jobs:
         include:
           - runner: '["x64"]'
     runs-on: ${{ fromJSON(matrix.runner) }}
+    # actions: write is for one call only: the last step cancelling this run
+    # when the host was too contended to judge it.
+    permissions:
+      contents: read
+      actions: write
     steps:
       # Runs BEFORE checkout, and does two things: points the harness's temp base
       # out of the workspace for every later step in this job, and clears whatever
@@ -192,6 +202,38 @@ jobs:
             echo "NORSK_RUNNER_OWNS_DOCKER=true" >> "$GITHUB_ENV"
           fi
 
+      # Contention verdicts (scripts/integration/contention.json) read host
+      # pressure from Linux PSI, and whether a runner can is a property of its
+      # kernel (CONFIG_PSI, the psi= boot flag) and of its container (cgroup v2,
+      # what is masked). This says which, on every run and whether or not the
+      # repo has opted in, so the log answers "does the signal exist on this
+      # host?" before anything depends on it. It only reports.
+      - name: Probe whether this runner can read host pressure (PSI)
+        run: |
+          set -uo pipefail
+          echo "host=$(hostname) kernel=$(uname -r) cpus=$(nproc) of $(grep -c ^processor /proc/cpuinfo 2>/dev/null || echo '?')"
+          echo "loadavg: $(cat /proc/loadavg 2>/dev/null || echo unreadable)"
+          grep -E '^(MemTotal|MemAvailable):' /proc/meminfo 2>/dev/null || true
+          readable=""
+          for f in /proc/pressure/cpu /proc/pressure/memory /proc/pressure/io /sys/fs/cgroup/cpu.pressure /sys/fs/cgroup/memory.pressure; do
+            if out="$(cat "$f" 2>&1)" && grep -q '^some avg10=' <<<"$out"; then
+              echo "$f: readable"
+              sed 's/^/  /' <<<"$out"
+              readable="$readable $f"
+            else
+              echo "$f: NOT readable (${out:-empty})"
+            fi
+          done
+          if [ -f /sys/fs/cgroup/cgroup.controllers ]; then echo "cgroup v2 at /sys/fs/cgroup; this process: $(cat /proc/self/cgroup 2>/dev/null)"; else echo "no cgroup v2 at /sys/fs/cgroup"; fi
+          case "$readable" in
+            *"/proc/pressure/cpu"*"/proc/pressure/memory"*) verdict="host-wide PSI readable (/proc/pressure)" ;;
+            *"/sys/fs/cgroup/cpu.pressure"*"/sys/fs/cgroup/memory.pressure"*) verdict="only this cgroup's PSI readable: blind to the engine containers, which are host siblings" ;;
+            *) verdict="PSI unavailable: contention verdicts would change nothing here" ;;
+          esac
+          if [ -f scripts/integration/contention.json ]; then optin="on (scripts/integration/contention.json)"; else optin="off (no scripts/integration/contention.json)"; fi
+          echo "PSI on $(hostname): $verdict; contention verdicts $optin"
+          echo "PSI on \`$(hostname)\`: $verdict; contention verdicts $optin" >> "$GITHUB_STEP_SUMMARY"
+
       # Integration needs this host's docker to be able to create a network: the
       # harness makes norsk-net and a network per instance. On 2026-10-02
       # ashton-nuc's filter table had lost DOCKER-ISOLATION-STAGE-2, so every
@@ -409,6 +451,9 @@ jobs:
           # and the first artifact built that way carried ten of another product's
           # instances alongside its own three. Same shape the test-temp base uses.
           NORSK_CI_LOG_DIR: ${{ runner.temp }}/norsk-logs-${{ github.run_id }}-${{ github.run_attempt }}
+          # Where contention-suite.sh puts each attempt's pressure timeline,
+          # verdict and summary, keyed like the log dir for the same reason.
+          NORSK_CONTENTION_DIR: ${{ runner.temp }}/contention-${{ github.run_id }}-${{ github.run_attempt }}
         run: |
           set -euo pipefail
           suite() {
@@ -424,7 +469,11 @@ jobs:
                 echo "running scripts/integration/prepare.sh"
                 ./scripts/integration/prepare.sh
               fi
-              bun run test:integration
+              if [ -f scripts/integration/contention.json ]; then
+                bash node_modules/@norskvideo/ctl-dev-kit/conventions/contention-suite.sh
+              else
+                bun run test:integration
+              fi
               if [ -x scripts/integration/verify.sh ]; then
                 echo "running scripts/integration/verify.sh"
                 ./scripts/integration/verify.sh
@@ -460,7 +509,25 @@ jobs:
           else
             echo "no writable /srv/ci-scratch on this runner: the suite runs without the host lock"
           fi
+          set +e
           suite 9>&- 2>&1 | tee "$RUNNER_TEMP/integration.log"
+          rc=$?
+          set -e
+          # 75 is contention-suite.sh saying "uncertain": every failure
+          # overlapped host contention, or the suite was stopped because the
+          # host was starved. Believed only with the verdict file that says so,
+          # since a suite could exit 75 for its own reasons. Not a failure:
+          # the last step of this job cancels the run instead, so it reads
+          # cancelled (not red) on the board and bot-prs does not merge it.
+          # Every other status, a real failure on a quiet box included, exits
+          # exactly as it did.
+          verdict="$NORSK_CONTENTION_DIR/verdict.json"
+          if [ "$rc" -eq 75 ] && [ "$(jq -r .status "$verdict" 2>/dev/null)" = uncertain ]; then
+            echo "::warning title=integration uncertain, the host was contended::$(jq -r .reason "$verdict"). This run will be cancelled rather than failed; the pressure timeline and per-test classification are in the job summary and the integration artifact."
+            echo "NORSK_CONTENTION_UNCERTAIN=1" >> "$GITHUB_ENV"
+            exit 0
+          fi
+          exit "$rc"
 
       # The log AND the harness evidence. Playwright and the harness write
       # screenshots, traces and captured container logs under test-results/;
@@ -475,6 +542,7 @@ jobs:
             ${{ runner.temp }}/integration.log
             test-results/
             ${{ runner.temp }}/norsk-logs-${{ github.run_id }}-${{ github.run_attempt }}
+            ${{ runner.temp }}/contention-${{ github.run_id }}-${{ github.run_attempt }}
           retention-days: 7
           if-no-files-found: warn
 
@@ -514,6 +582,30 @@ jobs:
             docker rm -f $s
           fi
 
+      # Last, so the evidence is uploaded and the host reaped first. The suite
+      # step flagged this run uncertain: its failures were the starved host's,
+      # not the product's, so it is cancelled rather than failed. A cancelled
+      # run is not reported to the CI board (notify is !cancelled()), and
+      # bot-prs merges only a success, so a bump cannot land on it.
+      # Cancellation is asynchronous: the step holds the job until it lands,
+      # and if it is refused or never lands the job FAILS, because an uncertain
+      # run must never finish reading green. contention-wired.test.ts.
+      - name: Cancel this run, the host was too contended to judge it
+        if: ${{ always() && env.NORSK_CONTENTION_UNCERTAIN == '1' }}
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          set -uo pipefail
+          code="$(curl -sS -o /dev/null -w '%{http_code}' -X POST \
+            -H "Authorization: Bearer $GH_TOKEN" \
+            -H "Accept: application/vnd.github+json" \
+            "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/cancel")"
+          [ "$code" = "202" ] || { echo "::error::could not cancel this uncertain run (HTTP $code), so it fails instead: read the contention summary before treating it as a product failure"; exit 1; }
+          echo "cancel accepted; waiting for it to land"
+          sleep 300
+          echo "::error::still running five minutes after the cancel was accepted, so it fails instead: read the contention summary before treating it as a product failure"
+          exit 1
+
   # Report this pipeline's result to the aggregated product CI dashboard
   # (id3as/ci-workflows). !cancelled() so a real pass/fail reports, but a
   # cancelled/superseded run does NOT — a cancellation dispatched as failure

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@norskvideo/ctl-dev-kit",
-  "version": "0.2.99",
+  "version": "0.2.100",
   "type": "module",
   "exports": {
     "./create-product": "./create-product/create-product.ts",