@norskvideo/ctl-dev-kit 0.2.86 → 0.2.88

package/conventions/integration-testing.md

@@ -101,6 +101,21 @@ On 2026-10-02 a reuters tier and a commentary tier each `compose up`ed
   port (bound inside the runner container, so not shared). In `direct` reach
   mode with `--internal-only` the studio port is not published at all.
 
+### One suite per host
+
+Names can be made unique; CPU cannot. A runner container is pinned to its own
+cores, but the media containers its suite launches are host siblings and run
+anywhere, so two suites on one host starve each other's engines. On 2026-10-15/16
+commentary's timing tests (ears, realtime drift, soak preview) failed in 10 of the
+12 test2 runs that overlapped another suite, most often a second commentary run on
+the other runner. The canonical `integration.yml` therefore runs its suite under
+an `flock` on `/srv/ci-scratch/locks/norsk-ctl-integration.lock`:
+`/srv/ci-scratch` is bind-mounted at that path into every jankings runner on a
+host, so the file is shared by that host's runners and by no other host's. A
+runner without the mount runs unserialised and says so in the log; a suite that
+has waited an hour runs anyway with a warning. A hand-owned `integration.yml`
+does not take the lock until it converges onto the canonical.
+
 ## Reaching an image product's control plane
 
 `product add --image` loopback-publishes the control plane on a
@@ -255,8 +270,11 @@ that way. Hand the suite a value by having both sides derive it (a tiny script
 the hook runs and the test imports), not through `$GITHUB_ENV`.
 
 `bun run test:integration` is the product's claim of what its tier is, and
-ctl's release gate believes it: a `ctl-candidate` dispatch runs this workflow
-and the reported result decides whether `latest` moves.
+two things believe it: the repo's nightly bump PR runs it on a new ctl pin and
+merges only when it is green, and ctl's release gate sends a `ctl-candidate`
+dispatch that runs this workflow and shows the result on the CI board. The
+gate is advisory — the result does not hold ctl's `latest` — so the bump PR
+is what keeps a ctl that breaks this product out of it.
 
 **Converging a repo.** The gate only takes over a copy whose first line is the
 canonical's marker (`# ctl-dev-kit convention: integration`); a copy without it

package/conventions/integration.yml

@@ -12,9 +12,10 @@
 # What the tier is: the product's slow suite (daemon + Docker + licence) run
 # against the RELEASED norsk-ctl binary, exactly as a customer would run it — a
 # split product repo has no ctl source. `bun run test:integration` is the
-# product's claim of what that suite is, and ctl's release gate believes it: a
-# ctl-candidate dispatch runs this and the result decides whether `latest`
-# moves.
+# product's claim of what that suite is, and two things believe it: this
+# repo's nightly bump PR, which runs it on a new ctl pin and merges only when
+# green, and ctl's (advisory) release gate, whose ctl-candidate dispatch runs it
+# and shows the result on the CI board.
 #
 # Per-repo variation lives in three optional hooks, not in this file:
 #   scripts/integration/prepull-extra.sh  extra images the suite launches (a
@@ -57,7 +58,7 @@ on:
   workflow_dispatch:
     inputs:
       ctl_channel:
-        description: "norsk-ctl S3 channel to test against in place of the repo's pin (flake.nix ctlVersion). Empty runs the pin; release-gate dispatches 'rc' to validate a candidate before promoting latest."
+        description: "norsk-ctl S3 channel to test against in place of the repo's pin (flake.nix ctlVersion). Empty runs the pin; release-gate dispatches 'rc' to report on a candidate (advisory: latest is not held on it)."
         type: string
         default: ""
 
@@ -66,7 +67,8 @@ permissions:
 
 # A ctl-candidate validation must not be cancellable out of a result, because
 # release-collect scores anything other than a literal success as red — a
-# CANCELLED run included — and answers a red with one re-dispatch.
+# CANCELLED run included. (Until the gate went advisory on 2026-10-17 it also
+# answered a red with one re-dispatch, which is what made this bite.)
 #
 # Keying on event_name alone was not enough: every candidate is the SAME event,
 # so they shared a group and cancelled each other, and the re-dispatch cancelled
@@ -428,7 +430,34 @@ jobs:
               fi
             '
           }
-          suite 2>&1 | tee "$RUNNER_TEMP/integration.log"
+          # One suite per HOST. A host's runners are sibling containers on one
+          # docker daemon, and the media containers a suite launches are not
+          # confined to its runner's cpuset, so two suites side by side starve
+          # each other's engines: on 2026-10-15/16 commentary's timing tests
+          # failed in 10 of the 12 test2 runs that overlapped another suite on the
+          # other runner, mostly commentary against itself, which a per-repo
+          # concurrency group cannot stop without dropping a queued candidate.
+          # /srv/ci-scratch is bind-mounted at that path into every runner on a
+          # jankings host, so an flock there is the host's mutex; the kernel
+          # drops it when this shell exits, however the job ends. Children close
+          # fd 9 so a straggler cannot hold the host after the step. After an
+          # hour the suite runs anyway, warned, rather than miss its result.
+          # integration-host-lock.test.ts.
+          if [ -d /srv/ci-scratch ] && [ -w /srv/ci-scratch ]; then
+            lock=/srv/ci-scratch/locks/norsk-ctl-integration.lock
+            mkdir -p "${lock%/*}"
+            exec 9>>"$lock"
+            echo "waiting for this host's integration lock ($lock)"
+            waited=$SECONDS
+            if flock -w 3600 9; then
+              echo "holding this host's integration lock after $((SECONDS - waited))s"
+            else
+              echo "::warning title=integration ran unserialised::waited an hour for $lock on $(hostname); timing-sensitive tests may fail for a reason that is not theirs"
+            fi
+          else
+            echo "no writable /srv/ci-scratch on this runner: the suite runs without the host lock"
+          fi
+          suite 9>&- 2>&1 | tee "$RUNNER_TEMP/integration.log"
 
       # The log AND the harness evidence. Playwright and the harness write
       # screenshots, traces and captured container logs under test-results/;
@@ -583,8 +612,9 @@ jobs:
           fi
 
   # Report the rc-candidate validation result to the norsk-ctl release gate. Only
-  # on a ctl-candidate dispatch; release-collect tallies these and promotes
-  # latest once every gating product is green on the candidate.
+  # on a ctl-candidate dispatch; release-collect records it for the CI board.
+  # Advisory: ctl promotes latest without waiting on it, and this repo takes a
+  # new ctl only through its bump PR, which runs this tier on the new pin.
   report-candidate:
     needs: integration
     if: ${{ always() && github.event_name == 'repository_dispatch' && github.event.action == 'ctl-candidate' }}
@@ -602,7 +632,8 @@ jobs:
           set -euo pipefail
           state=failure; [ "$RESULT" = "success" ] && state=success
           # `tier: integration`, named rather than defaulted: the release gate
-          # tallies per tier, and this is the one that gates (docs/release-gate.md).
+          # tallies per tier, and this is the one that launches instances
+          # (docs/release-gate.md).
           body="$(jq -nc --arg p "$product" --arg v "$VERSION" --arg r "$state" \
             '{event_type:"ctl-candidate-result",client_payload:{product:$p,version:$v,result:$r,tier:"integration"}}')"
           code="$(curl -sS -o /dev/null -w '%{http_code}' -X POST \

package/conventions/smoke.yml

@@ -30,7 +30,7 @@ on:
   workflow_dispatch:
     inputs:
       ctl_channel:
-        description: "norsk-ctl S3 channel to test against in place of the repo's pin (flake.nix ctlVersion). Empty runs the pin; release-gate dispatches 'rc' to validate a candidate before promoting latest."
+        description: "norsk-ctl S3 channel to test against in place of the repo's pin (flake.nix ctlVersion). Empty runs the pin; release-gate dispatches 'rc' to report on a candidate (advisory: latest is not held on it)."
         type: string
         default: ""
 
@@ -503,8 +503,9 @@ jobs:
           fi
 
   # Report the rc-candidate validation result to the norsk-ctl release gate. Only
-  # on a ctl-candidate dispatch; release-collect tallies these and promotes
-  # latest once every gating product is green on the candidate.
+  # on a ctl-candidate dispatch; release-collect records it for the CI board.
+  # Advisory: ctl promotes latest without waiting on it, and this repo takes a
+  # new ctl only through its bump PR, which runs this tier on the new pin.
   report-candidate:
     needs: smoke
     if: ${{ always() && github.event_name == 'repository_dispatch' && github.event.action == 'ctl-candidate' }}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@norskvideo/ctl-dev-kit",
-  "version": "0.2.86",
+  "version": "0.2.88",
   "type": "module",
   "exports": {
     "./create-product": "./create-product/create-product.ts",