@norskvideo/ctl-dev-kit 0.2.63 → 0.2.65

package/conventions/build-image.yml

@@ -21,9 +21,11 @@
 # (rc -> latest promotion) lands, flip the push default to `nightly` and let the
 # promotion gate move `latest`.
 #
-# BOT PRs: on a pull request from a bot branch (upgrade-latest/*,
-# sync-dev-kit/*, chore/sync-ctl-packages) the workflow builds both arches and
-# runs the guides gate exactly as a push would, and publishes, records and
+# BOT PRs: on a pull request from a bot branch (bump/nightly, and the retired
+# bots' upgrade-latest/*, sync-dev-kit/* and chore/sync-ctl-packages, which the
+# PR that brings bump.yml rides on) the workflow builds both arches and
+# runs the guides gate exactly as a push would (the deterministic tiers gate;
+# the engine tier reports beside it), and publishes, records and
 # reports nothing. A bump reaches main only once that has passed on it, and
 # bot-prs merges on it (ci-workflows docs/dashboard/ci-fragility.md, proposal
 # B): a dev-kit sync that brought a newer copy of THIS file broke
@@ -68,7 +70,7 @@ jobs:
   # Compute the immutable label ONCE, so both arch builds and the manifest share
   # it (a per-job `date` could straddle midnight and disagree).
   label:
-    if: ${{ github.event_name != 'pull_request' || (github.event.pull_request.head.repo.full_name == github.repository && (startsWith(github.head_ref, 'upgrade-latest/') || startsWith(github.head_ref, 'sync-dev-kit/') || github.head_ref == 'chore/sync-ctl-packages')) }}
+    if: ${{ github.event_name != 'pull_request' || (github.event.pull_request.head.repo.full_name == github.repository && (github.head_ref == 'bump/nightly' || startsWith(github.head_ref, 'upgrade-latest/') || startsWith(github.head_ref, 'sync-dev-kit/') || github.head_ref == 'chore/sync-ctl-packages')) }}
     runs-on: x64
     outputs:
       label: ${{ steps.gen.outputs.label }}
@@ -302,15 +304,13 @@ jobs:
   gate-guides:
     needs: label
     runs-on: x64
-    # One guide run per repo at a time, nightly and publish gate alike: both
-    # launch the same instance ids on the x64 pool's shared docker daemon, and
-    # ctl names each compose project after its instance id. Queue, never cancel.
-    concurrency:
-      group: doc-guides-${{ github.repository }}
-      cancel-in-progress: false
+    # No doc-guides concurrency group: the deterministic tiers launch no
+    # instances, so there is nothing to serialise, and a pending job in a group
+    # is cancelled by the next one queued. guides-engine takes the group.
     outputs:
       guides: ${{ steps.plan.outputs.guides }}
       reason: ${{ steps.plan.outputs.reason }}
+      engine: ${{ steps.plan.outputs.engine }}
     steps:
       # Runs BEFORE checkout, and does two things: points the harness's temp base
       # out of the workspace for every later step in this job, and clears whatever
@@ -390,6 +390,10 @@ jobs:
             ')"
             if [ "$decision" = "run" ]; then
               echo "guides=run" >> "$GITHUB_OUTPUT"
+              # The engine tier runs in guides-engine, beside this gate.
+              if jq -e '.scripts["docs:guides:engine"]' package.json >/dev/null; then
+                echo "engine=run" >> "$GITHUB_OUTPUT"
+              fi
             else
               skip "${decision#skip$'\t'}"
             fi
@@ -401,8 +405,33 @@ jobs:
           NORSK_LICENSE_V2: ${{ secrets.NORSK_LICENSE_V2 }}
         run: printf '%s' "$NORSK_LICENSE_V2" > "$RUNNER_TEMP/norsk-license.json"
 
+      # Every norsk-ctl daemon starts the node MoQ relay (norsk-moq-relay) unless
+      # told not to, and nothing removes it: it is a node singleton, and the reap
+      # step only takes containers that mount this job's test-temp. So the first
+      # test daemon on a shared runner left one, and every later one adopted it
+      # (probe's smoke ran in relay mode on 2026-10-10; relay's smoke lost 4443
+      # to it). This job's daemons run without the node relay and discovery, so
+      # they neither start nor adopt one, and whatever is already on the host is
+      # named here. It reports and never fails: the leftover may be someone's.
+      - name: Isolate this job from node singletons other jobs left on this host
+        run: |
+          set -uo pipefail
+          echo "NORSK_CTL_MOQ_RELAY_ENABLE=0" >> "$GITHUB_ENV"
+          echo "NORSK_CTL_DISCOVERY_ENABLE=0" >> "$GITHUB_ENV"
+          left="$(docker ps -a --filter label=norsk-ctl.role --format '{{.Names}} {{.Label "norsk-ctl.role"}} {{.Status}} (created {{.CreatedAt}})' 2>/dev/null || true)"
+          if [ -z "$left" ]; then echo "no norsk-ctl containers on this host"; exit 0; fi
+          echo "norsk-ctl containers already on this host (other jobs' running instances are expected):"
+          echo "$left"
+          while IFS= read -r line; do
+            role="$(awk '{print $2}' <<<"$line")"
+            case "$role" in
+              moq-relay|discovery)
+                echo "::warning title=node singleton left on this host::$line -- left by an earlier job (a daemon's auto-start, or a run killed before its reap). This job's daemons run without it and will not adopt it; remove it with docker rm -f if nothing on this host needs it." ;;
+            esac
+          done <<<"$left"
+
       # The norsk-ctl this commit pins (flake.nix ctlVersion, moved by
-      # upgrade-latest's bump PR), never S3's moving `latest`: a guide run is then
+      # bump.yml's nightly PR), never S3's moving `latest`: a guide run is then
       # a real repeat, and a red belongs to a commit (ci-fragility proposal A).
       - name: Download the pinned norsk-ctl binary (flake.nix ctlVersion)
         if: ${{ steps.plan.outputs.guides == 'run' }}
@@ -410,7 +439,7 @@ jobs:
           set -euo pipefail
           S3="https://s3.eu-west-1.amazonaws.com/norsk.video/norsk-ctl"
           ver="$( { sed -nE 's/^[[:space:]]*ctlVersion = "([^"]+)";.*/\1/p' flake.nix 2>/dev/null || true; } | head -1)"
-          [ -n "$ver" ] || { echo "::error::flake.nix pins no ctlVersion. CI runs the norsk-ctl this commit pins (upgrade-latest moves it); add ctlVersion = \"<version>\"; to flake.nix."; exit 1; }
+          [ -n "$ver" ] || { echo "::error::flake.nix pins no ctlVersion. CI runs the norsk-ctl this commit pins (the nightly bump.yml moves it); add ctlVersion = \"<version>\"; to flake.nix."; exit 1; }
           echo "norsk-ctl pinned by flake.nix ctlVersion -> $ver"
           curl -fsSL --retry 3 --retry-all-errors --retry-delay 2 "$S3/$ver/norsk-ctl-$ver-linux-x64" -o "$RUNNER_TEMP/norsk-ctl"
           chmod +x "$RUNNER_TEMP/norsk-ctl"
@@ -433,13 +462,15 @@ jobs:
             ./scripts/doc-guide/prepull-extra.sh
           fi
 
-      # Rebuild every workspace the manual depends on (a guide's engine tier packs
-      # the freshly-built dashboard + frontend dist into the launched template),
-      # then run the product's doc-guide tiers and assemble the manual.
-      # regen-manual.sh owns the slug list and which tiers run; with-display.sh
-      # wraps the whole run in an X display for headless chromium (the engine tier
-      # drives a raw chromium that needs one).
-      - name: Regenerate the illustrated manual (all tiers)
+      # The deterministic tiers (fixture and frontend: no daemon, no instances),
+      # then the manual assembled from what they captured. These gate. The engine
+      # tier runs on its own below: it stands up real instances, is known-flaky
+      # on container DNS (norsk-ctl task 57), and reports without gating until
+      # that is fixed (ci-workflows docs/dashboard/ci-fragility.md, proposal E).
+      # regen-manual.sh owns the slug list and honours SKIP_ENGINE; it rebuilds
+      # every workspace the manual depends on first, and with-display.sh gives
+      # headless chromium an X display.
+      - name: Regenerate the illustrated manual (deterministic tiers)
         if: ${{ steps.plan.outputs.guides == 'run' }}
         env:
           NORSK_CTL_BINARY: ${{ runner.temp }}/norsk-ctl
@@ -453,6 +484,7 @@ jobs:
           # them binding no host ports — so concurrent doc-guide instances can't
           # collide on the product's deterministic host-port bands.
           NORSK_TEST_NET: direct
+          SKIP_ENGINE: "1"
         run: |
           nix develop .#build --command bash -c '
             set -euo pipefail
@@ -486,6 +518,228 @@ jobs:
             docker rm -f $s
           fi
 
+  # The engine tier of the guides, beside the gate rather than in it (see the
+  # deterministic step above): it runs whenever the gate runs the guides and the
+  # product has an engine tier, and reports -- a warning, the run summary and
+  # the build-image cell's detail -- but publish does not wait on it, and a
+  # failure leaves the job green so bot-prs can still merge. To make it gate
+  # again once norsk-ctl task 57 is fixed, add it to publish's needs and drop
+  # continue-on-error from the engine step.
+  guides-engine:
+    needs: [label, gate-guides]
+    # !cancelled(): a red deterministic gate must not hide the engine tier's
+    # result. The plan's output is set before any guide runs.
+    if: ${{ !cancelled() && needs.gate-guides.outputs.engine == 'run' }}
+    runs-on: x64
+    # One engine-tier run per repo at a time, nightly (publish-docs.yml) and
+    # this alike: both launch the same instance ids on the x64 pool's shared
+    # docker daemon, and ctl names each compose project after its instance id.
+    # Queue, never cancel.
+    concurrency:
+      group: doc-guides-${{ github.repository }}
+      cancel-in-progress: false
+    outputs:
+      result: ${{ steps.engine.outcome }}
+    steps:
+      # Runs BEFORE checkout, and does two things: points the harness's temp base
+      # out of the workspace for every later step in this job, and clears whatever
+      # the old repo-local default left behind on this runner. The body carries the
+      # why. It fails the job only when legacy litter survives -- which would take
+      # the checkout down a step later anyway, but report nothing useful.
+      - name: Route test-temp out of the workspace (pre-checkout)
+        run: |
+          set -uo pipefail
+          # These runners are docker-OUTSIDE-of-docker, so a bind-mount source the
+          # HOST daemon has not seen before is created by dockerd, as ROOT, on the
+          # host -- even when the launch passes --container-user (reproduced
+          # 2026-09-04). The harness's default base is repo-local for an OrbStack
+          # inotify constraint that binds macOS dev and nothing here, so every run
+          # seeded the CHECKOUT with root-owned dirs a non-root `git clean -ffdx`
+          # cannot remove. That took commentary's checks AND integration red on
+          # 2026-09-03. Use the sanctioned override, per-run and outside the
+          # checkout -- as 8ff17a8d did in June for one debug workflow, and stopped.
+          base="$RUNNER_TEMP/nctt-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT"
+          mkdir -p "$base"
+          echo "NORSK_CTL_TEST_TMP=$base" >> "$GITHUB_ENV"
+          # Yesterday's bases hold dockerd-created root dirs, so a non-root rm
+          # EACCESes. They cannot wedge a checkout, but they do fill the disk.
+          for old in $(find "$RUNNER_TEMP" -maxdepth 1 -name 'nctt-*' -mtime +0 -printf '%f\n' 2>/dev/null || true); do
+            docker run --rm --user 0:0 -v "$RUNNER_TEMP":/t alpine sh -c 'rm -rf -- "/t/$1"' sh "$old" >/dev/null 2>&1 || true
+          done
+          # LEGACY: a runner that ran the old default still carries a workspace
+          # test-temp, and a checkout with clean:true dies on it. Reap the holders
+          # first -- rm cannot unlink a live mountpoint even as root (EBUSY), and a
+          # RESTARTING leaked container re-creates the path as root seconds after
+          # any clean. Match on mount SOURCE, not a label: the holders are untracked
+          # by construction, because a killed run labels nothing.
+          tt="$GITHUB_WORKSPACE/test-temp"
+          [ -d "$tt" ] || exit 0
+          stuck=""
+          for c in $(docker ps -aq 2>/dev/null || true); do
+            if docker inspect -f '{{range .Mounts}}{{println .Source}}{{end}}' "$c" 2>/dev/null | grep -q "^$tt/"; then
+              stuck="$stuck $c"
+            fi
+          done
+          if [ -n "$stuck" ]; then
+            echo "reaping containers holding mounts under $tt:$stuck"
+            docker rm -f $stuck || true
+          fi
+          # The DIRECTORY, not just its contents: an empty but root-owned base
+          # still EACCESes a later mkdtemp. Mounting the PARENT is what lets a root
+          # container unlink the leaf.
+          docker run --rm --user 0:0 -v "$GITHUB_WORKSPACE":/w alpine \
+            sh -c 'rm -rf /w/test-temp' || rm -rf "$tt" || true
+          # Fail HERE if it survived. The checkout fails on it either way, but
+          # reports only an EACCES rmdir with no clue what was holding the path.
+          if [ -e "$tt" ]; then
+            echo "::error::$tt survived the pre-checkout clean"
+            ls -lan "$tt" || true
+            exit 1
+          fi
+
+      - uses: actions/checkout@v5
+        with:
+          clean: true
+
+      - name: Write the Norsk license (from the org secret)
+        env:
+          NORSK_LICENSE_V2: ${{ secrets.NORSK_LICENSE_V2 }}
+        run: printf '%s' "$NORSK_LICENSE_V2" > "$RUNNER_TEMP/norsk-license.json"
+      # Every norsk-ctl daemon starts the node MoQ relay (norsk-moq-relay) unless
+      # told not to, and nothing removes it: it is a node singleton, and the reap
+      # step only takes containers that mount this job's test-temp. So the first
+      # test daemon on a shared runner left one, and every later one adopted it
+      # (probe's smoke ran in relay mode on 2026-10-10; relay's smoke lost 4443
+      # to it). This job's daemons run without the node relay and discovery, so
+      # they neither start nor adopt one, and whatever is already on the host is
+      # named here. It reports and never fails: the leftover may be someone's.
+      - name: Isolate this job from node singletons other jobs left on this host
+        run: |
+          set -uo pipefail
+          echo "NORSK_CTL_MOQ_RELAY_ENABLE=0" >> "$GITHUB_ENV"
+          echo "NORSK_CTL_DISCOVERY_ENABLE=0" >> "$GITHUB_ENV"
+          left="$(docker ps -a --filter label=norsk-ctl.role --format '{{.Names}} {{.Label "norsk-ctl.role"}} {{.Status}} (created {{.CreatedAt}})' 2>/dev/null || true)"
+          if [ -z "$left" ]; then echo "no norsk-ctl containers on this host"; exit 0; fi
+          echo "norsk-ctl containers already on this host (other jobs' running instances are expected):"
+          echo "$left"
+          while IFS= read -r line; do
+            role="$(awk '{print $2}' <<<"$line")"
+            case "$role" in
+              moq-relay|discovery)
+                echo "::warning title=node singleton left on this host::$line -- left by an earlier job (a daemon's auto-start, or a run killed before its reap). This job's daemons run without it and will not adopt it; remove it with docker rm -f if nothing on this host needs it." ;;
+            esac
+          done <<<"$left"
+
+      # The norsk-ctl this commit pins (flake.nix ctlVersion, moved by
+      # bump.yml's nightly PR), never S3's moving `latest`: a guide run is then
+      # a real repeat, and a red belongs to a commit (ci-fragility proposal A).
+      - name: Download the pinned norsk-ctl binary (flake.nix ctlVersion)
+        run: |
+          set -euo pipefail
+          S3="https://s3.eu-west-1.amazonaws.com/norsk.video/norsk-ctl"
+          ver="$( { sed -nE 's/^[[:space:]]*ctlVersion = "([^"]+)";.*/\1/p' flake.nix 2>/dev/null || true; } | head -1)"
+          [ -n "$ver" ] || { echo "::error::flake.nix pins no ctlVersion. CI runs the norsk-ctl this commit pins (the nightly bump.yml moves it); add ctlVersion = \"<version>\"; to flake.nix."; exit 1; }
+          echo "norsk-ctl pinned by flake.nix ctlVersion -> $ver"
+          curl -fsSL --retry 3 --retry-all-errors --retry-delay 2 "$S3/$ver/norsk-ctl-$ver-linux-x64" -o "$RUNNER_TEMP/norsk-ctl"
+          chmod +x "$RUNNER_TEMP/norsk-ctl"
+          "$RUNNER_TEMP/norsk-ctl" --version || true
+      # Cold-runner first pulls overrun the harness's per-test launch timeouts;
+      # pre-pull the universal media + studio images (every product pins them in
+      # manifest.seed.json) so compose/run hit local images. A product needing
+      # more (e.g. a WHIP driver) pulls them in scripts/doc-guide/prepull-extra.sh.
+      - name: Pre-pull the media + studio images
+        run: |
+          set -euo pipefail
+          for img in "$(jq -r '.latest.media' manifest.seed.json)" "$(jq -r '.latest.studio' manifest.seed.json)"; do
+            echo "pre-pulling $img"
+            docker pull "$img"
+          done
+          if [ -x scripts/doc-guide/prepull-extra.sh ]; then
+            echo "running scripts/doc-guide/prepull-extra.sh"
+            ./scripts/doc-guide/prepull-extra.sh
+          fi
+      - name: Install and build the workspaces the guides drive
+        run: |
+          nix develop .#build --command bash -c '
+            set -euo pipefail
+            find . -name node_modules -type d -prune -exec rm -rf {} + 2>/dev/null || true
+            bun install --frozen-lockfile
+            bun run build:no-lint
+          '
+
+      # The engine tier: real instances, driven in a browser. Reported, not
+      # gating (see the deterministic step). A non-zero exit is forgiven only
+      # when the tier's own results say a teardown hook timed out after every
+      # guide's test passed (doc-guide/engine-verdict.ts), as regen-manual.sh
+      # forgives it. A product with no docs:guides:engine script has no engine
+      # tier. Runs on the workspaces the step before it installed and built.
+      - name: Run the engine-tier guides (reported, not gating)
+        id: engine
+        continue-on-error: true
+        env:
+          NORSK_CTL_BINARY: ${{ runner.temp }}/norsk-ctl
+          NORSK_LICENSE_FILE: ${{ runner.temp }}/norsk-license.json
+          # This runner launches Studio as a HOST sibling (DooD), so the harness
+          # reaches host-published ports via the host-gateway alias, not
+          # localhost — same as the integration suite.
+          NORSK_TEST_HOST: host.docker.internal
+          # Reach launched instances over norsk-net by service DNS (as the
+          # integration tier already does) AND, on a ctl that supports it, launch
+          # them binding no host ports — so concurrent doc-guide instances can't
+          # collide on the product's deterministic host-port bands.
+          NORSK_TEST_NET: direct
+        run: |
+          nix develop .#build --command bash -c '
+            set -euo pipefail
+            if ! jq -e ".scripts[\"docs:guides:engine\"]" package.json >/dev/null; then
+              echo "no docs:guides:engine script: this product has no engine tier"
+              exit 0
+            fi
+            export DOCS_GENERATE=1
+            log="$(mktemp)"
+            if bash scripts/with-display.sh bun run docs:guides:engine >"$log" 2>&1; then
+              cat "$log"
+            else
+              cat "$log"
+              verdict="$(bun -e "console.log(Bun.resolveSync(\"@norskvideo/ctl-dev-kit/doc-guide/engine-verdict\", process.cwd()))")"
+              bun "$verdict" "$log"
+            fi
+          '
+
+      # Said three ways, so a failure is never silent: a warning on the run, a
+      # line in its summary, and the cell's detail (the notify job).
+      - name: Report the engine tier (not gating)
+        if: ${{ !cancelled() }}
+        env:
+          OUTCOME: ${{ steps.engine.outcome }}
+        run: |
+          case "$OUTCOME" in
+            success) echo "engine-tier guides passed (or this product has none)" ;;
+            failure)
+              echo "::warning title=engine-tier guides failed (not gating)::the engine tier does not gate until norsk-ctl task 57 is fixed. See the engine step's log."
+              printf '### Engine-tier guides: failed (not gating)\n\nThe deterministic guide tiers gate; the engine tier only reports until its container networking is fixed (norsk-ctl task 57). See the step log.\n' >> "$GITHUB_STEP_SUMMARY" ;;
+            *) echo "engine tier: ${OUTCOME:-not run}" ;;
+          esac
+
+      - name: Reap the containers this job launched
+        # Whatever the tests did. A teardown cut short by a test timeout leaves
+        # its instance running on the shared Docker daemon (see JOB_REAP_STEP in
+        # conventions/workflow-shell.ts); everything this job launched mounts its
+        # own test-temp, so that is what goes.
+        if: always()
+        run: |
+          base="${NORSK_CTL_TEST_TMP:-}"
+          tt="$GITHUB_WORKSPACE/test-temp"
+          s=""
+          for c in $(docker ps -aq); do
+            src="$(docker inspect -f '{{range .Mounts}}{{println .Source}}{{end}}' "$c" 2>/dev/null)" || continue
+            if { [ -n "$base" ] && grep -q "^$base/" <<<"$src"; } || grep -q "^$tt/" <<<"$src"; then s="$s $c"; fi
+          done
+          if [ -n "$s" ]; then
+            echo "::warning::removing containers this job left running:$s"
+            docker rm -f $s
+          fi
+
   # Combine the per-arch images into a multi-arch manifest and write the two
   # published tags. Skipped for a build-only product (empty PUBLISH_IMAGE) and on
   # a dry run.
@@ -733,7 +987,15 @@ jobs:
   # the pony/emoji from the dispatched result. always() so a red or manual run
   # still reports.
   notify:
-    needs: [label, build, gate-checks, gate-guides, publish, record]
+    needs: [
+      label,
+      build,
+      gate-checks,
+      gate-guides,
+      guides-engine,
+      publish,
+      record
+    ]
     if: ${{ !cancelled() && github.event_name != 'pull_request' }}
     runs-on: x64
     steps:
@@ -796,9 +1058,14 @@ jobs:
       - uses: actions/checkout@v5
         with:
           clean: true
+      # guides-engine reports in the detail below and never sets the state: a
+      # failed engine tier is not a failed build, nor is one cancelled in its
+      # concurrency queue.
       - id: meta
+        env:
+          NEEDS: ${{ toJSON(needs) }}
         run: |
-          if [ "${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}" = "true" ]; then
+          if jq -e 'del(.["guides-engine"]) | [.[].result] | any(. == "failure" or . == "cancelled")' <<<"$NEEDS" >/dev/null; then
             echo "state=failure" >> "$GITHUB_OUTPUT"
           else
             echo "state=success" >> "$GITHUB_OUTPUT"
@@ -811,7 +1078,8 @@ jobs:
           status: ${{ steps.meta.outputs.state }}
           # Only where something is published: on a build-only repo the gate has
           # nothing to gate, and saying so on every build is noise.
-          detail: "${{ needs.label.outputs.publish == 'true' && needs.gate-guides.outputs.guides == 'skip' && format('guides not gating: {0}', needs.gate-guides.outputs.reason) || '' }}"
+          # And when the engine-tier guides failed: they report, never gate.
+          detail: "${{ needs.label.outputs.publish == 'true' && needs.gate-guides.outputs.guides == 'skip' && format('guides not gating: {0}', needs.gate-guides.outputs.reason) || needs.guides-engine.outputs.result == 'failure' && 'engine-tier guides failed (not gating; norsk-ctl task 57)' || '' }}"
       - name: Reap the containers this job launched
         # Whatever the tests did. A teardown cut short by a test timeout leaves
         # its instance running on the shared Docker daemon (see JOB_REAP_STEP in