@norskvideo/ctl-dev-kit 0.1.84 → 0.1.85

package/conventions/check-drift.ts

@@ -62,6 +62,20 @@ export const PRODUCT_SENTINEL = "__PRODUCT__";
 // unfilled-sentinel check.
 export const PUBLISH_IMAGE_LINE = /^(\s*PUBLISH_IMAGE:\s*).*$/gm;
 
+// integration.yml (04c s9 #1) is universally present and was hand-owned in six
+// shapes, so on the day it joined the conventions set the gate could not demand
+// the canonical of every copy: sync-dev-kit commits straight to main when
+// checks are green, and would have rewritten six tiers before the hooks their
+// suites need existed. A copy therefore declares itself converged by carrying
+// the canonical's own first line; one without it is hand-owned and ungated
+// until its repo converges deliberately. When every repo carries the marker,
+// this becomes unconditional like smoke.yml.
+export const INTEGRATION_MARKER = "# ctl-dev-kit convention: integration";
+
+export function carriesIntegrationMarker(text: string): boolean {
+  return text.split("\n")[0]?.trim() === INTEGRATION_MARKER;
+}
+
 // smoke.yml carries a second per-repo line: the matrix `runner:` label array
 // the journey runs on ('["x64"]' for the pool, '["ARM64","nvidia"]' for the GPU
 // box). Masked like product:; any filled value is valid, so no sentinel check.
@@ -252,6 +266,9 @@ export interface CanonicalBytes {
   gitignoreCore: string;
   buildImage: string;
   smoke: string;
+  /** conventions/integration.yml — the integration tier. Gated only for a copy
+   *  carrying INTEGRATION_MARKER on line 1; same two masked lines as smoke. */
+  integration: string;
   /** conventions/demo.sh — the scripts/demo shim a product with tests/demo.spec.ts carries. */
   demoShim: string;
   /** doc-guide/with-display.sh — the X-display wrapper BOTH docs workflows invoke
@@ -546,6 +563,25 @@ export function checkDrift(repoRoot: string, canonical: CanonicalBytes, opts: Ch
     );
   }
 
+  // integration.yml: gated once a copy carries the convention marker (see
+  // INTEGRATION_MARKER for why not before); then it is smoke.yml's twin, with
+  // the `product:` key and the matrix `runner:` array as the per-repo lines.
+  const integrationPath = join(repoRoot, ".github", "workflows", "integration.yml");
+  if (existsSync(integrationPath)) {
+    const actual = readFileSync(integrationPath, "utf8");
+    if (carriesIntegrationMarker(actual)) {
+      push(
+        workflowProblem(
+          ".github/workflows/integration.yml",
+          "conventions/integration.yml",
+          actual,
+          canonical.integration,
+          [PRODUCT_LINE, RUNNER_LINE],
+        ),
+      );
+    }
+  }
+
   // publish-docs.yml is OPTIONAL — a product carries it once it has an
   // illustrated manual to publish nightly. Unlike the workflows above it has NO
   // per-repo line: it is deliberately product-agnostic (per-repo bits live in
@@ -623,6 +659,7 @@ if (import.meta.main) {
     gitignoreCore: readFileSync(join(import.meta.dir, "gitignore.core"), "utf8"),
     buildImage: readFileSync(join(import.meta.dir, "build-image.yml"), "utf8"),
     smoke: readFileSync(join(import.meta.dir, "smoke.yml"), "utf8"),
+    integration: readFileSync(join(import.meta.dir, "integration.yml"), "utf8"),
     demoShim: readFileSync(join(import.meta.dir, "demo.sh"), "utf8"),
   };
   const report = checkDrift(repoRoot, canonical, { repoName: resolveRepoName(repoRoot) });

package/conventions/integration-testing.md

@@ -138,3 +138,43 @@ clients that create media nodes (e.g. a WHIP driver) must answer the license
 creation with `10 ABORTED: License handshake required`. The `license is marked
 NON-PRODUCTION` warning is expected in CI, not an error. The media engine's
 `hugepages / rte_eal_init failed` line is **benign noise** — do not chase it.
+
+## The workflow: `integration.yml` is a convention
+
+`.github/workflows/integration.yml` is single-sourced in this dev-kit
+(`conventions/integration.yml`) and drift-gated like `smoke.yml`, with the same
+two per-repo lines: the `product:` dashboard key and the matrix `runner:` label
+array. Everything else a repo used to hand-edit into the workflow now lives in
+three optional hooks the workflow calls if they are executable:
+
+| Hook                                   | When                              | What goes there                                                                           |
+| -------------------------------------- | --------------------------------- | ----------------------------------------------------------------------------------------- |
+| `scripts/integration/prepull-extra.sh` | before the suite, outside nix     | `docker pull` of images the suite launches beyond media + studio (a driver, a sink)       |
+| `scripts/integration/prepare.sh`       | before `bun run test:integration` | builds the suite needs on disk — a frontend dist the daemon fetches, an image it launches |
+| `scripts/integration/verify.sh`        | after the suite                   | post-suite checks; absent, a repo with `tests/demo.spec.ts` gets `bun run demo -- check`  |
+
+`bun run test:integration` is the product's claim of what its tier is, and
+ctl's release gate believes it: a `ctl-candidate` dispatch runs this workflow
+and the reported result decides whether `latest` moves.
+
+**Converging a repo.** The gate only takes over a copy whose first line is the
+canonical's marker (`# ctl-dev-kit convention: integration`); a copy without it
+is hand-owned and ungated, which is the state every repo started in. To
+converge: move the repo's bespoke steps into the hooks above, replace the
+workflow with the canonical (product and runner filled in), and prove it with
+`gh workflow run integration.yml --ref <branch>` before merging — the tier
+only fires on `main` and on dispatch, so a PR's checks never run it. Once the
+marker is in, `sync-dev-kit` keeps the copy current and `check:drift` names any
+hand edit.
+
+**Why the pipe is shaped the way it is.** The suite runs inside a shell
+function piped into `tee` so the log survives as an artifact, and the step's
+first line is `set -euo pipefail`. GitHub's default `run:` shell is `bash -e
+{0}` with no pipefail, so without it the step exits with tee's status and a
+dead suite reads green; a `set -euo pipefail` inside the nested `bash -c` body
+cannot reach the pipeline one shell up. Both turnkeys ran a void tier for weeks
+that way. `tee-pipefail.test.ts` guards every convention workflow for it.
+
+The workflow also uploads `test-results/` beside the log, and reaps the
+instances the harness recorded in `$NORSK_LAUNCHED_MANIFEST` after a cancelled
+or failed run, so one killed job cannot poison the next with a held port.

package/conventions/integration.yml

@@ -0,0 +1,346 @@
+# ctl-dev-kit convention: integration
+#
+# The integration tier, single-sourced in @norskvideo/ctl-dev-kit
+# (conventions/integration.yml) and drift-checked — the two per-repo lines are
+# the ci-status-dispatch `product:` key and the `runner:` label array. The
+# marker on line 1 is what puts a copy under the gate: a repo converges by
+# replacing its hand-owned workflow with this file (product and runner filled
+# in) plus whatever hooks below its suite needs, proven by a workflow_dispatch
+# run on the branch. Until a copy carries the marker it is hand-owned and
+# ungated — which is the state fleet review 04c s3.1 found every product in.
+#
+# What the tier is: the product's slow suite (daemon + Docker + licence) run
+# against the RELEASED norsk-ctl binary, exactly as a customer would run it — a
+# split product repo has no ctl source. `bun run test:integration` is the
+# product's claim of what that suite is, and ctl's release gate believes it: a
+# ctl-candidate dispatch runs this and the result decides whether `latest`
+# moves.
+#
+# Per-repo variation lives in three optional hooks, not in this file:
+#   scripts/integration/prepull-extra.sh  extra images the suite launches (a
+#                                         driver, a sink, a sidecar)
+#   scripts/integration/prepare.sh        built before the suite runs (a
+#                                         frontend dist the daemon fetches, an
+#                                         image the workflow launches)
+#   scripts/integration/verify.sh         after the suite; absent, a product
+#                                         with tests/demo.spec.ts gets
+#                                         `bun run demo -- check`
+#
+# The suite is piped into tee so its log survives as an artifact, and the step
+# sets pipefail FIRST: the default `bash -e {0}` has none, so without it the
+# step exits with tee's status and a dead suite reads green — both turnkeys
+# ran a void tier for weeks that way, feeding a green candidate result to the
+# release gate. tee-pipefail.test.ts guards the line; it ships with this file.
+#
+# Runner selection is a one-entry matrix carrying a JSON label array, because a
+# string like '[ARM64, nvidia]' in runs-on is ONE label, not two. Default
+# '["x64"]' — the docker-outside-of-docker pool, where launched containers are
+# host siblings and the harness reaches them by compose name on norsk-net (the
+# topology step below says so in NORSK_TEST_HOST/NORSK_TEST_NET); the GPU box
+# is '["ARM64","nvidia"]'.
+name: integration
+
+on:
+  repository_dispatch:
+    types: [ctl-candidate]
+  push:
+    branches: [main]
+  workflow_dispatch:
+    inputs:
+      ctl_channel:
+        description: "norsk-ctl S3 channel to test against (release-gate dispatches 'rc' to validate a candidate before promoting latest)."
+        type: string
+        default: latest
+
+permissions:
+  contents: read
+
+# Key the group on event_name so a push never cancels an in-flight ctl-candidate
+# validation: pushes cancel pushes, candidate validations cancel only newer
+# candidates. A cancelled validation would score as a red at release-collect.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
+  cancel-in-progress: true
+
+jobs:
+  integration:
+    strategy:
+      matrix:
+        include:
+          - runner: '["x64"]'
+    runs-on: ${{ fromJSON(matrix.runner) }}
+    steps:
+      # BEFORE checkout: launched instances leave root-owned bind-mount target
+      # dirs under test-temp/ that the non-root runner user cannot remove, so
+      # actions/checkout's own cleanup would fail the job before anything runs.
+      # Nuke them from a throwaway root container first. Best-effort.
+      - name: Clear stale root-owned test-temp (pre-checkout)
+        run: |
+          set -uo pipefail
+          tt="$GITHUB_WORKSPACE/test-temp"
+          [ -d "$tt" ] || exit 0
+          docker run --rm --user 0:0 -v "$tt":/t alpine sh \
+            -c 'rm -rf /t/* /t/.[!.]* 2>/dev/null || true' || rm -rf "$tt"/* 2>/dev/null || true
+
+      - uses: actions/checkout@v5
+        with:
+          clean: true
+
+      # A containerised runner (docker-outside-of-docker) reaches host-published
+      # ports via the host-gateway alias and can join norsk-net for direct reach;
+      # a bare-metal host runner (the GPU box) must stay on localhost + publish.
+      # Detect rather than hardcode, so one byte-identical file serves both.
+      # NORSK_CTL_PRODUCT_REACH rides along because this step has already
+      # established the fact the daemon would otherwise sniff from its own
+      # cgroup: `product add --image` loopback-publishes the control plane and
+      # polls it for readiness, and on a DooD runner that loopback is the docker
+      # HOST's, so registration times out at 60s with nothing in the log saying
+      # why. The daemon's own detection stays the default; here the probe that
+      # already knows simply says so.
+      - name: Probe runner docker topology and set NORSK_TEST_HOST + NORSK_TEST_NET
+        run: |
+          set -uo pipefail
+          echo "hostname=$(hostname)"
+          if docker inspect "$(hostname)" >/dev/null 2>&1; then
+            echo "TOPO: hostname resolves to a container -> runner is containerized (DooD)"
+            echo "NORSK_TEST_HOST=host.docker.internal" >> "$GITHUB_ENV"
+            echo "NORSK_TEST_NET=direct" >> "$GITHUB_ENV"
+            echo "NORSK_CTL_PRODUCT_REACH=container-address" >> "$GITHUB_ENV"
+            echo "NORSK_RUNNER_OWNS_DOCKER=false" >> "$GITHUB_ENV"
+          else
+            echo "TOPO: hostname is not a container -> runner is host-based"
+            echo "NORSK_TEST_HOST=localhost" >> "$GITHUB_ENV"
+            echo "NORSK_TEST_NET=publish" >> "$GITHUB_ENV"
+            echo "NORSK_CTL_PRODUCT_REACH=host" >> "$GITHUB_ENV"
+            echo "NORSK_RUNNER_OWNS_DOCKER=true" >> "$GITHUB_ENV"
+          fi
+
+      # `product add --image` publishes the control plane on 127.0.0.1:14321. The
+      # daemon means to reap it on shutdown (daemon.ts, model B) but only does so
+      # gracefully, and this workflow cancels its own superseded runs -- a killed
+      # daemon reaps nothing and the port stays held for every later job here.
+      # Kill by PORT rather than by name, prune the endpoints a removal can
+      # strand, and then PROVE the port binds instead of assuming it. Only where
+      # the docker daemon is this runner's own and jobs are serial; the GPU box
+      # runs several runners as sibling CONTAINERS against one host daemon and is
+      # excluded -- there, a blanket reap would kill a concurrent job.
+      - name: Reap a stranded product control plane
+        if: ${{ env.NORSK_RUNNER_OWNS_DOCKER == 'true' }}
+        run: |
+          set -uo pipefail
+          port=14321
+          bindable() { docker run --rm -p "127.0.0.1:$port:1" alpine true >/dev/null 2>&1; }
+
+          held="$(docker ps -a --filter "publish=$port" --format '{{.Names}} {{.Status}} {{.Image}}')"
+          if [ -n "$held" ]; then
+            echo "::warning::$port was held on entry by: $held"
+            docker rm -f $(docker ps -aq --filter "publish=$port") || true
+          fi
+          leftover="$(docker ps -aq --filter 'name=^norsk-product-')"
+          [ -z "$leftover" ] || docker rm -f $leftover || true
+
+          if ! bindable; then
+            echo "::warning::$port still not bindable after removing containers -- pruning dangling endpoints"
+            docker network prune -f || true
+          fi
+          if ! bindable; then
+            echo "::error::cannot bind 127.0.0.1:$port; the suite would fail later and more obscurely"
+            docker ps -a --filter "publish=$port" || true
+            exit 1
+          fi
+          echo "$port is bindable"
+
+      - name: Write the Norsk license (from the org secret)
+        env:
+          NORSK_LICENSE_V2: ${{ secrets.NORSK_LICENSE_V2 }}
+        run: printf '%s' "$NORSK_LICENSE_V2" > "$RUNNER_TEMP/norsk-license.json"
+
+      # Pick the binary for THIS runner's architecture: the GPU box is aarch64,
+      # the x64 pool is x86_64, and the wrong one dies with "Exec format error"
+      # deep in the suite. No `|| true` on --version: a bad binary must fail here.
+      - name: Download the released norsk-ctl binary
+        run: |
+          set -euo pipefail
+          channel="${{ github.event.client_payload.channel || inputs.ctl_channel || 'latest' }}"
+          case "$(uname -m)" in
+            x86_64) arch=linux-x64 ;;
+            aarch64 | arm64) arch=linux-arm64 ;;
+            *) echo "unsupported runner arch: $(uname -m)" >&2; exit 1 ;;
+          esac
+          S3="https://s3.eu-west-1.amazonaws.com/norsk.video/norsk-ctl"
+          ver="$(curl -fsSL --retry 3 --retry-all-errors --retry-delay 2 "$S3/$channel")"
+          echo "norsk-ctl $channel channel -> $ver ($arch)"
+          curl -fsSL --retry 3 --retry-all-errors --retry-delay 2 "$S3/$ver/norsk-ctl-$ver-$arch" -o "$RUNNER_TEMP/norsk-ctl"
+          chmod +x "$RUNNER_TEMP/norsk-ctl"
+          "$RUNNER_TEMP/norsk-ctl" --version
+
+      # Everything below runs dev-kit and test-harness code out of node_modules,
+      # and `clean: true` on the checkout means this job starts without any.
+      - name: Install dependencies
+        run: nix develop .#build --command bun install --frozen-lockfile
+
+      # Before anything is built or launched: does this licence actually name
+      # this product? `product add` refuses one it does not, and that refusal
+      # reaches the log as a 60-second readiness timeout indistinguishable from
+      # a product that failed to start. reuters ran four "successful" hourly
+      # tiers dead at exactly that refusal (04c s3.1).
+      #
+      # The product name is the repo's own PRODUCT_NAME, not the short board key
+      # above (`funke` vs `funke-pegasus`): the licence names the product.
+      - name: Check the licence entitles this product
+        run: |
+          set -euo pipefail
+          product="$(sed -nE 's/.*PRODUCT_NAME[[:space:]]*=[[:space:]]*"([^"]+)".*/\1/p' shared/src/version.ts | head -1)"
+          [ -n "$product" ] || { echo "::error::could not read PRODUCT_NAME from shared/src/version.ts"; exit 1; }
+          nix develop .#build --command bun \
+            node_modules/@norskvideo/ctl-dev-kit/licence/check-entitlement.ts \
+            "$RUNNER_TEMP/norsk-license.json" "$product"
+
+      # Instance launch docker-composes the media + studio images; on a cold
+      # runner the first pull overruns the per-test timeout. Pre-pull them (tags
+      # from manifest.seed.json), plus whatever the repo lists in the optional
+      # scripts/integration/prepull-extra.sh hook (a driver image, a sink).
+      - name: Pre-pull the media + studio images
+        run: |
+          set -euo pipefail
+          for img in "$(jq -r '.latest.media' manifest.seed.json)" "$(jq -r '.latest.studio' manifest.seed.json)"; do
+            echo "pre-pulling $img"
+            docker pull "$img"
+          done
+          if [ -x scripts/integration/prepull-extra.sh ]; then
+            echo "running scripts/integration/prepull-extra.sh"
+            ./scripts/integration/prepull-extra.sh
+          fi
+
+      # The launched-instances manifest is what the reap step below reads: the
+      # harness appends every instance id it launches, so a run that dies
+      # mid-suite still names what it left behind.
+      - name: Run integration suite
+        env:
+          NORSK_CTL_BINARY: ${{ runner.temp }}/norsk-ctl
+          NORSK_LICENSE_FILE: ${{ runner.temp }}/norsk-license.json
+          NORSK_LAUNCHED_MANIFEST: ${{ runner.temp }}/norsk-launched-instances
+        run: |
+          set -euo pipefail
+          suite() {
+            nix develop .#build --command bash -c '
+              set -euo pipefail
+              # A frozen install does not reliably relink a hoisted package whose
+              # version moved: a stale test-harness once survived a bump and a new
+              # export was "not found" at import. Drop the harness copy so the
+              # pinned version runs.
+              rm -rf node_modules/@norskvideo/ctl-test-harness
+              bun install --frozen-lockfile
+              if [ -x scripts/integration/prepare.sh ]; then
+                echo "running scripts/integration/prepare.sh"
+                ./scripts/integration/prepare.sh
+              fi
+              bun run test:integration
+              if [ -x scripts/integration/verify.sh ]; then
+                echo "running scripts/integration/verify.sh"
+                ./scripts/integration/verify.sh
+              elif [ -f tests/demo.spec.ts ]; then
+                bun run demo -- check
+              fi
+            '
+          }
+          suite 2>&1 | tee "$RUNNER_TEMP/integration.log"
+
+      # The log AND the harness evidence. Playwright and the harness write
+      # screenshots, traces and captured container logs under test-results/;
+      # a workflow that uploads only its stdout leaves the reader to re-run the
+      # tier for evidence that already existed on the runner.
+      - name: Upload the run log and test evidence
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: integration-attempt-${{ github.run_attempt }}
+          path: |
+            ${{ runner.temp }}/integration.log
+            test-results/
+          retention-days: 7
+          if-no-files-found: warn
+
+      - name: Reap this run's leftover instance containers
+        if: always()
+        env:
+          NORSK_LAUNCHED_MANIFEST: ${{ runner.temp }}/norsk-launched-instances
+        run: |
+          set -uo pipefail
+          m="$NORSK_LAUNCHED_MANIFEST"
+          [ -f "$m" ] || { echo "no launched-instances manifest -- nothing to reap"; exit 0; }
+          sort -u "$m" | while IFS= read -r id; do
+            [ -n "$id" ] || continue
+            ids="$(docker ps -aq --filter "label=norsk-ctl.instance=$id" || true)"
+            if [ -n "$ids" ]; then
+              echo "reaping leftover instance '$id': $ids"
+              docker rm -f $ids || true
+            fi
+          done
+          rm -f "$m" || true
+
+  # Report this pipeline's result to the aggregated product CI dashboard
+  # (id3as/ci-workflows). !cancelled() so a real pass/fail reports, but a
+  # cancelled/superseded run does NOT — a cancellation dispatched as failure
+  # would false-red the board. Candidate validations report to ctl instead.
+  notify:
+    needs: integration
+    if: ${{ !cancelled() && github.event_name != 'repository_dispatch' }}
+    runs-on: x64
+    steps:
+      # Same reason as the first checkout in this file: clear root-owned leftovers
+      # from a root container so the checkout below can clean properly. Best-effort
+      # by construction -- no docker, no test-temp, or a failed run all fall through.
+      - name: Clear stale root-owned test-temp (pre-checkout)
+        run: |
+          set -uo pipefail
+          tt="$GITHUB_WORKSPACE/test-temp"
+          [ -d "$tt" ] || exit 0
+          docker run --rm --user 0:0 -v "$tt":/t alpine sh \
+            -c 'rm -rf /t/* /t/.[!.]* 2>/dev/null || true' || rm -rf "$tt"/* 2>/dev/null || true
+
+      - uses: actions/checkout@v5
+        with:
+          clean: true
+      - id: meta
+        run: |
+          if [ "${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}" = "true" ]; then
+            echo "state=failure" >> "$GITHUB_OUTPUT"
+          else
+            echo "state=success" >> "$GITHUB_OUTPUT"
+          fi
+      - uses: ./.github/actions/ci-status-dispatch
+        with:
+          token: ${{ secrets.CI_DISPATCH_TOKEN }}
+          product: __PRODUCT__
+          pipeline: integration
+          status: ${{ steps.meta.outputs.state }}
+
+  # Report the rc-candidate validation result to the norsk-ctl release gate. Only
+  # on a ctl-candidate dispatch; release-collect tallies these and promotes
+  # latest once every gating product is green on the candidate.
+  report-candidate:
+    needs: integration
+    if: ${{ always() && github.event_name == 'repository_dispatch' && github.event.action == 'ctl-candidate' }}
+    runs-on: x64
+    steps:
+      - name: Fire ctl-candidate-result at norsk-ctl
+        env:
+          GH_TOKEN: ${{ secrets.CI_DISPATCH_TOKEN }}
+          # Lowercase on purpose: the drift gate masks every `product:` line, so
+          # one per-repo key serves both the notify and the report jobs.
+          product: __PRODUCT__
+          VERSION: ${{ github.event.client_payload.version }}
+          RESULT: ${{ needs.integration.result }}
+        run: |
+          set -euo pipefail
+          state=failure; [ "$RESULT" = "success" ] && state=success
+          # `tier: integration`, named rather than defaulted: the release gate
+          # tallies per tier, and this is the one that gates (docs/release-gate.md).
+          body="$(jq -nc --arg p "$product" --arg v "$VERSION" --arg r "$state" \
+            '{event_type:"ctl-candidate-result",client_payload:{product:$p,version:$v,result:$r,tier:"integration"}}')"
+          code="$(curl -sS -o /dev/null -w '%{http_code}' -X POST \
+            -H "Authorization: Bearer $GH_TOKEN" \
+            -H "Accept: application/vnd.github+json" \
+            https://api.github.com/repos/id3as/norsk-ctl/dispatches -d "$body")"
+          [ "$code" = "204" ] || { echo "::error::report to norsk-ctl failed (HTTP $code)"; exit 1; }

package/conventions/sync-drift.ts

@@ -25,6 +25,7 @@ import {
   type CanonicalBytes,
   CTL_HASH_LINE,
   CTL_VERSION_LINE,
+  carriesIntegrationMarker,
   END,
   GITIGNORE_BEGIN,
   GITIGNORE_END,
@@ -116,6 +117,33 @@ function syncSmoke(repoRoot: string, canonical: string, r: SyncReport): void {
   writeIfChanged(path, out, rel, r.written);
 }
 
+// integration.yml: smoke.yml's twin, but only a copy that already carries the
+// convention marker is rewritten — a hand-owned one is left untouched and named,
+// because converging it is a deliberate per-repo change (hooks, a proving run),
+// not something a nightly sync may do to a tier the release gate believes.
+function syncIntegration(repoRoot: string, canonical: string, r: SyncReport): void {
+  const rel = ".github/workflows/integration.yml";
+  const path = join(repoRoot, rel);
+  if (!existsSync(path)) {
+    r.skipped.push(`${rel} (absent)`);
+    return;
+  }
+  const existing = readFileSync(path, "utf8");
+  if (!carriesIntegrationMarker(existing)) {
+    r.skipped.push(`${rel} (hand-owned: no convention marker on line 1 — converge it deliberately)`);
+    return;
+  }
+  const key = existing.match(/^\s*product:\s*(\S+)/m)?.[1];
+  if (!key || key === PRODUCT_SENTINEL) {
+    r.skipped.push(`${rel} (no product: key to preserve)`);
+    return;
+  }
+  const runnerLine = existing.match(/^\s*-?\s*runner:.*$/m)?.[0];
+  let out = canonical.replace(PRODUCT_LINE, `$1${key}`);
+  if (runnerLine !== undefined) out = out.replace(RUNNER_LINE, () => runnerLine);
+  writeIfChanged(path, out, rel, r.written);
+}
+
 // flake.nix is verbatim except the dev-only ctl pin (ctlVersion + the four
 // per-platform hashes), which floats per repo. Re-emit canonical STRUCTURE with
 // the repo's pin restored in order — so a structural edit (e.g. a stray comment)
@@ -250,6 +278,7 @@ export function syncDrift(repoRoot: string, canonical: CanonicalBytes): SyncRepo
   );
   syncBuildImage(repoRoot, canonical.buildImage, r);
   syncSmoke(repoRoot, canonical.smoke, r);
+  syncIntegration(repoRoot, canonical.integration, r);
 
   // publish-docs.yml is optional and verbatim (no per-repo line) — only re-sync a
   // repo that already carries it, unlike biome/tsconfig which every repo must have.
@@ -294,6 +323,7 @@ if (import.meta.main) {
     gitignoreCore: readFileSync(join(dir, "gitignore.core"), "utf8"),
     buildImage: readFileSync(join(dir, "build-image.yml"), "utf8"),
     smoke: readFileSync(join(dir, "smoke.yml"), "utf8"),
+    integration: readFileSync(join(dir, "integration.yml"), "utf8"),
     demoShim: readFileSync(join(dir, "demo.sh"), "utf8"),
   };
   const report = syncDrift(repoRoot, canonical);

package/conventions/workflow-shell.ts

@@ -25,3 +25,33 @@ export function bashBlockLines(yaml: string): Set<number> {
   });
   return inside;
 }
+
+/** The `run: |` steps that pipe into `tee` without `pipefail` set on the
+ *  block's first non-comment line — each reported at its `run: |` line. The
+ *  default `bash -e {0}` has no pipefail, so such a step exits with TEE's
+ *  status and a dead suite reads green; `set -euo pipefail` inside a nested
+ *  `bash -c '...'` body cannot reach the pipeline one shell up. Read from the
+ *  raw text, not a YAML parse: what matters is the ORDER of lines in the script
+ *  the runner executes. */
+export function teeWithoutPipefail(yaml: string): { line: number; text: string }[] {
+  const out: { line: number; text: string }[] = [];
+  const lines = yaml.split("\n");
+  for (let i = 0; i < lines.length; i++) {
+    const head = lines[i] as string;
+    if (!/^\s*run: \|\s*$/.test(head)) continue;
+    const indent = (lines[i + 1] ?? "").match(/^\s*/)?.[0].length ?? 0;
+    const body: string[] = [];
+    for (let j = i + 1; j < lines.length; j++) {
+      const line = lines[j] as string;
+      if (line.trim() === "") continue;
+      if ((line.match(/^\s*/)?.[0].length ?? 0) < indent) break;
+      body.push(line.trim());
+    }
+    if (!body.some((l) => /\|\s*tee\b/.test(l))) continue;
+    const first = body.find((l) => !l.startsWith("#"));
+    const pipefailFirst =
+      first !== undefined && /^set\s+(-[a-zA-Z]*o\s+pipefail|-euo\s+pipefail|-o\s+pipefail)/.test(first);
+    if (!pipefailFirst) out.push({ line: i + 1, text: head });
+  }
+  return out;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@norskvideo/ctl-dev-kit",
-  "version": "0.1.84",
+  "version": "0.1.85",
   "type": "module",
   "exports": {
     "./create-product": "./create-product/create-product.ts",