@norskvideo/ctl-dev-kit 0.1.84 → 0.1.86
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/conventions/build-docs.yml +102 -21
- package/conventions/build-image.yml +204 -52
- package/conventions/check-drift.ts +37 -0
- package/conventions/checks.yml +153 -39
- package/conventions/integration-testing.md +40 -0
- package/conventions/integration.yml +435 -0
- package/conventions/publish-docs.yml +102 -21
- package/conventions/smoke.yml +102 -13
- package/conventions/sync-ctl-packages.yml +51 -13
- package/conventions/sync-dev-kit.yml +102 -26
- package/conventions/sync-drift.ts +30 -0
- package/conventions/upgrade-latest.yml +102 -26
- package/conventions/workflow-shell.ts +136 -0
- package/package.json +1 -1
|
@@ -0,0 +1,435 @@
|
|
|
1
|
+
# ctl-dev-kit convention: integration
|
|
2
|
+
#
|
|
3
|
+
# The integration tier, single-sourced in @norskvideo/ctl-dev-kit
|
|
4
|
+
# (conventions/integration.yml) and drift-checked — the two per-repo lines are
|
|
5
|
+
# the ci-status-dispatch `product:` key and the `runner:` label array. The
|
|
6
|
+
# marker on line 1 is what puts a copy under the gate: a repo converges by
|
|
7
|
+
# replacing its hand-owned workflow with this file (product and runner filled
|
|
8
|
+
# in) plus whatever hooks below its suite needs, proven by a workflow_dispatch
|
|
9
|
+
# run on the branch. Until a copy carries the marker it is hand-owned and
|
|
10
|
+
# ungated — which is the state fleet review 04c s3.1 found every product in.
|
|
11
|
+
#
|
|
12
|
+
# What the tier is: the product's slow suite (daemon + Docker + licence) run
|
|
13
|
+
# against the RELEASED norsk-ctl binary, exactly as a customer would run it — a
|
|
14
|
+
# split product repo has no ctl source. `bun run test:integration` is the
|
|
15
|
+
# product's claim of what that suite is, and ctl's release gate believes it: a
|
|
16
|
+
# ctl-candidate dispatch runs this and the result decides whether `latest`
|
|
17
|
+
# moves.
|
|
18
|
+
#
|
|
19
|
+
# Per-repo variation lives in three optional hooks, not in this file:
|
|
20
|
+
# scripts/integration/prepull-extra.sh extra images the suite launches (a
|
|
21
|
+
# driver, a sink, a sidecar)
|
|
22
|
+
# scripts/integration/prepare.sh built before the suite runs (a
|
|
23
|
+
# frontend dist the daemon fetches, an
|
|
24
|
+
# image the workflow launches)
|
|
25
|
+
# scripts/integration/verify.sh after the suite; absent, a product
|
|
26
|
+
# with tests/demo.spec.ts gets
|
|
27
|
+
# `bun run demo -- check`
|
|
28
|
+
#
|
|
29
|
+
# The suite is piped into tee so its log survives as an artifact, and the step
|
|
30
|
+
# sets pipefail FIRST: the default `bash -e {0}` has none, so without it the
|
|
31
|
+
# step exits with tee's status and a dead suite reads green — both turnkeys
|
|
32
|
+
# ran a void tier for weeks that way, feeding a green candidate result to the
|
|
33
|
+
# release gate. tee-pipefail.test.ts guards the line; it ships with this file.
|
|
34
|
+
#
|
|
35
|
+
# Runner selection is a one-entry matrix carrying a JSON label array, because a
|
|
36
|
+
# string like '[ARM64, nvidia]' in runs-on is ONE label, not two. Default
|
|
37
|
+
# '["x64"]' — the docker-outside-of-docker pool, where launched containers are
|
|
38
|
+
# host siblings and the harness reaches them by compose name on norsk-net (the
|
|
39
|
+
# topology step below says so in NORSK_TEST_HOST/NORSK_TEST_NET); the GPU box
|
|
40
|
+
# is '["ARM64","nvidia"]'.
|
|
41
|
+
name: integration
|
|
42
|
+
|
|
43
|
+
on:
|
|
44
|
+
repository_dispatch:
|
|
45
|
+
types: [ctl-candidate]
|
|
46
|
+
push:
|
|
47
|
+
branches: [main]
|
|
48
|
+
workflow_dispatch:
|
|
49
|
+
inputs:
|
|
50
|
+
ctl_channel:
|
|
51
|
+
description: "norsk-ctl S3 channel to test against (release-gate dispatches 'rc' to validate a candidate before promoting latest)."
|
|
52
|
+
type: string
|
|
53
|
+
default: latest
|
|
54
|
+
|
|
55
|
+
permissions:
|
|
56
|
+
contents: read
|
|
57
|
+
|
|
58
|
+
# Key the group on event_name so a push never cancels an in-flight ctl-candidate
|
|
59
|
+
# validation: pushes cancel pushes, candidate validations cancel only newer
|
|
60
|
+
# candidates. A cancelled validation would score as a red at release-collect.
|
|
61
|
+
concurrency:
|
|
62
|
+
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
|
|
63
|
+
cancel-in-progress: true
|
|
64
|
+
|
|
65
|
+
jobs:
|
|
66
|
+
integration:
|
|
67
|
+
strategy:
|
|
68
|
+
matrix:
|
|
69
|
+
include:
|
|
70
|
+
- runner: '["x64"]'
|
|
71
|
+
runs-on: ${{ fromJSON(matrix.runner) }}
|
|
72
|
+
steps:
|
|
73
|
+
# Runs BEFORE checkout, and does two things: points the harness's temp base
|
|
74
|
+
# out of the workspace for every later step in this job, and clears whatever
|
|
75
|
+
# the old repo-local default left behind on this runner. The body carries the
|
|
76
|
+
# why. It fails the job only when legacy litter survives -- which would take
|
|
77
|
+
# the checkout down a step later anyway, but report nothing useful.
|
|
78
|
+
- name: Route test-temp out of the workspace (pre-checkout)
|
|
79
|
+
run: |
|
|
80
|
+
set -uo pipefail
|
|
81
|
+
# These runners are docker-OUTSIDE-of-docker, so a bind-mount source the
|
|
82
|
+
# HOST daemon has not seen before is created by dockerd, as ROOT, on the
|
|
83
|
+
# host -- even when the launch passes --container-user (reproduced
|
|
84
|
+
# 2026-09-04). The harness's default base is repo-local for an OrbStack
|
|
85
|
+
# inotify constraint that binds macOS dev and nothing here, so every run
|
|
86
|
+
# seeded the CHECKOUT with root-owned dirs a non-root `git clean -ffdx`
|
|
87
|
+
# cannot remove. That took commentary's checks AND integration red on
|
|
88
|
+
# 2026-09-03. Use the sanctioned override, per-run and outside the
|
|
89
|
+
# checkout -- as 8ff17a8d did in June for one debug workflow, and stopped.
|
|
90
|
+
base="$RUNNER_TEMP/nctt-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT"
|
|
91
|
+
mkdir -p "$base"
|
|
92
|
+
echo "NORSK_CTL_TEST_TMP=$base" >> "$GITHUB_ENV"
|
|
93
|
+
# Yesterday's bases hold dockerd-created root dirs, so a non-root rm
|
|
94
|
+
# EACCESes. They cannot wedge a checkout, but they do fill the disk.
|
|
95
|
+
for old in $(find "$RUNNER_TEMP" -maxdepth 1 -name 'nctt-*' -mtime +0 -printf '%f\n' 2>/dev/null || true); do
|
|
96
|
+
docker run --rm --user 0:0 -v "$RUNNER_TEMP":/t alpine sh -c 'rm -rf -- "/t/$1"' sh "$old" >/dev/null 2>&1 || true
|
|
97
|
+
done
|
|
98
|
+
# LEGACY: a runner that ran the old default still carries a workspace
|
|
99
|
+
# test-temp, and a checkout with clean:true dies on it. Reap the holders
|
|
100
|
+
# first -- rm cannot unlink a live mountpoint even as root (EBUSY), and a
|
|
101
|
+
# RESTARTING leaked container re-creates the path as root seconds after
|
|
102
|
+
# any clean. Match on mount SOURCE, not a label: the holders are untracked
|
|
103
|
+
# by construction, because a killed run labels nothing.
|
|
104
|
+
tt="$GITHUB_WORKSPACE/test-temp"
|
|
105
|
+
[ -d "$tt" ] || exit 0
|
|
106
|
+
stuck=""
|
|
107
|
+
for c in $(docker ps -aq 2>/dev/null || true); do
|
|
108
|
+
if docker inspect -f '{{range .Mounts}}{{println .Source}}{{end}}' "$c" 2>/dev/null | grep -q "^$tt/"; then
|
|
109
|
+
stuck="$stuck $c"
|
|
110
|
+
fi
|
|
111
|
+
done
|
|
112
|
+
if [ -n "$stuck" ]; then
|
|
113
|
+
echo "reaping containers holding mounts under $tt:$stuck"
|
|
114
|
+
docker rm -f $stuck || true
|
|
115
|
+
fi
|
|
116
|
+
# The DIRECTORY, not just its contents: an empty but root-owned base
|
|
117
|
+
# still EACCESes a later mkdtemp. Mounting the PARENT is what lets a root
|
|
118
|
+
# container unlink the leaf.
|
|
119
|
+
docker run --rm --user 0:0 -v "$GITHUB_WORKSPACE":/w alpine \
|
|
120
|
+
sh -c 'rm -rf /w/test-temp' || rm -rf "$tt" || true
|
|
121
|
+
# Fail HERE if it survived. The checkout fails on it either way, but
|
|
122
|
+
# reports only an EACCES rmdir with no clue what was holding the path.
|
|
123
|
+
if [ -e "$tt" ]; then
|
|
124
|
+
echo "::error::$tt survived the pre-checkout clean"
|
|
125
|
+
ls -lan "$tt" || true
|
|
126
|
+
exit 1
|
|
127
|
+
fi
|
|
128
|
+
|
|
129
|
+
- uses: actions/checkout@v5
|
|
130
|
+
with:
|
|
131
|
+
clean: true
|
|
132
|
+
|
|
133
|
+
# A containerised runner (docker-outside-of-docker) reaches host-published
|
|
134
|
+
# ports via the host-gateway alias and can join norsk-net for direct reach;
|
|
135
|
+
# a bare-metal host runner (the GPU box) must stay on localhost + publish.
|
|
136
|
+
# Detect rather than hardcode, so one byte-identical file serves both.
|
|
137
|
+
# NORSK_CTL_PRODUCT_REACH rides along because this step has already
|
|
138
|
+
# established the fact the daemon would otherwise sniff from its own
|
|
139
|
+
# cgroup: `product add --image` loopback-publishes the control plane and
|
|
140
|
+
# polls it for readiness, and on a DooD runner that loopback is the docker
|
|
141
|
+
# HOST's, so registration times out at 60s with nothing in the log saying
|
|
142
|
+
# why. The daemon's own detection stays the default; here the probe that
|
|
143
|
+
# already knows simply says so.
|
|
144
|
+
- name: Probe runner docker topology and set NORSK_TEST_HOST + NORSK_TEST_NET
|
|
145
|
+
run: |
|
|
146
|
+
set -uo pipefail
|
|
147
|
+
echo "hostname=$(hostname)"
|
|
148
|
+
if docker inspect "$(hostname)" >/dev/null 2>&1; then
|
|
149
|
+
echo "TOPO: hostname resolves to a container -> runner is containerized (DooD)"
|
|
150
|
+
echo "NORSK_TEST_HOST=host.docker.internal" >> "$GITHUB_ENV"
|
|
151
|
+
echo "NORSK_TEST_NET=direct" >> "$GITHUB_ENV"
|
|
152
|
+
echo "NORSK_CTL_PRODUCT_REACH=container-address" >> "$GITHUB_ENV"
|
|
153
|
+
echo "NORSK_RUNNER_OWNS_DOCKER=false" >> "$GITHUB_ENV"
|
|
154
|
+
else
|
|
155
|
+
echo "TOPO: hostname is not a container -> runner is host-based"
|
|
156
|
+
echo "NORSK_TEST_HOST=localhost" >> "$GITHUB_ENV"
|
|
157
|
+
echo "NORSK_TEST_NET=publish" >> "$GITHUB_ENV"
|
|
158
|
+
echo "NORSK_CTL_PRODUCT_REACH=host" >> "$GITHUB_ENV"
|
|
159
|
+
echo "NORSK_RUNNER_OWNS_DOCKER=true" >> "$GITHUB_ENV"
|
|
160
|
+
fi
|
|
161
|
+
|
|
162
|
+
# `product add --image` publishes the control plane on 127.0.0.1:14321. The
|
|
163
|
+
# daemon means to reap it on shutdown (daemon.ts, model B) but only does so
|
|
164
|
+
# gracefully, and this workflow cancels its own superseded runs -- a killed
|
|
165
|
+
# daemon reaps nothing and the port stays held for every later job here.
|
|
166
|
+
# Kill by PORT rather than by name, prune the endpoints a removal can
|
|
167
|
+
# strand, and then PROVE the port binds instead of assuming it. Only where
|
|
168
|
+
# the docker daemon is this runner's own and jobs are serial; the GPU box
|
|
169
|
+
# runs several runners as sibling CONTAINERS against one host daemon and is
|
|
170
|
+
# excluded -- there, a blanket reap would kill a concurrent job.
|
|
171
|
+
- name: Reap a stranded product control plane
|
|
172
|
+
if: ${{ env.NORSK_RUNNER_OWNS_DOCKER == 'true' }}
|
|
173
|
+
run: |
|
|
174
|
+
set -uo pipefail
|
|
175
|
+
port=14321
|
|
176
|
+
bindable() { docker run --rm -p "127.0.0.1:$port:1" alpine true >/dev/null 2>&1; }
|
|
177
|
+
|
|
178
|
+
held="$(docker ps -a --filter "publish=$port" --format '{{.Names}} {{.Status}} {{.Image}}')"
|
|
179
|
+
if [ -n "$held" ]; then
|
|
180
|
+
echo "::warning::$port was held on entry by: $held"
|
|
181
|
+
docker rm -f $(docker ps -aq --filter "publish=$port") || true
|
|
182
|
+
fi
|
|
183
|
+
leftover="$(docker ps -aq --filter 'name=^norsk-product-')"
|
|
184
|
+
[ -z "$leftover" ] || docker rm -f $leftover || true
|
|
185
|
+
|
|
186
|
+
if ! bindable; then
|
|
187
|
+
echo "::warning::$port still not bindable after removing containers -- pruning dangling endpoints"
|
|
188
|
+
docker network prune -f || true
|
|
189
|
+
fi
|
|
190
|
+
if ! bindable; then
|
|
191
|
+
echo "::error::cannot bind 127.0.0.1:$port; the suite would fail later and more obscurely"
|
|
192
|
+
docker ps -a --filter "publish=$port" || true
|
|
193
|
+
exit 1
|
|
194
|
+
fi
|
|
195
|
+
echo "$port is bindable"
|
|
196
|
+
|
|
197
|
+
- name: Write the Norsk license (from the org secret)
|
|
198
|
+
env:
|
|
199
|
+
NORSK_LICENSE_V2: ${{ secrets.NORSK_LICENSE_V2 }}
|
|
200
|
+
run: printf '%s' "$NORSK_LICENSE_V2" > "$RUNNER_TEMP/norsk-license.json"
|
|
201
|
+
|
|
202
|
+
# Pick the binary for THIS runner's architecture: the GPU box is aarch64,
|
|
203
|
+
# the x64 pool is x86_64, and the wrong one dies with "Exec format error"
|
|
204
|
+
# deep in the suite. No `|| true` on --version: a bad binary must fail here.
|
|
205
|
+
- name: Download the released norsk-ctl binary
|
|
206
|
+
run: |
|
|
207
|
+
set -euo pipefail
|
|
208
|
+
channel="${{ github.event.client_payload.channel || inputs.ctl_channel || 'latest' }}"
|
|
209
|
+
case "$(uname -m)" in
|
|
210
|
+
x86_64) arch=linux-x64 ;;
|
|
211
|
+
aarch64 | arm64) arch=linux-arm64 ;;
|
|
212
|
+
*) echo "unsupported runner arch: $(uname -m)" >&2; exit 1 ;;
|
|
213
|
+
esac
|
|
214
|
+
S3="https://s3.eu-west-1.amazonaws.com/norsk.video/norsk-ctl"
|
|
215
|
+
ver="$(curl -fsSL --retry 3 --retry-all-errors --retry-delay 2 "$S3/$channel")"
|
|
216
|
+
echo "norsk-ctl $channel channel -> $ver ($arch)"
|
|
217
|
+
curl -fsSL --retry 3 --retry-all-errors --retry-delay 2 "$S3/$ver/norsk-ctl-$ver-$arch" -o "$RUNNER_TEMP/norsk-ctl"
|
|
218
|
+
chmod +x "$RUNNER_TEMP/norsk-ctl"
|
|
219
|
+
"$RUNNER_TEMP/norsk-ctl" --version
|
|
220
|
+
|
|
221
|
+
# Everything below runs dev-kit and test-harness code out of node_modules,
|
|
222
|
+
# and `clean: true` on the checkout means this job starts without any.
|
|
223
|
+
- name: Install dependencies
|
|
224
|
+
run: nix develop .#build --command bun install --frozen-lockfile
|
|
225
|
+
|
|
226
|
+
# Before anything is built or launched: does this licence actually name
|
|
227
|
+
# this product? `product add` refuses one it does not, and that refusal
|
|
228
|
+
# reaches the log as a 60-second readiness timeout indistinguishable from
|
|
229
|
+
# a product that failed to start. reuters ran four "successful" hourly
|
|
230
|
+
# tiers dead at exactly that refusal (04c s3.1).
|
|
231
|
+
#
|
|
232
|
+
# The product name is the repo's own PRODUCT_NAME, not the short board key
|
|
233
|
+
# above (`funke` vs `funke-pegasus`): the licence names the product.
|
|
234
|
+
- name: Check the licence entitles this product
|
|
235
|
+
run: |
|
|
236
|
+
set -euo pipefail
|
|
237
|
+
product="$(sed -nE 's/.*PRODUCT_NAME[[:space:]]*=[[:space:]]*"([^"]+)".*/\1/p' shared/src/version.ts | head -1)"
|
|
238
|
+
[ -n "$product" ] || { echo "::error::could not read PRODUCT_NAME from shared/src/version.ts"; exit 1; }
|
|
239
|
+
nix develop .#build --command bun \
|
|
240
|
+
node_modules/@norskvideo/ctl-dev-kit/licence/check-entitlement.ts \
|
|
241
|
+
"$RUNNER_TEMP/norsk-license.json" "$product"
|
|
242
|
+
|
|
243
|
+
# Instance launch docker-composes the media + studio images; on a cold
|
|
244
|
+
# runner the first pull overruns the per-test timeout. Pre-pull them (tags
|
|
245
|
+
# from manifest.seed.json), plus whatever the repo lists in the optional
|
|
246
|
+
# scripts/integration/prepull-extra.sh hook (a driver image, a sink).
|
|
247
|
+
- name: Pre-pull the media + studio images
|
|
248
|
+
run: |
|
|
249
|
+
set -euo pipefail
|
|
250
|
+
for img in "$(jq -r '.latest.media' manifest.seed.json)" "$(jq -r '.latest.studio' manifest.seed.json)"; do
|
|
251
|
+
echo "pre-pulling $img"
|
|
252
|
+
docker pull "$img"
|
|
253
|
+
done
|
|
254
|
+
if [ -x scripts/integration/prepull-extra.sh ]; then
|
|
255
|
+
echo "running scripts/integration/prepull-extra.sh"
|
|
256
|
+
./scripts/integration/prepull-extra.sh
|
|
257
|
+
fi
|
|
258
|
+
|
|
259
|
+
# The launched-instances manifest is what the reap step below reads: the
|
|
260
|
+
# harness appends every instance id it launches, so a run that dies
|
|
261
|
+
# mid-suite still names what it left behind.
|
|
262
|
+
- name: Run integration suite
|
|
263
|
+
env:
|
|
264
|
+
NORSK_CTL_BINARY: ${{ runner.temp }}/norsk-ctl
|
|
265
|
+
NORSK_LICENSE_FILE: ${{ runner.temp }}/norsk-license.json
|
|
266
|
+
NORSK_LAUNCHED_MANIFEST: ${{ runner.temp }}/norsk-launched-instances
|
|
267
|
+
run: |
|
|
268
|
+
set -euo pipefail
|
|
269
|
+
suite() {
|
|
270
|
+
nix develop .#build --command bash -c '
|
|
271
|
+
set -euo pipefail
|
|
272
|
+
# A frozen install does not reliably relink a hoisted package whose
|
|
273
|
+
# version moved: a stale test-harness once survived a bump and a new
|
|
274
|
+
# export was "not found" at import. Drop the harness copy so the
|
|
275
|
+
# pinned version runs.
|
|
276
|
+
rm -rf node_modules/@norskvideo/ctl-test-harness
|
|
277
|
+
bun install --frozen-lockfile
|
|
278
|
+
if [ -x scripts/integration/prepare.sh ]; then
|
|
279
|
+
echo "running scripts/integration/prepare.sh"
|
|
280
|
+
./scripts/integration/prepare.sh
|
|
281
|
+
fi
|
|
282
|
+
bun run test:integration
|
|
283
|
+
if [ -x scripts/integration/verify.sh ]; then
|
|
284
|
+
echo "running scripts/integration/verify.sh"
|
|
285
|
+
./scripts/integration/verify.sh
|
|
286
|
+
elif [ -f tests/demo.spec.ts ]; then
|
|
287
|
+
bun run demo -- check
|
|
288
|
+
fi
|
|
289
|
+
'
|
|
290
|
+
}
|
|
291
|
+
suite 2>&1 | tee "$RUNNER_TEMP/integration.log"
|
|
292
|
+
|
|
293
|
+
# The log AND the harness evidence. Playwright and the harness write
|
|
294
|
+
# screenshots, traces and captured container logs under test-results/;
|
|
295
|
+
# a workflow that uploads only its stdout leaves the reader to re-run the
|
|
296
|
+
# tier for evidence that already existed on the runner.
|
|
297
|
+
- name: Upload the run log and test evidence
|
|
298
|
+
if: always()
|
|
299
|
+
uses: actions/upload-artifact@v4
|
|
300
|
+
with:
|
|
301
|
+
name: integration-attempt-${{ github.run_attempt }}
|
|
302
|
+
path: |
|
|
303
|
+
${{ runner.temp }}/integration.log
|
|
304
|
+
test-results/
|
|
305
|
+
retention-days: 7
|
|
306
|
+
if-no-files-found: warn
|
|
307
|
+
|
|
308
|
+
- name: Reap this run's leftover instance containers
|
|
309
|
+
if: always()
|
|
310
|
+
env:
|
|
311
|
+
NORSK_LAUNCHED_MANIFEST: ${{ runner.temp }}/norsk-launched-instances
|
|
312
|
+
run: |
|
|
313
|
+
set -uo pipefail
|
|
314
|
+
m="$NORSK_LAUNCHED_MANIFEST"
|
|
315
|
+
[ -f "$m" ] || { echo "no launched-instances manifest -- nothing to reap"; exit 0; }
|
|
316
|
+
sort -u "$m" | while IFS= read -r id; do
|
|
317
|
+
[ -n "$id" ] || continue
|
|
318
|
+
ids="$(docker ps -aq --filter "label=norsk-ctl.instance=$id" || true)"
|
|
319
|
+
if [ -n "$ids" ]; then
|
|
320
|
+
echo "reaping leftover instance '$id': $ids"
|
|
321
|
+
docker rm -f $ids || true
|
|
322
|
+
fi
|
|
323
|
+
done
|
|
324
|
+
rm -f "$m" || true
|
|
325
|
+
|
|
326
|
+
# Report this pipeline's result to the aggregated product CI dashboard
|
|
327
|
+
# (id3as/ci-workflows). !cancelled() so a real pass/fail reports, but a
|
|
328
|
+
# cancelled/superseded run does NOT — a cancellation dispatched as failure
|
|
329
|
+
# would false-red the board. Candidate validations report to ctl instead.
|
|
330
|
+
notify:
|
|
331
|
+
needs: integration
|
|
332
|
+
if: ${{ !cancelled() && github.event_name != 'repository_dispatch' }}
|
|
333
|
+
runs-on: x64
|
|
334
|
+
steps:
|
|
335
|
+
# Runs BEFORE checkout, and does two things: points the harness's temp base
|
|
336
|
+
# out of the workspace for every later step in this job, and clears whatever
|
|
337
|
+
# the old repo-local default left behind on this runner. The body carries the
|
|
338
|
+
# why. It fails the job only when legacy litter survives -- which would take
|
|
339
|
+
# the checkout down a step later anyway, but report nothing useful.
|
|
340
|
+
- name: Route test-temp out of the workspace (pre-checkout)
|
|
341
|
+
run: |
|
|
342
|
+
set -uo pipefail
|
|
343
|
+
# These runners are docker-OUTSIDE-of-docker, so a bind-mount source the
|
|
344
|
+
# HOST daemon has not seen before is created by dockerd, as ROOT, on the
|
|
345
|
+
# host -- even when the launch passes --container-user (reproduced
|
|
346
|
+
# 2026-09-04). The harness's default base is repo-local for an OrbStack
|
|
347
|
+
# inotify constraint that binds macOS dev and nothing here, so every run
|
|
348
|
+
# seeded the CHECKOUT with root-owned dirs a non-root `git clean -ffdx`
|
|
349
|
+
# cannot remove. That took commentary's checks AND integration red on
|
|
350
|
+
# 2026-09-03. Use the sanctioned override, per-run and outside the
|
|
351
|
+
# checkout -- as 8ff17a8d did in June for one debug workflow, and stopped.
|
|
352
|
+
base="$RUNNER_TEMP/nctt-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT"
|
|
353
|
+
mkdir -p "$base"
|
|
354
|
+
echo "NORSK_CTL_TEST_TMP=$base" >> "$GITHUB_ENV"
|
|
355
|
+
# Yesterday's bases hold dockerd-created root dirs, so a non-root rm
|
|
356
|
+
# EACCESes. They cannot wedge a checkout, but they do fill the disk.
|
|
357
|
+
for old in $(find "$RUNNER_TEMP" -maxdepth 1 -name 'nctt-*' -mtime +0 -printf '%f\n' 2>/dev/null || true); do
|
|
358
|
+
docker run --rm --user 0:0 -v "$RUNNER_TEMP":/t alpine sh -c 'rm -rf -- "/t/$1"' sh "$old" >/dev/null 2>&1 || true
|
|
359
|
+
done
|
|
360
|
+
# LEGACY: a runner that ran the old default still carries a workspace
|
|
361
|
+
# test-temp, and a checkout with clean:true dies on it. Reap the holders
|
|
362
|
+
# first -- rm cannot unlink a live mountpoint even as root (EBUSY), and a
|
|
363
|
+
# RESTARTING leaked container re-creates the path as root seconds after
|
|
364
|
+
# any clean. Match on mount SOURCE, not a label: the holders are untracked
|
|
365
|
+
# by construction, because a killed run labels nothing.
|
|
366
|
+
tt="$GITHUB_WORKSPACE/test-temp"
|
|
367
|
+
[ -d "$tt" ] || exit 0
|
|
368
|
+
stuck=""
|
|
369
|
+
for c in $(docker ps -aq 2>/dev/null || true); do
|
|
370
|
+
if docker inspect -f '{{range .Mounts}}{{println .Source}}{{end}}' "$c" 2>/dev/null | grep -q "^$tt/"; then
|
|
371
|
+
stuck="$stuck $c"
|
|
372
|
+
fi
|
|
373
|
+
done
|
|
374
|
+
if [ -n "$stuck" ]; then
|
|
375
|
+
echo "reaping containers holding mounts under $tt:$stuck"
|
|
376
|
+
docker rm -f $stuck || true
|
|
377
|
+
fi
|
|
378
|
+
# The DIRECTORY, not just its contents: an empty but root-owned base
|
|
379
|
+
# still EACCESes a later mkdtemp. Mounting the PARENT is what lets a root
|
|
380
|
+
# container unlink the leaf.
|
|
381
|
+
docker run --rm --user 0:0 -v "$GITHUB_WORKSPACE":/w alpine \
|
|
382
|
+
sh -c 'rm -rf /w/test-temp' || rm -rf "$tt" || true
|
|
383
|
+
# Fail HERE if it survived. The checkout fails on it either way, but
|
|
384
|
+
# reports only an EACCES rmdir with no clue what was holding the path.
|
|
385
|
+
if [ -e "$tt" ]; then
|
|
386
|
+
echo "::error::$tt survived the pre-checkout clean"
|
|
387
|
+
ls -lan "$tt" || true
|
|
388
|
+
exit 1
|
|
389
|
+
fi
|
|
390
|
+
|
|
391
|
+
- uses: actions/checkout@v5
|
|
392
|
+
with:
|
|
393
|
+
clean: true
|
|
394
|
+
- id: meta
|
|
395
|
+
run: |
|
|
396
|
+
if [ "${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}" = "true" ]; then
|
|
397
|
+
echo "state=failure" >> "$GITHUB_OUTPUT"
|
|
398
|
+
else
|
|
399
|
+
echo "state=success" >> "$GITHUB_OUTPUT"
|
|
400
|
+
fi
|
|
401
|
+
- uses: ./.github/actions/ci-status-dispatch
|
|
402
|
+
with:
|
|
403
|
+
token: ${{ secrets.CI_DISPATCH_TOKEN }}
|
|
404
|
+
product: __PRODUCT__
|
|
405
|
+
pipeline: integration
|
|
406
|
+
status: ${{ steps.meta.outputs.state }}
|
|
407
|
+
|
|
408
|
+
# Report the rc-candidate validation result to the norsk-ctl release gate. Only
|
|
409
|
+
# on a ctl-candidate dispatch; release-collect tallies these and promotes
|
|
410
|
+
# latest once every gating product is green on the candidate.
|
|
411
|
+
report-candidate:
|
|
412
|
+
needs: integration
|
|
413
|
+
if: ${{ always() && github.event_name == 'repository_dispatch' && github.event.action == 'ctl-candidate' }}
|
|
414
|
+
runs-on: x64
|
|
415
|
+
steps:
|
|
416
|
+
- name: Fire ctl-candidate-result at norsk-ctl
|
|
417
|
+
env:
|
|
418
|
+
GH_TOKEN: ${{ secrets.CI_DISPATCH_TOKEN }}
|
|
419
|
+
# Lowercase on purpose: the drift gate masks every `product:` line, so
|
|
420
|
+
# one per-repo key serves both the notify and the report jobs.
|
|
421
|
+
product: __PRODUCT__
|
|
422
|
+
VERSION: ${{ github.event.client_payload.version }}
|
|
423
|
+
RESULT: ${{ needs.integration.result }}
|
|
424
|
+
run: |
|
|
425
|
+
set -euo pipefail
|
|
426
|
+
state=failure; [ "$RESULT" = "success" ] && state=success
|
|
427
|
+
# `tier: integration`, named rather than defaulted: the release gate
|
|
428
|
+
# tallies per tier, and this is the one that gates (docs/release-gate.md).
|
|
429
|
+
body="$(jq -nc --arg p "$product" --arg v "$VERSION" --arg r "$state" \
|
|
430
|
+
'{event_type:"ctl-candidate-result",client_payload:{product:$p,version:$v,result:$r,tier:"integration"}}')"
|
|
431
|
+
code="$(curl -sS -o /dev/null -w '%{http_code}' -X POST \
|
|
432
|
+
-H "Authorization: Bearer $GH_TOKEN" \
|
|
433
|
+
-H "Accept: application/vnd.github+json" \
|
|
434
|
+
https://api.github.com/repos/id3as/norsk-ctl/dispatches -d "$body")"
|
|
435
|
+
[ "$code" = "204" ] || { echo "::error::report to norsk-ctl failed (HTTP $code)"; exit 1; }
|
|
@@ -42,36 +42,117 @@ jobs:
|
|
|
42
42
|
publish:
|
|
43
43
|
runs-on: x64
|
|
44
44
|
steps:
|
|
45
|
-
#
|
|
46
|
-
#
|
|
47
|
-
#
|
|
48
|
-
#
|
|
49
|
-
#
|
|
50
|
-
- name:
|
|
45
|
+
# Runs BEFORE checkout, and does two things: points the harness's temp base
|
|
46
|
+
# out of the workspace for every later step in this job, and clears whatever
|
|
47
|
+
# the old repo-local default left behind on this runner. The body carries the
|
|
48
|
+
# why. It fails the job only when legacy litter survives -- which would take
|
|
49
|
+
# the checkout down a step later anyway, but report nothing useful.
|
|
50
|
+
- name: Route test-temp out of the workspace (pre-checkout)
|
|
51
51
|
run: |
|
|
52
52
|
set -uo pipefail
|
|
53
|
+
# These runners are docker-OUTSIDE-of-docker, so a bind-mount source the
|
|
54
|
+
# HOST daemon has not seen before is created by dockerd, as ROOT, on the
|
|
55
|
+
# host -- even when the launch passes --container-user (reproduced
|
|
56
|
+
# 2026-09-04). The harness's default base is repo-local for an OrbStack
|
|
57
|
+
# inotify constraint that binds macOS dev and nothing here, so every run
|
|
58
|
+
# seeded the CHECKOUT with root-owned dirs a non-root `git clean -ffdx`
|
|
59
|
+
# cannot remove. That took commentary's checks AND integration red on
|
|
60
|
+
# 2026-09-03. Use the sanctioned override, per-run and outside the
|
|
61
|
+
# checkout -- as 8ff17a8d did in June for one debug workflow, and stopped.
|
|
62
|
+
base="$RUNNER_TEMP/nctt-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT"
|
|
63
|
+
mkdir -p "$base"
|
|
64
|
+
echo "NORSK_CTL_TEST_TMP=$base" >> "$GITHUB_ENV"
|
|
65
|
+
# Yesterday's bases hold dockerd-created root dirs, so a non-root rm
|
|
66
|
+
# EACCESes. They cannot wedge a checkout, but they do fill the disk.
|
|
67
|
+
for old in $(find "$RUNNER_TEMP" -maxdepth 1 -name 'nctt-*' -mtime +0 -printf '%f\n' 2>/dev/null || true); do
|
|
68
|
+
docker run --rm --user 0:0 -v "$RUNNER_TEMP":/t alpine sh -c 'rm -rf -- "/t/$1"' sh "$old" >/dev/null 2>&1 || true
|
|
69
|
+
done
|
|
70
|
+
# LEGACY: a runner that ran the old default still carries a workspace
|
|
71
|
+
# test-temp, and a checkout with clean:true dies on it. Reap the holders
|
|
72
|
+
# first -- rm cannot unlink a live mountpoint even as root (EBUSY), and a
|
|
73
|
+
# RESTARTING leaked container re-creates the path as root seconds after
|
|
74
|
+
# any clean. Match on mount SOURCE, not a label: the holders are untracked
|
|
75
|
+
# by construction, because a killed run labels nothing.
|
|
53
76
|
tt="$GITHUB_WORKSPACE/test-temp"
|
|
54
77
|
[ -d "$tt" ] || exit 0
|
|
55
|
-
|
|
56
|
-
|
|
78
|
+
stuck=""
|
|
79
|
+
for c in $(docker ps -aq 2>/dev/null || true); do
|
|
80
|
+
if docker inspect -f '{{range .Mounts}}{{println .Source}}{{end}}' "$c" 2>/dev/null | grep -q "^$tt/"; then
|
|
81
|
+
stuck="$stuck $c"
|
|
82
|
+
fi
|
|
83
|
+
done
|
|
84
|
+
if [ -n "$stuck" ]; then
|
|
85
|
+
echo "reaping containers holding mounts under $tt:$stuck"
|
|
86
|
+
docker rm -f $stuck || true
|
|
87
|
+
fi
|
|
88
|
+
# The DIRECTORY, not just its contents: an empty but root-owned base
|
|
89
|
+
# still EACCESes a later mkdtemp. Mounting the PARENT is what lets a root
|
|
90
|
+
# container unlink the leaf.
|
|
91
|
+
docker run --rm --user 0:0 -v "$GITHUB_WORKSPACE":/w alpine \
|
|
92
|
+
sh -c 'rm -rf /w/test-temp' || rm -rf "$tt" || true
|
|
93
|
+
# Fail HERE if it survived. The checkout fails on it either way, but
|
|
94
|
+
# reports only an EACCES rmdir with no clue what was holding the path.
|
|
95
|
+
if [ -e "$tt" ]; then
|
|
96
|
+
echo "::error::$tt survived the pre-checkout clean"
|
|
97
|
+
ls -lan "$tt" || true
|
|
98
|
+
exit 1
|
|
99
|
+
fi
|
|
57
100
|
|
|
58
|
-
#
|
|
59
|
-
#
|
|
60
|
-
#
|
|
61
|
-
#
|
|
62
|
-
#
|
|
63
|
-
|
|
64
|
-
# time it landed on a cold runner (probe, 2026-09-01). Clear the leftovers
|
|
65
|
-
# from a root container FIRST, and the checkout can clean properly again.
|
|
66
|
-
# Best-effort by construction: no docker, no test-temp, or a failed run all
|
|
67
|
-
# fall through without failing the job.
|
|
68
|
-
- name: Clear stale root-owned test-temp (pre-checkout)
|
|
101
|
+
# Runs BEFORE checkout, and does two things: points the harness's temp base
|
|
102
|
+
# out of the workspace for every later step in this job, and clears whatever
|
|
103
|
+
# the old repo-local default left behind on this runner. The body carries the
|
|
104
|
+
# why. It fails the job only when legacy litter survives -- which would take
|
|
105
|
+
# the checkout down a step later anyway, but report nothing useful.
|
|
106
|
+
- name: Route test-temp out of the workspace (pre-checkout)
|
|
69
107
|
run: |
|
|
70
108
|
set -uo pipefail
|
|
109
|
+
# These runners are docker-OUTSIDE-of-docker, so a bind-mount source the
|
|
110
|
+
# HOST daemon has not seen before is created by dockerd, as ROOT, on the
|
|
111
|
+
# host -- even when the launch passes --container-user (reproduced
|
|
112
|
+
# 2026-09-04). The harness's default base is repo-local for an OrbStack
|
|
113
|
+
# inotify constraint that binds macOS dev and nothing here, so every run
|
|
114
|
+
# seeded the CHECKOUT with root-owned dirs a non-root `git clean -ffdx`
|
|
115
|
+
# cannot remove. That took commentary's checks AND integration red on
|
|
116
|
+
# 2026-09-03. Use the sanctioned override, per-run and outside the
|
|
117
|
+
# checkout -- as 8ff17a8d did in June for one debug workflow, and stopped.
|
|
118
|
+
base="$RUNNER_TEMP/nctt-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT"
|
|
119
|
+
mkdir -p "$base"
|
|
120
|
+
echo "NORSK_CTL_TEST_TMP=$base" >> "$GITHUB_ENV"
|
|
121
|
+
# Yesterday's bases hold dockerd-created root dirs, so a non-root rm
|
|
122
|
+
# EACCESes. They cannot wedge a checkout, but they do fill the disk.
|
|
123
|
+
for old in $(find "$RUNNER_TEMP" -maxdepth 1 -name 'nctt-*' -mtime +0 -printf '%f\n' 2>/dev/null || true); do
|
|
124
|
+
docker run --rm --user 0:0 -v "$RUNNER_TEMP":/t alpine sh -c 'rm -rf -- "/t/$1"' sh "$old" >/dev/null 2>&1 || true
|
|
125
|
+
done
|
|
126
|
+
# LEGACY: a runner that ran the old default still carries a workspace
|
|
127
|
+
# test-temp, and a checkout with clean:true dies on it. Reap the holders
|
|
128
|
+
# first -- rm cannot unlink a live mountpoint even as root (EBUSY), and a
|
|
129
|
+
# RESTARTING leaked container re-creates the path as root seconds after
|
|
130
|
+
# any clean. Match on mount SOURCE, not a label: the holders are untracked
|
|
131
|
+
# by construction, because a killed run labels nothing.
|
|
71
132
|
tt="$GITHUB_WORKSPACE/test-temp"
|
|
72
133
|
[ -d "$tt" ] || exit 0
|
|
73
|
-
|
|
74
|
-
|
|
134
|
+
stuck=""
|
|
135
|
+
for c in $(docker ps -aq 2>/dev/null || true); do
|
|
136
|
+
if docker inspect -f '{{range .Mounts}}{{println .Source}}{{end}}' "$c" 2>/dev/null | grep -q "^$tt/"; then
|
|
137
|
+
stuck="$stuck $c"
|
|
138
|
+
fi
|
|
139
|
+
done
|
|
140
|
+
if [ -n "$stuck" ]; then
|
|
141
|
+
echo "reaping containers holding mounts under $tt:$stuck"
|
|
142
|
+
docker rm -f $stuck || true
|
|
143
|
+
fi
|
|
144
|
+
# The DIRECTORY, not just its contents: an empty but root-owned base
|
|
145
|
+
# still EACCESes a later mkdtemp. Mounting the PARENT is what lets a root
|
|
146
|
+
# container unlink the leaf.
|
|
147
|
+
docker run --rm --user 0:0 -v "$GITHUB_WORKSPACE":/w alpine \
|
|
148
|
+
sh -c 'rm -rf /w/test-temp' || rm -rf "$tt" || true
|
|
149
|
+
# Fail HERE if it survived. The checkout fails on it either way, but
|
|
150
|
+
# reports only an EACCES rmdir with no clue what was holding the path.
|
|
151
|
+
if [ -e "$tt" ]; then
|
|
152
|
+
echo "::error::$tt survived the pre-checkout clean"
|
|
153
|
+
ls -lan "$tt" || true
|
|
154
|
+
exit 1
|
|
155
|
+
fi
|
|
75
156
|
|
|
76
157
|
- uses: actions/checkout@v5
|
|
77
158
|
with:
|