@norskvideo/ctl-dev-kit 0.2.145 → 0.2.147
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -35,6 +35,16 @@
|
|
|
35
35
|
#
|
|
36
36
|
# Repo-local for now; to be hoisted into id3as/ci-workflows as a reusable
|
|
37
37
|
# workflow (RFC 0001 Workstream E).
|
|
38
|
+
#
|
|
39
|
+
# The jobs that log in to Docker Hub (build, publish) run docker with
|
|
40
|
+
# DOCKER_CONFIG set to a job-local directory under $RUNNER_TEMP that holds only
|
|
41
|
+
# this job's own login, deleted when the job ends. The runners are shared and
|
|
42
|
+
# their own ~/.docker is not ours: on the shared arm pool a stale Docker Hub
|
|
43
|
+
# login there made the build's pull of public bases (node:22-slim, oven/bun)
|
|
44
|
+
# fail with 401 "incorrect username or password" before this job had logged in.
|
|
45
|
+
# The bases are public, so those pulls are now anonymous. The runner images ship
|
|
46
|
+
# buildx as a system plugin and use the default docker socket, so the empty
|
|
47
|
+
# config loses nothing. norsk-cloud's build-images.yml does the same (f7c1898e).
|
|
38
48
|
name: build-image
|
|
39
49
|
|
|
40
50
|
on:
|
|
@@ -178,16 +188,32 @@ jobs:
|
|
|
178
188
|
|
|
179
189
|
build:
|
|
180
190
|
needs: label
|
|
191
|
+
# The arm64 leg runs on the shared arm pool (label `arm`: jankings
|
|
192
|
+
# arm-runner-1..4 and arm-runner-quadra, docker-outside-of-docker on
|
|
193
|
+
# arm.norsk). A build runs no containers and binds no ports, so the pool's
|
|
194
|
+
# shared daemon and Docker 25 are fine for it. It used to wait for the single
|
|
195
|
+
# ar-arm-vm runner behind every other arm64 job (id3as/norsk-cloud#111).
|
|
196
|
+
# CTL_ARM64_RUNNER (an Actions variable, repo or org level, one runner
|
|
197
|
+
# label) sends it elsewhere without a commit -- e.g. `ar-arm-vm` if the
|
|
198
|
+
# pool is down; unset means `arm`.
|
|
181
199
|
strategy:
|
|
182
200
|
fail-fast: false
|
|
183
201
|
matrix:
|
|
184
202
|
include:
|
|
185
203
|
- runner: x64
|
|
186
204
|
arch: amd64
|
|
187
|
-
- runner:
|
|
205
|
+
- runner: arm
|
|
188
206
|
arch: arm64
|
|
189
|
-
runs-on: ${{ matrix.runner }}
|
|
207
|
+
runs-on: ${{ matrix.arch == 'arm64' && vars.CTL_ARM64_RUNNER || matrix.runner }}
|
|
190
208
|
steps:
|
|
209
|
+
- name: Job-local docker config
|
|
210
|
+
# Before anything touches docker (see the header): this job's own login
|
|
211
|
+
# only, never the runner's ~/.docker.
|
|
212
|
+
run: |
|
|
213
|
+
rm -rf "$RUNNER_TEMP/docker-config"
|
|
214
|
+
mkdir -m 700 "$RUNNER_TEMP/docker-config"
|
|
215
|
+
echo "DOCKER_CONFIG=$RUNNER_TEMP/docker-config" >> "$GITHUB_ENV"
|
|
216
|
+
|
|
191
217
|
# Runs BEFORE checkout, and does two things: points the harness's temp base
|
|
192
218
|
# out of the workspace for every later step in this job, and clears whatever
|
|
193
219
|
# the old repo-local default left behind on this runner. The body carries the
|
|
@@ -282,6 +308,7 @@ jobs:
|
|
|
282
308
|
with:
|
|
283
309
|
username: norskvideo
|
|
284
310
|
password: ${{ secrets.DOCKER_PAT }}
|
|
311
|
+
logout: false # the last step deletes the whole config, the login with it
|
|
285
312
|
|
|
286
313
|
- name: Push the per-arch image to Docker Hub
|
|
287
314
|
if: ${{ env.PUBLISH_IMAGE != '' && github.event_name != 'pull_request' }}
|
|
@@ -304,6 +331,9 @@ jobs:
|
|
|
304
331
|
echo "::warning::removing containers this job left running:$s"
|
|
305
332
|
docker rm -f $s
|
|
306
333
|
fi
|
|
334
|
+
- name: Remove the job-local docker config
|
|
335
|
+
if: always()
|
|
336
|
+
run: rm -rf "$RUNNER_TEMP/docker-config"
|
|
307
337
|
|
|
308
338
|
# THE PUBLISH GATE. Nothing is published -- no dated tag, no channel tag, no
|
|
309
339
|
# channels row -- until both halves pass: the repo's own checks, and its guides
|
|
@@ -864,6 +894,14 @@ jobs:
|
|
|
864
894
|
if: ${{ needs.label.outputs.publish == 'true' && !inputs.dry_run && github.event_name != 'pull_request' }}
|
|
865
895
|
runs-on: x64
|
|
866
896
|
steps:
|
|
897
|
+
- name: Job-local docker config
|
|
898
|
+
# Before anything touches docker (see the header): this job's own login
|
|
899
|
+
# only, never the runner's ~/.docker.
|
|
900
|
+
run: |
|
|
901
|
+
rm -rf "$RUNNER_TEMP/docker-config"
|
|
902
|
+
mkdir -m 700 "$RUNNER_TEMP/docker-config"
|
|
903
|
+
echo "DOCKER_CONFIG=$RUNNER_TEMP/docker-config" >> "$GITHUB_ENV"
|
|
904
|
+
|
|
867
905
|
# Runs BEFORE checkout, and does two things: points the harness's temp base
|
|
868
906
|
# out of the workspace for every later step in this job, and clears whatever
|
|
869
907
|
# the old repo-local default left behind on this runner. The body carries the
|
|
@@ -932,6 +970,7 @@ jobs:
|
|
|
932
970
|
with:
|
|
933
971
|
username: norskvideo
|
|
934
972
|
password: ${{ secrets.DOCKER_PAT }}
|
|
973
|
+
logout: false # the last step deletes the whole config, the login with it
|
|
935
974
|
|
|
936
975
|
- name: Create + push the multi-arch manifest (dated label + channel)
|
|
937
976
|
run: |
|
|
@@ -964,6 +1003,9 @@ jobs:
|
|
|
964
1003
|
echo "::warning::removing containers this job left running:$s"
|
|
965
1004
|
docker rm -f $s
|
|
966
1005
|
fi
|
|
1006
|
+
- name: Remove the job-local docker config
|
|
1007
|
+
if: always()
|
|
1008
|
+
run: rm -rf "$RUNNER_TEMP/docker-config"
|
|
967
1009
|
|
|
968
1010
|
# Record the published image in this product's channels object
|
|
969
1011
|
# (s3://norsk.video/products/<product>/channels.json), which ctl reads to
|
|
@@ -88,8 +88,8 @@ export function carriesIntegrationMarker(text: string): boolean {
|
|
|
88
88
|
}
|
|
89
89
|
|
|
90
90
|
// smoke.yml carries a second per-repo line: the matrix `runner:` label array
|
|
91
|
-
// the journey runs on ('["
|
|
92
|
-
//
|
|
91
|
+
// the journey runs on ('["arm"]' for the shared arm pool, '["ARM64","nvidia"]'
|
|
92
|
+
// for a GPU product). Masked like product:; any filled value is valid, so no sentinel check.
|
|
93
93
|
export const RUNNER_LINE = /^(\s*-?\s*runner:\s*).*$/gm;
|
|
94
94
|
|
|
95
95
|
function workflowProblem(
|
package/conventions/smoke.yml
CHANGED
|
@@ -12,13 +12,14 @@
|
|
|
12
12
|
# customer would run it.
|
|
13
13
|
#
|
|
14
14
|
# Runner selection is a one-entry matrix carrying a JSON label array, because a
|
|
15
|
-
# string like '[ARM64, nvidia]' in runs-on is ONE label, not two.
|
|
16
|
-
#
|
|
17
|
-
#
|
|
18
|
-
#
|
|
19
|
-
#
|
|
20
|
-
#
|
|
21
|
-
#
|
|
15
|
+
# string like '[ARM64, nvidia]' in runs-on is ONE label, not two. Default
|
|
16
|
+
# '["arm"]': the shared arm pool (jankings arm-runner-1..4 and -quadra,
|
|
17
|
+
# containers on one docker daemon); a GPU product asks for '["ARM64","nvidia"]',
|
|
18
|
+
# labels the same pool's runners carry.
|
|
19
|
+
# Either kind of runner works: the topology probe below detects a
|
|
20
|
+
# docker-outside-of-docker runner and reaches the product another way, and
|
|
21
|
+
# on a shared daemon the journey step takes a per-product host lock and the
|
|
22
|
+
# harness bands its ports per runner, so runs on sibling runners never meet.
|
|
22
23
|
name: smoke
|
|
23
24
|
|
|
24
25
|
on:
|
|
@@ -54,7 +55,7 @@ jobs:
|
|
|
54
55
|
strategy:
|
|
55
56
|
matrix:
|
|
56
57
|
include:
|
|
57
|
-
- runner: '["
|
|
58
|
+
- runner: '["arm"]'
|
|
58
59
|
runs-on: ${{ fromJSON(matrix.runner) }}
|
|
59
60
|
steps:
|
|
60
61
|
# Runs BEFORE checkout, and does two things: points the harness's temp base
|
|
@@ -165,7 +166,7 @@ jobs:
|
|
|
165
166
|
# Hence: kill by PORT rather than by name, prune the endpoints a removal
|
|
166
167
|
# can strand, and then PROVE the port binds instead of assuming it. This
|
|
167
168
|
# branch only runs where the docker daemon is this runner's own and jobs
|
|
168
|
-
# are serial, so there is nothing else here to protect. The
|
|
169
|
+
# are serial, so there is nothing else here to protect. The arm pool runs
|
|
169
170
|
# several runners as sibling CONTAINERS against one host daemon and is
|
|
170
171
|
# excluded -- there, a blanket reap would kill a concurrent job.
|
|
171
172
|
- name: Reap a stranded product control plane
|
|
@@ -404,22 +405,49 @@ jobs:
|
|
|
404
405
|
# Outer pipefail so the step reports the journey's exit, not tee's --
|
|
405
406
|
# otherwise a failed smoke run over the `| tee` below scores green.
|
|
406
407
|
set -euo pipefail
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
408
|
+
journey() {
|
|
409
|
+
nix develop .#build --command bash -c '
|
|
410
|
+
set -euo pipefail
|
|
411
|
+
# A frozen install does not reliably relink a hoisted package whose
|
|
412
|
+
# version moved: a stale test-harness once survived a bump and a new
|
|
413
|
+
# export was "not found" at import. Drop the harness copy so the
|
|
414
|
+
# pinned version runs. Belt and braces now the checkout is clean.
|
|
415
|
+
rm -rf node_modules/@norskvideo/ctl-test-harness
|
|
416
|
+
bun install --frozen-lockfile
|
|
417
|
+
pub="$(sed -nE "s/^[[:space:]]*PUBLISH_IMAGE:[[:space:]]*\"?([^\" ]*)\"?[[:space:]]*$/\\1/p" .github/workflows/build-image.yml | head -1)"
|
|
418
|
+
def="$(sed -nE "s/.*IMAGE_TAG:-([^}]+)\\}.*/\\1/p" deployment/build-image.sh | head -1)"
|
|
419
|
+
export IMAGE_TAG="${pub:+$pub:smoke}"
|
|
420
|
+
export IMAGE_TAG="${IMAGE_TAG:-$def}"
|
|
421
|
+
echo "smoke image: $IMAGE_TAG"
|
|
422
|
+
bun run build:image
|
|
423
|
+
SMOKE_IMAGE_TAG="$IMAGE_TAG" bun run test:smoke
|
|
424
|
+
'
|
|
425
|
+
}
|
|
426
|
+
# One run of THIS product per docker host at a time. On a shared
|
|
427
|
+
# daemon (the arm pool's runners are sibling containers) two runs of
|
|
428
|
+
# one product collide on names only it uses: the registered image's
|
|
429
|
+
# control plane norsk-product-<name>, the instance smoke-<slug> and
|
|
430
|
+
# the <repo>:smoke tag the journey builds. A per-repo concurrency
|
|
431
|
+
# group cannot stop that without dropping a queued ctl-candidate
|
|
432
|
+
# run. /srv/ci-scratch is bind-mounted at that path into every runner
|
|
433
|
+
# on a jankings host, so an flock there is the host's mutex; the
|
|
434
|
+
# kernel drops it when this shell exits. Children close fd 9. After
|
|
435
|
+
# an hour the journey runs anyway, warned. smoke-host-lock.test.ts.
|
|
436
|
+
if [ -d /srv/ci-scratch ] && [ -w /srv/ci-scratch ]; then
|
|
437
|
+
lock="/srv/ci-scratch/locks/norsk-ctl-smoke-${GITHUB_REPOSITORY//\//_}.lock"
|
|
438
|
+
mkdir -p "${lock%/*}"
|
|
439
|
+
exec 9>>"$lock"
|
|
440
|
+
echo "waiting for this host's smoke lock for this product ($lock)"
|
|
441
|
+
waited=$SECONDS
|
|
442
|
+
if flock -w 3600 9; then
|
|
443
|
+
echo "holding this host's smoke lock after $((SECONDS - waited))s"
|
|
444
|
+
else
|
|
445
|
+
echo "::warning title=smoke ran unserialised::waited an hour for $lock on $(hostname); a concurrent run of this product on this host may collide with it"
|
|
446
|
+
fi
|
|
447
|
+
else
|
|
448
|
+
echo "no writable /srv/ci-scratch on this runner: smoke runs without the host lock"
|
|
449
|
+
fi
|
|
450
|
+
journey 9>&- 2>&1 | tee "$RUNNER_TEMP/smoke.log"
|
|
423
451
|
|
|
424
452
|
# We pretty much always want the Norsk logs from a smoke run: the journey
|
|
425
453
|
# runs the RELEASED media + studio images, and the harness dumps each
|
|
@@ -366,3 +366,67 @@ export function jobsEnteringUnrootedShell(yaml: string): string[] {
|
|
|
366
366
|
});
|
|
367
367
|
return bad;
|
|
368
368
|
}
|
|
369
|
+
|
|
370
|
+
/** The first step of every job that logs in to a registry, and its always()
|
|
371
|
+
* partner at the end, named so the guard below can find them. */
|
|
372
|
+
export const DOCKER_CONFIG_STEP = "Job-local docker config";
|
|
373
|
+
export const DOCKER_CONFIG_REMOVE_STEP = "Remove the job-local docker config";
|
|
374
|
+
|
|
375
|
+
// The arm pool is shared, and its runners carry their own ~/.docker with a
|
|
376
|
+
// stale Docker Hub login. A build that read it failed its pull of public bases
|
|
377
|
+
// (node:22-slim, oven/bun) with "401 incorrect username or password" before the
|
|
378
|
+
// job's own login had run (norsk-relay, 2026-11-17, once its arm leg moved to
|
|
379
|
+
// the pool). So a job that logs in points DOCKER_CONFIG at a directory of its
|
|
380
|
+
// own under $RUNNER_TEMP before anything touches docker -- the pre-checkout
|
|
381
|
+
// step already pulls alpine -- logs in there only, and deletes it at the end.
|
|
382
|
+
// The login action's own logout is off: by its post step the directory is gone.
|
|
383
|
+
// norsk-cloud's build-images.yml did the same first (f7c1898e).
|
|
384
|
+
const DOCKER_CONFIG_REQUIREMENTS: { name: string; ok: (body: string) => boolean }[] = [
|
|
385
|
+
{ name: "export", ok: (b) => /DOCKER_CONFIG=\$RUNNER_TEMP\/[\w-]+"? >> "\$GITHUB_ENV"/.test(b) },
|
|
386
|
+
{ name: "fresh", ok: (b) => /rm -rf "\$RUNNER_TEMP\/[\w-]+"/.test(b) && /mkdir -m 700 /.test(b) },
|
|
387
|
+
];
|
|
388
|
+
|
|
389
|
+
/** Jobs that use `docker/login-action` without the job-local docker config:
|
|
390
|
+
* `<job>: <what is missing>`, where missing is `first` (no valid
|
|
391
|
+
* {@link DOCKER_CONFIG_STEP} as the job's first step), `logout` (a login that
|
|
392
|
+
* leaves the action's logout on) or `remove` (no always() removal step). */
|
|
393
|
+
export function jobDockerConfigProblems(yaml: string): string[] {
|
|
394
|
+
const lines = yaml.split("\n");
|
|
395
|
+
const out: string[] = [];
|
|
396
|
+
let job: string | undefined;
|
|
397
|
+
let start = 0;
|
|
398
|
+
const settle = (end: number) => {
|
|
399
|
+
if (job === undefined) return;
|
|
400
|
+
const body = lines.slice(start, end);
|
|
401
|
+
const logins = body.flatMap((l, i) => (/^\s*-?\s*uses: docker\/login-action@/.test(l) ? [i] : []));
|
|
402
|
+
if (logins.length === 0) return;
|
|
403
|
+
const missing: string[] = [];
|
|
404
|
+
const firstStep = body.findIndex((l) => /^ {6}- /.test(l));
|
|
405
|
+
const first = firstStep < 0 ? "" : (body[firstStep] as string);
|
|
406
|
+
const firstOk =
|
|
407
|
+
first.includes(`- name: ${DOCKER_CONFIG_STEP}`) &&
|
|
408
|
+
DOCKER_CONFIG_REQUIREMENTS.every((r) => r.ok(stepBody(lines, start + firstStep)));
|
|
409
|
+
if (!firstOk) missing.push("first");
|
|
410
|
+
const loggedOut = logins.some((i) => {
|
|
411
|
+
const next = body.findIndex((l, j) => j > i && /^ {6}- /.test(l));
|
|
412
|
+
return !body.slice(i, next < 0 ? body.length : next).some((l) => /^\s*logout: false\b/.test(l));
|
|
413
|
+
});
|
|
414
|
+
if (loggedOut) missing.push("logout");
|
|
415
|
+
const remove = body.findIndex((l) => l.includes(`- name: ${DOCKER_CONFIG_REMOVE_STEP}`));
|
|
416
|
+
const removeOk =
|
|
417
|
+
remove >= 0 &&
|
|
418
|
+
body.slice(remove + 1, remove + 4).some((l) => /^\s*if: always\(\)\s*$/.test(l)) &&
|
|
419
|
+
/rm -rf "\$RUNNER_TEMP\/[\w-]+"/.test(body.slice(remove, remove + 5).join("\n"));
|
|
420
|
+
if (!removeOk) missing.push("remove");
|
|
421
|
+
if (missing.length > 0) out.push(`${job}: ${missing.join(", ")}`);
|
|
422
|
+
};
|
|
423
|
+
lines.forEach((line, i) => {
|
|
424
|
+
const head = line.match(/^ {2}([A-Za-z_][\w-]*):\s*$/);
|
|
425
|
+
if (!head) return;
|
|
426
|
+
settle(i);
|
|
427
|
+
job = head[1];
|
|
428
|
+
start = i;
|
|
429
|
+
});
|
|
430
|
+
settle(lines.length);
|
|
431
|
+
return out;
|
|
432
|
+
}
|