@norskvideo/ctl-test-harness 0.1.58 → 0.1.60

package/container-logs.d.ts

@@ -28,6 +28,17 @@ export declare function formatContainerLogs(instanceId: string, entries: Contain
 export declare function collectInstanceContainerLogs(instanceId: string, tail?: number, spawn?: SpawnLike): ContainerLog[];
 /** The ready-to-print diagnostic block (empty-marker when nothing is found). */
 export declare function dumpInstanceContainerLogs(instanceId: string, tail?: number, spawn?: SpawnLike): string;
+/** Write every container of one instance to `<dest>/<container>.log`, for a CI
+ *  artifact. The companion to {@link preserveEngineLogs}, which keeps the media
+ *  container's bind-mounted engine tree: this keeps what ONLY Docker has, and
+ *  the studio container's stdout is where "Refusing to run workflow" is printed
+ *  — the line the gate failures point at and that teardown then destroys.
+ *
+ *  Call BEFORE cleanup. Best-effort throughout and never throws: a diagnostic
+ *  that fails the run it was meant to explain is worse than no diagnostic.
+ *  Returns the number of logs written, and creates nothing when there are none.
+ */
+export declare function preserveInstanceContainerLogs(instanceId: string, dest: string, tail?: number, spawn?: SpawnLike): number;
 export interface LogHit {
     container: string;
     line: string;

package/container-logs.js

@@ -8,7 +8,7 @@
 // Pure argv/parse/format helpers; only the two exported probes shell out,
 // through an injectable spawn.
 import { spawnSync } from "node:child_process";
-import { cpSync, existsSync, readdirSync, statSync } from "node:fs";
+import { cpSync, existsSync, mkdirSync, readdirSync, statSync, writeFileSync } from "node:fs";
 import { join, relative } from "node:path";
 const defaultSpawn = (cmd, args, opts) => {
     const r = spawnSync(cmd, args, opts);
@@ -94,6 +94,40 @@ export function collectInstanceContainerLogs(instanceId, tail = DEFAULT_TAIL, sp
 export function dumpInstanceContainerLogs(instanceId, tail = DEFAULT_TAIL, spawn = defaultSpawn) {
     return formatContainerLogs(instanceId, collectInstanceContainerLogs(instanceId, tail, spawn));
 }
+/** Write every container of one instance to `<dest>/<container>.log`, for a CI
+ *  artifact. The companion to {@link preserveEngineLogs}, which keeps the media
+ *  container's bind-mounted engine tree: this keeps what ONLY Docker has, and
+ *  the studio container's stdout is where "Refusing to run workflow" is printed
+ *  — the line the gate failures point at and that teardown then destroys.
+ *
+ *  Call BEFORE cleanup. Best-effort throughout and never throws: a diagnostic
+ *  that fails the run it was meant to explain is worse than no diagnostic.
+ *  Returns the number of logs written, and creates nothing when there are none.
+ */
+export function preserveInstanceContainerLogs(instanceId, dest, tail = DEFAULT_TAIL, spawn = defaultSpawn) {
+    const entries = collectInstanceContainerLogs(instanceId, tail, spawn);
+    if (entries.length === 0)
+        return 0;
+    let written = 0;
+    try {
+        mkdirSync(dest, { recursive: true });
+    }
+    catch {
+        return 0;
+    }
+    for (const e of entries) {
+        try {
+            // The state goes in the file, not just the name: an `exited` container
+            // with an empty log says something a missing file does not.
+            writeFileSync(join(dest, `${e.name}.log`), `# ${e.name} (${e.state})\n${e.log}`);
+            written++;
+        }
+        catch {
+            // Keep going: one unwritable file must not cost the others.
+        }
+    }
+    return written;
+}
 /** Copy every engine `logs` tree under an instance-runtime root to `dest`,
  *  preserving the relative structure. Why this and not `docker logs`:
  *  norsk-ctl bind-mounts each media container's `/var/log/norsk` to

package/demo/harness.js

@@ -16,6 +16,7 @@
 import { netReachMode, STUDIO_INTERNAL_PORT, studioBaseFrom } from "../container-net.js";
 import { ctlSupportsPublishDebugPorts } from "../launch.js";
 import { pollUntil } from "../poll.js";
+import { launchWithPortConflictRetry, reapInstance } from "../reap.js";
 import { DemoGateFailure } from "./gates.js";
 import { DemoSession, defaultDemoDeps, demoSlug, expandHome, resolveIngestPort, } from "./run.js";
 const STUDIO_HOST_PORT_PARAM = "STUDIO_HOST_PORT";
@@ -165,7 +166,16 @@ export class ProductHarness {
         // the reap step whether or not this process lives to teardown.
         if (this.launchedManifest)
             this.deps.writeFile(this.launchedManifest, `${this.launchedIds.join("\n")}\n`);
-        await s.cliOk(launchArgv);
+        // A run cancelled before teardown leaves this id's compose stack up, still
+        // holding its host ports, and nothing that asks a live daemon can see it
+        // (the leaker's daemon is gone). Reap by label first, then retry once if the
+        // bind still loses the advisory-check race — see reap.ts.
+        reapInstance(instanceId);
+        await launchWithPortConflictRetry({
+            instanceId,
+            attempt: () => s.cliOk(launchArgv),
+            log: deps.log,
+        });
         await s.waitRunning(instanceId);
         const rows = await s.ingestPorts(instanceId);
         const templateParams = { declared, overrides: specParams };

package/demo/run.d.ts

@@ -315,6 +315,10 @@ export declare class DemoSession {
     /** Tear down in reverse: sources, instance, then — private only — the
      *  daemon (with its proxy), the dev backend, the store. Reuse leaves the
      *  developer's daemon, store and registration as they are. */
+    /** Keep each instance's container logs under NORSK_CI_LOG_DIR for the
+     *  workflow's artifact upload. No env var (local dev) means no-op; every
+     *  failure inside is swallowed by the helper. */
+    private preserveContainerLogs;
     teardown(opts: {
         instances: string[];
         handles: SourceHandle[];

package/demo/run.js

@@ -29,6 +29,7 @@ import { existsSync, lstatSync, mkdirSync, readdirSync, readFileSync, readlinkSy
 import { homedir } from "node:os";
 import { dirname, isAbsolute, join, resolve } from "node:path";
 import { parseManifestSeed, repoOf } from "@norskvideo/ctl-sdk/manifest-seed";
+import { preserveInstanceContainerLogs } from "../container-logs.js";
 import { DOCKER_NETWORK_NAME, ensureRunnerOnNetwork, netReachMode } from "../container-net.js";
 import { cleanupDaemon, requireLicenseFile, runCli, startDaemon, } from "../daemon.js";
 import { hashSlot } from "../harness-config.js";
@@ -653,7 +654,27 @@ export class DemoSession {
     /** Tear down in reverse: sources, instance, then — private only — the
      *  daemon (with its proxy), the dev backend, the store. Reuse leaves the
      *  developer's daemon, store and registration as they are. */
+    /** Keep each instance's container logs under NORSK_CI_LOG_DIR for the
+     *  workflow's artifact upload. No env var (local dev) means no-op; every
+     *  failure inside is swallowed by the helper. */
+    preserveContainerLogs(instances) {
+        const ciLogDir = process.env.NORSK_CI_LOG_DIR;
+        if (!ciLogDir)
+            return;
+        for (const id of instances) {
+            const n = preserveInstanceContainerLogs(id, join(ciLogDir, "containers", id));
+            if (n > 0)
+                this.deps.log(`preserved ${n} container log(s) for ${id} -> ${join(ciLogDir, "containers", id)}`);
+        }
+    }
     async teardown(opts) {
+        // FIRST, while the containers are still standing. Everything below removes
+        // them, and `docker logs` is the only copy of the studio container's stdout
+        // — the "Refusing to run workflow" line that the workflow-never-started
+        // gates tell you to go and read. Unconditional rather than on-failure,
+        // because teardown is not told the verdict, and a tail per container is
+        // cheap next to a re-run.
+        this.preserveContainerLogs(opts.instances);
         if (opts.handles.length)
             await this.deps.stopSources(opts.handles).catch(() => { });
         if (this.policy === "reuse") {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@norskvideo/ctl-test-harness",
-  "version": "0.1.58",
+  "version": "0.1.60",
   "type": "module",
   "exports": {
     ".": {
@@ -27,6 +27,10 @@
       "types": "./launch.d.ts",
       "default": "./launch.js"
     },
+    "./reap": {
+      "types": "./reap.d.ts",
+      "default": "./reap.js"
+    },
     "./source-pump": {
       "types": "./source-pump.d.ts",
       "default": "./source-pump.js"

package/reap.d.ts

@@ -0,0 +1,37 @@
+import { spawnSync } from "node:child_process";
+/** True when `text` is a host-port binding conflict — the signal to reap and
+ *  retry. Deliberately narrow: an unhealthy dependency is also a launch
+ *  failure, and retrying that one just burns the clock. */
+export declare function isPortConflict(text: string): boolean;
+/** The conflicting host port, or null when the message carries no extractable
+ *  one (still a conflict per {@link isPortConflict}, just not attributable to a
+ *  port, so only the instance-scoped reap can run). */
+export declare function extractConflictPort(text: string): number | null;
+/** `docker ps` argv listing container ids for ONE instance id — the exact scope
+ *  that spares co-running products. */
+export declare function instancePsArgs(instanceId: string): string[];
+/** `docker ps` argv listing norsk-ctl container ids publishing a given host
+ *  port. Role-filtered, so a squatter outside norsk-ctl is never touched. */
+export declare function portPsArgs(port: number): string[];
+type Spawn = typeof spawnSync;
+/** Reap every leftover container for one instance id (media, studio and source
+ *  all carry the `norsk-ctl.instance` label). */
+export declare function reapInstance(instanceId: string, spawn?: Spawn): string[];
+/** Reap whatever norsk-ctl container holds a specific host port — the fallback
+ *  when the holder belongs to a different instance, so the instance-id reap
+ *  alone would not free it. */
+export declare function reapPort(port: number, spawn?: Spawn): string[];
+/**
+ * Run a launch, and on a host-port conflict reap the holder and retry ONCE.
+ *
+ * Once, not a loop: one retry clears a leaked holder or a lost race, and a
+ * second conflict means something is genuinely sitting on the port — where the
+ * launch's own message, naming the port, is more use than another attempt.
+ */
+export declare function launchWithPortConflictRetry<T>(opts: {
+    instanceId: string;
+    attempt: () => Promise<T>;
+    spawn?: Spawn;
+    log?: (line: string) => void;
+}): Promise<T>;
+export {};

package/reap.js

@@ -0,0 +1,129 @@
+// Orphan-container reaper for launches that lose a host port.
+//
+// ctl's port-availability check at validation is ADVISORY — the probe holds no
+// reservation, and docker binds host ports at container START — so a port free
+// when the launch was validated can be taken by the time compose binds it
+// (`launch-service.ts`, HOST_PORT_CONFLICT). Its own message says to launch
+// again. On a shared self-hosted runner that race is routine rather than
+// exotic, in two flavours:
+//
+//   * a CANCELLED run: `cancel-in-progress` SIGKILLs `bun test` before the
+//     harness can tear down, so a `compose up -d` stack outlives its (now dead)
+//     daemon and keeps holding the port. Nothing that asks a LIVE daemon can
+//     see it, because the leaker's daemon is gone — which is why this reaps off
+//     Docker by label rather than through the API.
+//   * co-running products: six product repos' CI can fire within seconds of
+//     each other on one Docker daemon, and a fixed host port (the MoQ preview's
+//     `preferred: 4443`) is picked by each of them independently.
+//
+// Scoping is the whole safety argument. Every norsk-ctl container carries
+// `norsk-ctl.role`; per-instance ones also carry `norsk-ctl.instance=<id>`. We
+// reap by the exact instance id, and by port only among containers that carry
+// the role label — so a co-running product is untouched and a squatter outside
+// norsk-ctl is never removed, only reported by the launch that then fails.
+//
+// The regex and argv builders are pure so tests/reap.test.ts covers them with
+// zero Docker; only `reapInstance`/`reapPort` shell out.
+//
+// Lifted from norsk-ctl-product-playout's tests/integration/support/reap.ts,
+// which is where this was worked out. It lived in one product while probe,
+// commentary, reuters and funke had no equivalent and lost tests to the same
+// race, so it belongs here instead.
+import { spawnSync } from "node:child_process";
+// A launch failure that means "the host port I need is already bound". Covers
+// the daemon's own wrapper and the raw docker/kernel strings it quotes, since
+// either can surface depending on where the bind loses the race.
+const CONFLICT_PATTERNS = [
+    /was taken between validation and container start/i,
+    /port is already allocated/i,
+    /address already in use/i,
+];
+// Same failure, with the offending port captured. Ordered most-specific first:
+// the daemon wrapper names the port up front, the raw docker/kernel lines carry
+// it in the bind address.
+const PORT_PATTERNS = [
+    /host port (\d{2,5})\b/i,
+    /bind for [\d.:[\]]*?:(\d{2,5})\b[^:]*:\s*port is already allocated/i,
+    /:(\d{2,5}):\s*bind:\s*address already in use/i,
+    /port (\d{2,5}) .*already/i,
+];
+/** True when `text` is a host-port binding conflict — the signal to reap and
+ *  retry. Deliberately narrow: an unhealthy dependency is also a launch
+ *  failure, and retrying that one just burns the clock. */
+export function isPortConflict(text) {
+    return CONFLICT_PATTERNS.some((re) => re.test(text));
+}
+/** The conflicting host port, or null when the message carries no extractable
+ *  one (still a conflict per {@link isPortConflict}, just not attributable to a
+ *  port, so only the instance-scoped reap can run). */
+export function extractConflictPort(text) {
+    for (const re of PORT_PATTERNS) {
+        const m = text.match(re);
+        if (m?.[1])
+            return Number(m[1]);
+    }
+    return null;
+}
+/** `docker ps` argv listing container ids for ONE instance id — the exact scope
+ *  that spares co-running products. */
+export function instancePsArgs(instanceId) {
+    return ["ps", "-aq", "--filter", `label=norsk-ctl.instance=${instanceId}`];
+}
+/** `docker ps` argv listing norsk-ctl container ids publishing a given host
+ *  port. Role-filtered, so a squatter outside norsk-ctl is never touched. */
+export function portPsArgs(port) {
+    return ["ps", "-aq", "--filter", "label=norsk-ctl.role", "--filter", `publish=${port}`];
+}
+/** Force-remove the containers `docker ps <psArgs>` lists. Best-effort and
+ *  synchronous (it must finish before the relaunch); a Docker that is absent or
+ *  errors leaves the launch to fail with its own clearer message. Returns the
+ *  ids it removed, for logging. */
+function reap(psArgs, spawn) {
+    try {
+        const ps = spawn("docker", psArgs, { encoding: "utf8", timeout: 15_000 });
+        const ids = (ps.stdout ?? "")
+            .split("\n")
+            .map((s) => s.trim())
+            .filter(Boolean);
+        if (ids.length > 0)
+            spawn("docker", ["rm", "-f", ...ids], { encoding: "utf8", timeout: 30_000 });
+        return ids;
+    }
+    catch {
+        return [];
+    }
+}
+/** Reap every leftover container for one instance id (media, studio and source
+ *  all carry the `norsk-ctl.instance` label). */
+export function reapInstance(instanceId, spawn = spawnSync) {
+    return reap(instancePsArgs(instanceId), spawn);
+}
+/** Reap whatever norsk-ctl container holds a specific host port — the fallback
+ *  when the holder belongs to a different instance, so the instance-id reap
+ *  alone would not free it. */
+export function reapPort(port, spawn = spawnSync) {
+    return reap(portPsArgs(port), spawn);
+}
+/**
+ * Run a launch, and on a host-port conflict reap the holder and retry ONCE.
+ *
+ * Once, not a loop: one retry clears a leaked holder or a lost race, and a
+ * second conflict means something is genuinely sitting on the port — where the
+ * launch's own message, naming the port, is more use than another attempt.
+ */
+export async function launchWithPortConflictRetry(opts) {
+    const spawn = opts.spawn ?? spawnSync;
+    try {
+        return await opts.attempt();
+    }
+    catch (e) {
+        const text = e instanceof Error ? e.message : String(e);
+        if (!isPortConflict(text))
+            throw e;
+        const port = extractConflictPort(text);
+        const reaped = [...reapInstance(opts.instanceId, spawn), ...(port !== null ? reapPort(port, spawn) : [])];
+        opts.log?.(`host port conflict on ${port !== null ? `:${port}` : "an unnamed port"} launching ${opts.instanceId}; ` +
+            `reaped ${reaped.length} container(s) and retrying once`);
+        return await opts.attempt();
+    }
+}

package/smoke.js

@@ -19,12 +19,13 @@
 // rule); a product wanting typed commands bridges with toArgv in its own spec.
 import { existsSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
-import { dumpInstanceContainerLogs, preserveEngineLogs } from "./container-logs.js";
+import { dumpInstanceContainerLogs, preserveEngineLogs, preserveInstanceContainerLogs } from "./container-logs.js";
 import { ensureRunnerOnNetwork, netReachMode, STUDIO_INTERNAL_PORT, studioBaseFrom, } from "./container-net.js";
 import { cleanupDaemon, requireLicenseFile, runCli, startDaemon, } from "./daemon.js";
 import { hashSlot } from "./harness-config.js";
 import { ctlSupportsNoPublish, ctlSupportsPublishDebugPorts, runnerContainerUser } from "./launch.js";
 import { pollUntil } from "./poll.js";
+import { launchWithPortConflictRetry, reapInstance } from "./reap.js";
 import { nukePathAsRoot, reportForeignOwners } from "./root-nuke.js";
 import { readProxySecret, startSrtSources } from "./source-pump.js";
 import { applyTestHost, fetchComponentState, fetchComponents, fetchStreamMappings, isSrtListenerState, } from "./studio-state.js";
@@ -339,7 +340,15 @@ export async function runProductSmoke(slug, spec, deps = defaultSmokeDeps) {
             launchArgv.push("--publish-debug-ports");
         }
         launched = true;
-        await cliOk(launchArgv);
+        // Same advisory-check race as the demo tier, same remedy: reap this id's
+        // orphans from a run that was killed before teardown, then retry once if the
+        // bind loses anyway (reap.ts).
+        reapInstance(instanceId);
+        await launchWithPortConflictRetry({
+            instanceId,
+            attempt: () => cliOk(launchArgv),
+            log: deps.log,
+        });
         await pollUntil(async () => {
             const r = await cli(["instance", "list"], { output: "json" });
             if (r.exitCode !== 0)
@@ -425,6 +434,11 @@ export async function runProductSmoke(slug, spec, deps = defaultSmokeDeps) {
     if (ciLogDir) {
         const n = preserveEngineLogs(storeDir, join(ciLogDir, slug));
         deps.log(`${slug}: preserved ${n} engine log tree(s) -> ${join(ciLogDir, slug)}`);
+        // And the half the bind-mounted tree does not hold: the STUDIO container's
+        // stdout, where "Refusing to run workflow" is printed. Still standing at
+        // this point; everything below removes it.
+        const c = preserveInstanceContainerLogs(instanceId, join(ciLogDir, slug, "containers", instanceId));
+        deps.log(`${slug}: preserved ${c} container log(s) -> ${join(ciLogDir, slug, "containers", instanceId)}`);
     }
     if (handles.length)
         await deps.stopSources(handles).catch(() => { });