@norskvideo/ctl-test-harness 0.1.58 → 0.1.59

package/demo/harness.js

@@ -16,6 +16,7 @@
 import { netReachMode, STUDIO_INTERNAL_PORT, studioBaseFrom } from "../container-net.js";
 import { ctlSupportsPublishDebugPorts } from "../launch.js";
 import { pollUntil } from "../poll.js";
+import { launchWithPortConflictRetry, reapInstance } from "../reap.js";
 import { DemoGateFailure } from "./gates.js";
 import { DemoSession, defaultDemoDeps, demoSlug, expandHome, resolveIngestPort, } from "./run.js";
 const STUDIO_HOST_PORT_PARAM = "STUDIO_HOST_PORT";
@@ -165,7 +166,16 @@ export class ProductHarness {
         // the reap step whether or not this process lives to teardown.
         if (this.launchedManifest)
             this.deps.writeFile(this.launchedManifest, `${this.launchedIds.join("\n")}\n`);
-        await s.cliOk(launchArgv);
+        // A run cancelled before teardown leaves this id's compose stack up, still
+        // holding its host ports, and nothing that asks a live daemon can see it
+        // (the leaker's daemon is gone). Reap by label first, then retry once if the
+        // bind still loses the advisory-check race — see reap.ts.
+        reapInstance(instanceId);
+        await launchWithPortConflictRetry({
+            instanceId,
+            attempt: () => s.cliOk(launchArgv),
+            log: deps.log,
+        });
         await s.waitRunning(instanceId);
         const rows = await s.ingestPorts(instanceId);
         const templateParams = { declared, overrides: specParams };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@norskvideo/ctl-test-harness",
-  "version": "0.1.58",
+  "version": "0.1.59",
   "type": "module",
   "exports": {
     ".": {
@@ -27,6 +27,10 @@
       "types": "./launch.d.ts",
       "default": "./launch.js"
     },
+    "./reap": {
+      "types": "./reap.d.ts",
+      "default": "./reap.js"
+    },
     "./source-pump": {
       "types": "./source-pump.d.ts",
       "default": "./source-pump.js"

package/reap.d.ts

@@ -0,0 +1,37 @@
+import { spawnSync } from "node:child_process";
+/** True when `text` is a host-port binding conflict — the signal to reap and
+ *  retry. Deliberately narrow: an unhealthy dependency is also a launch
+ *  failure, and retrying that one just burns the clock. */
+export declare function isPortConflict(text: string): boolean;
+/** The conflicting host port, or null when the message carries no extractable
+ *  one (still a conflict per {@link isPortConflict}, just not attributable to a
+ *  port, so only the instance-scoped reap can run). */
+export declare function extractConflictPort(text: string): number | null;
+/** `docker ps` argv listing container ids for ONE instance id — the exact scope
+ *  that spares co-running products. */
+export declare function instancePsArgs(instanceId: string): string[];
+/** `docker ps` argv listing norsk-ctl container ids publishing a given host
+ *  port. Role-filtered, so a squatter outside norsk-ctl is never touched. */
+export declare function portPsArgs(port: number): string[];
+type Spawn = typeof spawnSync;
+/** Reap every leftover container for one instance id (media, studio and source
+ *  all carry the `norsk-ctl.instance` label). */
+export declare function reapInstance(instanceId: string, spawn?: Spawn): string[];
+/** Reap whatever norsk-ctl container holds a specific host port — the fallback
+ *  when the holder belongs to a different instance, so the instance-id reap
+ *  alone would not free it. */
+export declare function reapPort(port: number, spawn?: Spawn): string[];
+/**
+ * Run a launch, and on a host-port conflict reap the holder and retry ONCE.
+ *
+ * Once, not a loop: one retry clears a leaked holder or a lost race, and a
+ * second conflict means something is genuinely sitting on the port — where the
+ * launch's own message, naming the port, is more use than another attempt.
+ */
+export declare function launchWithPortConflictRetry<T>(opts: {
+    instanceId: string;
+    attempt: () => Promise<T>;
+    spawn?: Spawn;
+    log?: (line: string) => void;
+}): Promise<T>;
+export {};

package/reap.js

@@ -0,0 +1,129 @@
+// Orphan-container reaper for launches that lose a host port.
+//
+// ctl's port-availability check at validation is ADVISORY — the probe holds no
+// reservation, and docker binds host ports at container START — so a port free
+// when the launch was validated can be taken by the time compose binds it
+// (`launch-service.ts`, HOST_PORT_CONFLICT). Its own message says to launch
+// again. On a shared self-hosted runner that race is routine rather than
+// exotic, in two flavours:
+//
+//   * a CANCELLED run: `cancel-in-progress` SIGKILLs `bun test` before the
+//     harness can tear down, so a `compose up -d` stack outlives its (now dead)
+//     daemon and keeps holding the port. Nothing that asks a LIVE daemon can
+//     see it, because the leaker's daemon is gone — which is why this reaps off
+//     Docker by label rather than through the API.
+//   * co-running products: six product repos' CI can fire within seconds of
+//     each other on one Docker daemon, and a fixed host port (the MoQ preview's
+//     `preferred: 4443`) is picked by each of them independently.
+//
+// Scoping is the whole safety argument. Every norsk-ctl container carries
+// `norsk-ctl.role`; per-instance ones also carry `norsk-ctl.instance=<id>`. We
+// reap by the exact instance id, and by port only among containers that carry
+// the role label — so a co-running product is untouched and a squatter outside
+// norsk-ctl is never removed, only reported by the launch that then fails.
+//
+// The regex and argv builders are pure so tests/reap.test.ts covers them with
+// zero Docker; only `reapInstance`/`reapPort` shell out.
+//
+// Lifted from norsk-ctl-product-playout's tests/integration/support/reap.ts,
+// which is where this was worked out. It lived in one product while probe,
+// commentary, reuters and funke had no equivalent and lost tests to the same
+// race, so it belongs here instead.
+import { spawnSync } from "node:child_process";
+// A launch failure that means "the host port I need is already bound". Covers
+// the daemon's own wrapper and the raw docker/kernel strings it quotes, since
+// either can surface depending on where the bind loses the race.
+const CONFLICT_PATTERNS = [
+    /was taken between validation and container start/i,
+    /port is already allocated/i,
+    /address already in use/i,
+];
+// Same failure, with the offending port captured. Ordered most-specific first:
+// the daemon wrapper names the port up front, the raw docker/kernel lines carry
+// it in the bind address.
+const PORT_PATTERNS = [
+    /host port (\d{2,5})\b/i,
+    /bind for [\d.:[\]]*?:(\d{2,5})\b[^:]*:\s*port is already allocated/i,
+    /:(\d{2,5}):\s*bind:\s*address already in use/i,
+    /port (\d{2,5}) .*already/i,
+];
+/** True when `text` is a host-port binding conflict — the signal to reap and
+ *  retry. Deliberately narrow: an unhealthy dependency is also a launch
+ *  failure, and retrying that one just burns the clock. */
+export function isPortConflict(text) {
+    return CONFLICT_PATTERNS.some((re) => re.test(text));
+}
+/** The conflicting host port, or null when the message carries no extractable
+ *  one (still a conflict per {@link isPortConflict}, just not attributable to a
+ *  port, so only the instance-scoped reap can run). */
+export function extractConflictPort(text) {
+    for (const re of PORT_PATTERNS) {
+        const m = text.match(re);
+        if (m?.[1])
+            return Number(m[1]);
+    }
+    return null;
+}
+/** `docker ps` argv listing container ids for ONE instance id — the exact scope
+ *  that spares co-running products. */
+export function instancePsArgs(instanceId) {
+    return ["ps", "-aq", "--filter", `label=norsk-ctl.instance=${instanceId}`];
+}
+/** `docker ps` argv listing norsk-ctl container ids publishing a given host
+ *  port. Role-filtered, so a squatter outside norsk-ctl is never touched. */
+export function portPsArgs(port) {
+    return ["ps", "-aq", "--filter", "label=norsk-ctl.role", "--filter", `publish=${port}`];
+}
+/** Force-remove the containers `docker ps <psArgs>` lists. Best-effort and
+ *  synchronous (it must finish before the relaunch); a Docker that is absent or
+ *  errors leaves the launch to fail with its own clearer message. Returns the
+ *  ids it removed, for logging. */
+function reap(psArgs, spawn) {
+    try {
+        const ps = spawn("docker", psArgs, { encoding: "utf8", timeout: 15_000 });
+        const ids = (ps.stdout ?? "")
+            .split("\n")
+            .map((s) => s.trim())
+            .filter(Boolean);
+        if (ids.length > 0)
+            spawn("docker", ["rm", "-f", ...ids], { encoding: "utf8", timeout: 30_000 });
+        return ids;
+    }
+    catch {
+        return [];
+    }
+}
+/** Reap every leftover container for one instance id (media, studio and source
+ *  all carry the `norsk-ctl.instance` label). */
+export function reapInstance(instanceId, spawn = spawnSync) {
+    return reap(instancePsArgs(instanceId), spawn);
+}
+/** Reap whatever norsk-ctl container holds a specific host port — the fallback
+ *  when the holder belongs to a different instance, so the instance-id reap
+ *  alone would not free it. */
+export function reapPort(port, spawn = spawnSync) {
+    return reap(portPsArgs(port), spawn);
+}
+/**
+ * Run a launch, and on a host-port conflict reap the holder and retry ONCE.
+ *
+ * Once, not a loop: one retry clears a leaked holder or a lost race, and a
+ * second conflict means something is genuinely sitting on the port — where the
+ * launch's own message, naming the port, is more use than another attempt.
+ */
+export async function launchWithPortConflictRetry(opts) {
+    const spawn = opts.spawn ?? spawnSync;
+    try {
+        return await opts.attempt();
+    }
+    catch (e) {
+        const text = e instanceof Error ? e.message : String(e);
+        if (!isPortConflict(text))
+            throw e;
+        const port = extractConflictPort(text);
+        const reaped = [...reapInstance(opts.instanceId, spawn), ...(port !== null ? reapPort(port, spawn) : [])];
+        opts.log?.(`host port conflict on ${port !== null ? `:${port}` : "an unnamed port"} launching ${opts.instanceId}; ` +
+            `reaped ${reaped.length} container(s) and retrying once`);
+        return await opts.attempt();
+    }
+}

package/smoke.js

@@ -25,6 +25,7 @@ import { cleanupDaemon, requireLicenseFile, runCli, startDaemon, } from "./daemo
 import { hashSlot } from "./harness-config.js";
 import { ctlSupportsNoPublish, ctlSupportsPublishDebugPorts, runnerContainerUser } from "./launch.js";
 import { pollUntil } from "./poll.js";
+import { launchWithPortConflictRetry, reapInstance } from "./reap.js";
 import { nukePathAsRoot, reportForeignOwners } from "./root-nuke.js";
 import { readProxySecret, startSrtSources } from "./source-pump.js";
 import { applyTestHost, fetchComponentState, fetchComponents, fetchStreamMappings, isSrtListenerState, } from "./studio-state.js";
@@ -339,7 +340,15 @@ export async function runProductSmoke(slug, spec, deps = defaultSmokeDeps) {
             launchArgv.push("--publish-debug-ports");
         }
         launched = true;
-        await cliOk(launchArgv);
+        // Same advisory-check race as the demo tier, same remedy: reap this id's
+        // orphans from a run that was killed before teardown, then retry once if the
+        // bind loses anyway (reap.ts).
+        reapInstance(instanceId);
+        await launchWithPortConflictRetry({
+            instanceId,
+            attempt: () => cliOk(launchArgv),
+            log: deps.log,
+        });
         await pollUntil(async () => {
             const r = await cli(["instance", "list"], { output: "json" });
             if (r.exitCode !== 0)