@norskvideo/ctl-test-harness 0.1.57 → 0.1.59

package/daemon-stores.d.ts

@@ -0,0 +1,5 @@
+/** Called by startDaemon. Last writer wins: a port reused by a later daemon in
+ *  the same process belongs to that daemon. */
+export declare function rememberDaemonStore(port: number, storeDir: string): void;
+export declare function forgetDaemonStore(port: number): void;
+export declare function storeDirForDaemonPort(port: number): string | undefined;

package/daemon-stores.js

@@ -0,0 +1,31 @@
+// Which store dir the daemon on a given port was started with.
+//
+// The daemon's ingress secret lives in its store (`<storeDir>/proxy-secret`),
+// and RFC 0003 D10b guards /api/* for loopback callers too — so anything
+// driving a test daemon's API needs that file. Every caller already has the
+// PORT; almost none of them has the store dir, because a test holds a harness
+// that started the daemon for it. That gap is why `startCameraSource` shipped
+// with a `proxySecret` option that nearly nobody passed, and why five product
+// repos' smoke tiers were 307'd for days.
+//
+// `startDaemon` knows both, so it records the pairing here and the pump looks
+// it up. Deliberately a process-global: the two are one-to-one within a test
+// process (one daemon per port), and the alternative — threading a store dir
+// through every helper signature and every call site — is the arrangement that
+// already failed.
+//
+// A daemon this process did not start (the demo driver's `reuse` policy, a
+// developer's own `norsk-ctl serve`) is absent, and callers that know their
+// store read it themselves.
+const storeByPort = new Map();
+/** Called by startDaemon. Last writer wins: a port reused by a later daemon in
+ *  the same process belongs to that daemon. */
+export function rememberDaemonStore(port, storeDir) {
+    storeByPort.set(port, storeDir);
+}
+export function forgetDaemonStore(port) {
+    storeByPort.delete(port);
+}
+export function storeDirForDaemonPort(port) {
+    return storeByPort.get(port);
+}

package/daemon.js

@@ -12,6 +12,7 @@ import { existsSync, readFileSync, rmSync, writeFileSync } from "node:fs";
 import { createServer } from "node:net";
 import { join } from "node:path";
 import { cliCommand } from "./cli-command.js";
+import { rememberDaemonStore } from "./daemon-stores.js";
 import { pollUntil } from "./poll.js";
 import { makeStoreDir, makeTempDir, TEST_TMP_BASE } from "./temp-dir.js";
 export const HEALTH_TIMEOUT_MS = 120_000;
@@ -146,6 +147,9 @@ export function seedDaemonConfig(storeDir) {
 }
 export function startDaemon(storeDir, options) {
     const port = options?.port ?? 8333;
+    // Before the spawn, so anything reaching for this daemon's ingress secret can
+    // find its store by the port it will answer on (daemon-stores.ts).
+    rememberDaemonStore(port, storeDir);
     killProcessOnPort(port);
     if (options?.seedConfig !== false)
         seedDaemonConfig(storeDir);

package/demo/harness.js

@@ -16,6 +16,7 @@
 import { netReachMode, STUDIO_INTERNAL_PORT, studioBaseFrom } from "../container-net.js";
 import { ctlSupportsPublishDebugPorts } from "../launch.js";
 import { pollUntil } from "../poll.js";
+import { launchWithPortConflictRetry, reapInstance } from "../reap.js";
 import { DemoGateFailure } from "./gates.js";
 import { DemoSession, defaultDemoDeps, demoSlug, expandHome, resolveIngestPort, } from "./run.js";
 const STUDIO_HOST_PORT_PARAM = "STUDIO_HOST_PORT";
@@ -165,7 +166,16 @@ export class ProductHarness {
         // the reap step whether or not this process lives to teardown.
         if (this.launchedManifest)
             this.deps.writeFile(this.launchedManifest, `${this.launchedIds.join("\n")}\n`);
-        await s.cliOk(launchArgv);
+        // A run cancelled before teardown leaves this id's compose stack up, still
+        // holding its host ports, and nothing that asks a live daemon can see it
+        // (the leaker's daemon is gone). Reap by label first, then retry once if the
+        // bind still loses the advisory-check race — see reap.ts.
+        reapInstance(instanceId);
+        await launchWithPortConflictRetry({
+            instanceId,
+            attempt: () => s.cliOk(launchArgv),
+            log: deps.log,
+        });
         await s.waitRunning(instanceId);
         const rows = await s.ingestPorts(instanceId);
         const templateParams = { declared, overrides: specParams };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@norskvideo/ctl-test-harness",
-  "version": "0.1.57",
+  "version": "0.1.59",
   "type": "module",
   "exports": {
     ".": {
@@ -15,6 +15,10 @@
       "types": "./daemon.d.ts",
       "default": "./daemon.js"
     },
+    "./daemon-stores": {
+      "types": "./daemon-stores.d.ts",
+      "default": "./daemon-stores.js"
+    },
     "./image-surface": {
       "types": "./image-surface.d.ts",
       "default": "./image-surface.js"
@@ -23,6 +27,10 @@
       "types": "./launch.d.ts",
       "default": "./launch.js"
     },
+    "./reap": {
+      "types": "./reap.d.ts",
+      "default": "./reap.js"
+    },
     "./source-pump": {
       "types": "./source-pump.d.ts",
       "default": "./source-pump.js"

package/reap.d.ts

@@ -0,0 +1,37 @@
+import { spawnSync } from "node:child_process";
+/** True when `text` is a host-port binding conflict — the signal to reap and
+ *  retry. Deliberately narrow: an unhealthy dependency is also a launch
+ *  failure, and retrying that one just burns the clock. */
+export declare function isPortConflict(text: string): boolean;
+/** The conflicting host port, or null when the message carries no extractable
+ *  one (still a conflict per {@link isPortConflict}, just not attributable to a
+ *  port, so only the instance-scoped reap can run). */
+export declare function extractConflictPort(text: string): number | null;
+/** `docker ps` argv listing container ids for ONE instance id — the exact scope
+ *  that spares co-running products. */
+export declare function instancePsArgs(instanceId: string): string[];
+/** `docker ps` argv listing norsk-ctl container ids publishing a given host
+ *  port. Role-filtered, so a squatter outside norsk-ctl is never touched. */
+export declare function portPsArgs(port: number): string[];
+type Spawn = typeof spawnSync;
+/** Reap every leftover container for one instance id (media, studio and source
+ *  all carry the `norsk-ctl.instance` label). */
+export declare function reapInstance(instanceId: string, spawn?: Spawn): string[];
+/** Reap whatever norsk-ctl container holds a specific host port — the fallback
+ *  when the holder belongs to a different instance, so the instance-id reap
+ *  alone would not free it. */
+export declare function reapPort(port: number, spawn?: Spawn): string[];
+/**
+ * Run a launch, and on a host-port conflict reap the holder and retry ONCE.
+ *
+ * Once, not a loop: one retry clears a leaked holder or a lost race, and a
+ * second conflict means something is genuinely sitting on the port — where the
+ * launch's own message, naming the port, is more use than another attempt.
+ */
+export declare function launchWithPortConflictRetry<T>(opts: {
+    instanceId: string;
+    attempt: () => Promise<T>;
+    spawn?: Spawn;
+    log?: (line: string) => void;
+}): Promise<T>;
+export {};

package/reap.js

@@ -0,0 +1,129 @@
+// Orphan-container reaper for launches that lose a host port.
+//
+// ctl's port-availability check at validation is ADVISORY — the probe holds no
+// reservation, and docker binds host ports at container START — so a port free
+// when the launch was validated can be taken by the time compose binds it
+// (`launch-service.ts`, HOST_PORT_CONFLICT). Its own message says to launch
+// again. On a shared self-hosted runner that race is routine rather than
+// exotic, in two flavours:
+//
+//   * a CANCELLED run: `cancel-in-progress` SIGKILLs `bun test` before the
+//     harness can tear down, so a `compose up -d` stack outlives its (now dead)
+//     daemon and keeps holding the port. Nothing that asks a LIVE daemon can
+//     see it, because the leaker's daemon is gone — which is why this reaps off
+//     Docker by label rather than through the API.
+//   * co-running products: six product repos' CI can fire within seconds of
+//     each other on one Docker daemon, and a fixed host port (the MoQ preview's
+//     `preferred: 4443`) is picked by each of them independently.
+//
+// Scoping is the whole safety argument. Every norsk-ctl container carries
+// `norsk-ctl.role`; per-instance ones also carry `norsk-ctl.instance=<id>`. We
+// reap by the exact instance id, and by port only among containers that carry
+// the role label — so a co-running product is untouched and a squatter outside
+// norsk-ctl is never removed, only reported by the launch that then fails.
+//
+// The regex and argv builders are pure so tests/reap.test.ts covers them with
+// zero Docker; only `reapInstance`/`reapPort` shell out.
+//
+// Lifted from norsk-ctl-product-playout's tests/integration/support/reap.ts,
+// which is where this was worked out. It lived in one product while probe,
+// commentary, reuters and funke had no equivalent and lost tests to the same
+// race, so it belongs here instead.
+import { spawnSync } from "node:child_process";
+// A launch failure that means "the host port I need is already bound". Covers
+// the daemon's own wrapper and the raw docker/kernel strings it quotes, since
+// either can surface depending on where the bind loses the race.
+const CONFLICT_PATTERNS = [
+    /was taken between validation and container start/i,
+    /port is already allocated/i,
+    /address already in use/i,
+];
+// Same failure, with the offending port captured. Ordered most-specific first:
+// the daemon wrapper names the port up front, the raw docker/kernel lines carry
+// it in the bind address.
+const PORT_PATTERNS = [
+    /host port (\d{2,5})\b/i,
+    /bind for [\d.:[\]]*?:(\d{2,5})\b[^:]*:\s*port is already allocated/i,
+    /:(\d{2,5}):\s*bind:\s*address already in use/i,
+    /port (\d{2,5}) .*already/i,
+];
+/** True when `text` is a host-port binding conflict — the signal to reap and
+ *  retry. Deliberately narrow: an unhealthy dependency is also a launch
+ *  failure, and retrying that one just burns the clock. */
+export function isPortConflict(text) {
+    return CONFLICT_PATTERNS.some((re) => re.test(text));
+}
+/** The conflicting host port, or null when the message carries no extractable
+ *  one (still a conflict per {@link isPortConflict}, just not attributable to a
+ *  port, so only the instance-scoped reap can run). */
+export function extractConflictPort(text) {
+    for (const re of PORT_PATTERNS) {
+        const m = text.match(re);
+        if (m?.[1])
+            return Number(m[1]);
+    }
+    return null;
+}
+/** `docker ps` argv listing container ids for ONE instance id — the exact scope
+ *  that spares co-running products. */
+export function instancePsArgs(instanceId) {
+    return ["ps", "-aq", "--filter", `label=norsk-ctl.instance=${instanceId}`];
+}
+/** `docker ps` argv listing norsk-ctl container ids publishing a given host
+ *  port. Role-filtered, so a squatter outside norsk-ctl is never touched. */
+export function portPsArgs(port) {
+    return ["ps", "-aq", "--filter", "label=norsk-ctl.role", "--filter", `publish=${port}`];
+}
+/** Force-remove the containers `docker ps <psArgs>` lists. Best-effort and
+ *  synchronous (it must finish before the relaunch); a Docker that is absent or
+ *  errors leaves the launch to fail with its own clearer message. Returns the
+ *  ids it removed, for logging. */
+function reap(psArgs, spawn) {
+    try {
+        const ps = spawn("docker", psArgs, { encoding: "utf8", timeout: 15_000 });
+        const ids = (ps.stdout ?? "")
+            .split("\n")
+            .map((s) => s.trim())
+            .filter(Boolean);
+        if (ids.length > 0)
+            spawn("docker", ["rm", "-f", ...ids], { encoding: "utf8", timeout: 30_000 });
+        return ids;
+    }
+    catch {
+        return [];
+    }
+}
+/** Reap every leftover container for one instance id (media, studio and source
+ *  all carry the `norsk-ctl.instance` label). */
+export function reapInstance(instanceId, spawn = spawnSync) {
+    return reap(instancePsArgs(instanceId), spawn);
+}
+/** Reap whatever norsk-ctl container holds a specific host port — the fallback
+ *  when the holder belongs to a different instance, so the instance-id reap
+ *  alone would not free it. */
+export function reapPort(port, spawn = spawnSync) {
+    return reap(portPsArgs(port), spawn);
+}
+/**
+ * Run a launch, and on a host-port conflict reap the holder and retry ONCE.
+ *
+ * Once, not a loop: one retry clears a leaked holder or a lost race, and a
+ * second conflict means something is genuinely sitting on the port — where the
+ * launch's own message, naming the port, is more use than another attempt.
+ */
+export async function launchWithPortConflictRetry(opts) {
+    const spawn = opts.spawn ?? spawnSync;
+    try {
+        return await opts.attempt();
+    }
+    catch (e) {
+        const text = e instanceof Error ? e.message : String(e);
+        if (!isPortConflict(text))
+            throw e;
+        const port = extractConflictPort(text);
+        const reaped = [...reapInstance(opts.instanceId, spawn), ...(port !== null ? reapPort(port, spawn) : [])];
+        opts.log?.(`host port conflict on ${port !== null ? `:${port}` : "an unnamed port"} launching ${opts.instanceId}; ` +
+            `reaped ${reaped.length} container(s) and retrying once`);
+        return await opts.attempt();
+    }
+}

package/smoke.js

@@ -25,6 +25,7 @@ import { cleanupDaemon, requireLicenseFile, runCli, startDaemon, } from "./daemo
 import { hashSlot } from "./harness-config.js";
 import { ctlSupportsNoPublish, ctlSupportsPublishDebugPorts, runnerContainerUser } from "./launch.js";
 import { pollUntil } from "./poll.js";
+import { launchWithPortConflictRetry, reapInstance } from "./reap.js";
 import { nukePathAsRoot, reportForeignOwners } from "./root-nuke.js";
 import { readProxySecret, startSrtSources } from "./source-pump.js";
 import { applyTestHost, fetchComponentState, fetchComponents, fetchStreamMappings, isSrtListenerState, } from "./studio-state.js";
@@ -339,7 +340,15 @@ export async function runProductSmoke(slug, spec, deps = defaultSmokeDeps) {
             launchArgv.push("--publish-debug-ports");
         }
         launched = true;
-        await cliOk(launchArgv);
+        // Same advisory-check race as the demo tier, same remedy: reap this id's
+        // orphans from a run that was killed before teardown, then retry once if the
+        // bind loses anyway (reap.ts).
+        reapInstance(instanceId);
+        await launchWithPortConflictRetry({
+            instanceId,
+            attempt: () => cliOk(launchArgv),
+            log: deps.log,
+        });
         await pollUntil(async () => {
             const r = await cli(["instance", "list"], { output: "json" });
             if (r.exitCode !== 0)

package/source-pump.d.ts

@@ -42,16 +42,17 @@ export interface StartCameraSourceOptions {
      *  (kept even when the fallback is used, so the handle is stable). Required
      *  when the same asset must run multiple times within an instance. */
     name?: string;
-    /** Sent as `x-norsk-proxy`. **Effectively required.** `readProxySecret`
-     *  reads it out of the store the daemon was started with; pass what that
-     *  returns.
+    /** Sent as `x-norsk-proxy`. **Usually leave this out**: when the daemon on
+     *  `daemonPort` was started by this process's harness, the secret is resolved
+     *  from its store automatically. Pass it only for a daemon the harness did
+     *  not start — the demo driver's `reuse` policy, or a developer's own
+     *  `norsk-ctl serve`.
      *
-     *  BREAKING for callers that omitted it (RFC 0003 D10b, ctl >= this
-     *  release): the daemon used to admit any loopback caller without a secret
-     *  while TLS was off, and that bypass is gone — a loopback caller is a host
-     *  admin, and a host admin can read the secret file. A harness or guide that
-     *  relied on it now fails on the guard's 307, which `guardRedirectError`
-     *  names rather than following. */
+     *  RFC 0003 D10b: the daemon used to admit any loopback caller without a
+     *  secret while TLS was off, and that bypass is gone — a loopback caller is a
+     *  host admin, and a host admin can read the secret file. When neither this
+     *  option nor the registry supplies one, the call meets the guard's 307,
+     *  which `guardRedirectError` names rather than following. */
     proxySecret?: string;
     timeoutMs?: number;
     /** Test seams. */

package/source-pump.js

@@ -18,6 +18,7 @@
 // file the daemon writes.
 import { existsSync, readFileSync } from "node:fs";
 import { basename, extname, join } from "node:path";
+import { storeDirForDaemonPort } from "./daemon-stores.js";
 import { pollUntil } from "./poll.js";
 /** Where every daemon the harness starts writes its ingress secret: the store
  *  root (the guard's stateDir is `<storeDir>` under NORSK_CTL_STORE_DIR). One
@@ -58,6 +59,12 @@ function assetFields(asset, fileExists) {
     }
     return { ...asset };
 }
+/** The secret of whichever daemon this process started on `port`, or undefined
+ *  for a daemon it did not start. */
+function secretForDaemonPort(port) {
+    const storeDir = storeDirForDaemonPort(port);
+    return storeDir === undefined ? undefined : readProxySecret(storeDir);
+}
 /** A 307/308 off `/api/*` is the daemon's guard, not a route that moved: the
  *  API itself never redirects. */
 function isGuardRedirect(res) {
@@ -70,7 +77,9 @@ function guardRedirectError(method, res, sentSecret) {
             ? "The x-norsk-proxy secret that was sent is not the one this daemon holds — it is re-issued on start, so a " +
                 "secret read before the daemon booted is stale (read it after `ready`)."
             : "No x-norsk-proxy secret was sent, and RFC 0003 D10b removed the loopback bypass, so a local caller needs " +
-                "one too. Read it with `readProxySecret(storeDir)` and pass it as `proxySecret`.") +
+                "one too. This process started no daemon on that port, so there was no store to read one from: either start " +
+                "it through the harness's startDaemon, or read the secret yourself with `readProxySecret(storeDir)` and pass " +
+                "it as `proxySecret`.") +
         " Following this redirect instead is what turns a missing secret into an unrelated-looking ConnectionRefused or " +
         "a `Failed to parse JSON` on nginx's HTML.");
 }
@@ -81,7 +90,12 @@ export async function startCameraSource(opts) {
     const asset = opts.asset ?? { preset: opts.preset ?? "camera1" };
     const name = opts.name ?? defaultSourceName(asset);
     const fetchImpl = opts.fetch ?? ((url, init) => fetch(url, init));
-    const headers = opts.proxySecret ? { "x-norsk-proxy": opts.proxySecret } : {};
+    // Resolved here, not at the call site: every caller has the port and almost
+    // none has the store dir, which is exactly why the explicit option went
+    // unpassed. Read now rather than when the daemon was registered — the guard
+    // issues the secret as the proxy comes up, after startDaemon returns.
+    const secret = opts.proxySecret ?? secretForDaemonPort(opts.daemonPort);
+    const headers = secret ? { "x-norsk-proxy": secret } : {};
     const base = `http://localhost:${opts.daemonPort}`;
     const body = {
         instanceId: opts.instanceId,