@norskvideo/ctl-sdk 0.1.24 → 0.1.26

package/docker-runner.d.ts

@@ -13,8 +13,19 @@ export interface ProductRunOpts {
 }
 /** argv (sans leading "docker") that launches a product container: detached,
  *  auto-removed, loopback-published to its internal 4321, and role-labelled so
- *  it shows up in the Infrastructure tab. */
-export declare function productRunArgs(image: string, hostPort: number, opts?: ProductRunOpts): string[];
+ *  it shows up in the Infrastructure tab.
+ *
+ *  The host side of the publish is EMPTY on purpose. ctl used to pick it from
+ *  a fixed 14321+ range while consulting only its own registrations, which is
+ *  no allocator at all: it cannot see another ctl daemon, another compose
+ *  stack, or any other process on the host. Sibling CI runner containers
+ *  sharing one docker daemon both picked 14321 and the second died with
+ *  "driver failed programming external connectivity". Worse, a daemon that is
+ *  itself a container cannot even probe for the clash — a bind in its own
+ *  network namespace says nothing about the docker host's port space, which is
+ *  where the publish actually lands. So the allocation belongs to the docker
+ *  daemon, which owns that space; ctl reads back what it was given. */
+export declare function productRunArgs(image: string, opts?: ProductRunOpts): string[];
 /** Stable container name for a product, derived from its manifest name, so
  *  `docker ps` shows `norsk-product-studio` instead of a random docker alias.
  *  Mirrors the singleton naming of norsk-proxy / norsk-ctl-cpu-monitor. */
@@ -23,7 +34,23 @@ export declare function productContainerName(productName: string): string;
  *  so there is nothing to refresh; a tag can move under a registration. */
 export declare function isDigestRef(image: string): boolean;
 export declare function dockerPull(image: string): Promise<void>;
-export declare function dockerRun(image: string, hostPort: number, opts?: ProductRunOpts): Promise<string>;
+/** Where a started product container ended up: its id, and the host port
+ *  docker chose for it. The two are produced together because they change
+ *  together — the container is `--rm`, so every restart is a new container
+ *  with a new assignment and a stored port from the last one is already
+ *  stale. */
+export interface ProductPlacement {
+    containerId: string;
+    hostPort: number;
+}
+/** The host port from `docker inspect`'s HostPort lookup. Docker prints one
+ *  line per binding (an IPv4 and an IPv6 row for the same publish is normal),
+ *  and `<no value>` when the template finds nothing. */
+export declare function parsePublishedPort(inspectOutput: string): number | undefined;
+/** The host port docker published a container's internal 4321 on, or undefined
+ *  when docker cannot say. */
+export declare function dockerPublishedPort(containerId: string): Promise<number | undefined>;
+export declare function dockerRun(image: string, opts?: ProductRunOpts): Promise<ProductPlacement>;
 /** The container's address on its first network (the default bridge for a
  *  product container), or undefined when docker cannot say. */
 export declare function dockerAddress(containerId: string): Promise<string | undefined>;

package/docker-runner.js

@@ -7,8 +7,19 @@ export const CONTAINER_INTERNAL_PORT = 4321;
 export const PRODUCT_ROLE_LABEL = "norsk-ctl.role=product";
 /** argv (sans leading "docker") that launches a product container: detached,
  *  auto-removed, loopback-published to its internal 4321, and role-labelled so
- *  it shows up in the Infrastructure tab. */
-export function productRunArgs(image, hostPort, opts = {}) {
+ *  it shows up in the Infrastructure tab.
+ *
+ *  The host side of the publish is EMPTY on purpose. ctl used to pick it from
+ *  a fixed 14321+ range while consulting only its own registrations, which is
+ *  no allocator at all: it cannot see another ctl daemon, another compose
+ *  stack, or any other process on the host. Sibling CI runner containers
+ *  sharing one docker daemon both picked 14321 and the second died with
+ *  "driver failed programming external connectivity". Worse, a daemon that is
+ *  itself a container cannot even probe for the clash — a bind in its own
+ *  network namespace says nothing about the docker host's port space, which is
+ *  where the publish actually lands. So the allocation belongs to the docker
+ *  daemon, which owns that space; ctl reads back what it was given. */
+export function productRunArgs(image, opts = {}) {
     return [
         "run",
         "-d",
@@ -17,7 +28,7 @@ export function productRunArgs(image, hostPort, opts = {}) {
         PRODUCT_ROLE_LABEL,
         ...(opts.network ? ["--network", opts.network] : []),
         "-p",
-        `127.0.0.1:${hostPort}:${CONTAINER_INTERNAL_PORT}`,
+        `127.0.0.1::${CONTAINER_INTERNAL_PORT}`,
         image,
     ];
 }
@@ -44,15 +55,50 @@ export async function dockerPull(image) {
         throw new ProductError("DOCKER_PULL_FAILED", `docker pull failed (exit ${exitCode}): ${stderr || "no stderr"}`);
     }
 }
-export async function dockerRun(image, hostPort, opts = {}) {
-    const proc = Bun.spawn(["docker", ...productRunArgs(image, hostPort, opts)], { stdout: "pipe", stderr: "pipe" });
+/** The host port from `docker inspect`'s HostPort lookup. Docker prints one
+ *  line per binding (an IPv4 and an IPv6 row for the same publish is normal),
+ *  and `<no value>` when the template finds nothing. */
+export function parsePublishedPort(inspectOutput) {
+    for (const line of inspectOutput.split(/\r?\n/)) {
+        const port = Number.parseInt(line.trim(), 10);
+        if (Number.isInteger(port) && port > 0)
+            return port;
+    }
+    return undefined;
+}
+/** The host port docker published a container's internal 4321 on, or undefined
+ *  when docker cannot say. */
+export async function dockerPublishedPort(containerId) {
+    const proc = Bun.spawn([
+        "docker",
+        "inspect",
+        "-f",
+        `{{range index .NetworkSettings.Ports "${CONTAINER_INTERNAL_PORT}/tcp"}}{{.HostPort}}\n{{end}}`,
+        containerId,
+    ], { stdout: "pipe", stderr: "pipe" });
+    const exitCode = await proc.exited;
+    const stdout = await new Response(proc.stdout).text();
+    if (exitCode !== 0)
+        return undefined;
+    return parsePublishedPort(stdout);
+}
+export async function dockerRun(image, opts = {}) {
+    const proc = Bun.spawn(["docker", ...productRunArgs(image, opts)], { stdout: "pipe", stderr: "pipe" });
     const exitCode = await proc.exited;
     const stdout = (await new Response(proc.stdout).text()).trim();
     const stderr = (await new Response(proc.stderr).text()).trim();
     if (exitCode !== 0 || !stdout) {
         throw new ProductError("DOCKER_RUN_FAILED", `docker run failed (exit ${exitCode}): ${stderr || "no stderr"}`);
     }
-    return stdout;
+    const containerId = stdout;
+    const hostPort = await dockerPublishedPort(containerId);
+    if (hostPort === undefined) {
+        // Nothing to fall back to: the port is docker's to report, and a
+        // registration without one cannot be dialled or restarted.
+        await dockerRm(containerId);
+        throw new ProductError("DOCKER_RUN_FAILED", `docker did not report a published host port for ${CONTAINER_INTERNAL_PORT}/tcp`);
+    }
+    return { containerId, hostPort };
 }
 /** The container's address on its first network (the default bridge for a
  *  product container), or undefined when docker cannot say. */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@norskvideo/ctl-sdk",
-  "version": "0.1.24",
+  "version": "0.1.26",
   "type": "module",
   "exports": {
     ".": {

package/product-health-monitor.d.ts

@@ -8,14 +8,30 @@ export interface ProductHealthState {
     restartAttempts: number;
     /** Clock value of the most recent restart attempt; gates backoff. */
     lastRestartAt?: number;
+    /** URL the last failing probe dialled. Cleared on a healthy probe. */
+    lastProbeUrl?: string;
+    /** Why the last probe failed — status line or transport error. Cleared on
+     *  a healthy probe. */
+    lastProbeDetail?: string;
+}
+/** What one liveness probe saw. A bare boolean here cost three CI runs that
+ *  reported a product restart-looping with no record of what was dialled or
+ *  what came back, so the URL and the reason travel with the verdict. */
+export interface ProductProbeResult {
+    ok: boolean;
+    /** The address dialled, or "" when the probe declined to dial at all. */
+    url: string;
+    /** Present whenever `ok` is false: the HTTP status, the transport error, or
+     *  the reason no dial was attempted. */
+    detail?: string;
 }
 export interface ProductHealthMonitorOptions {
     /** Products to consider. Non-container products are ignored (externally
      *  owned), so this can safely be the unfiltered `productService.list`. */
     listProducts: () => ProductRegistration[];
-    /** Liveness probe — true means healthy. Injected so tests don't hit the
-     *  network; the daemon wires `probeProductHealth`. */
-    probe: (reg: ProductRegistration) => Promise<boolean>;
+    /** Liveness probe. Injected so tests don't hit the network; the daemon
+     *  wires `probeProductHealth`. */
+    probe: (reg: ProductRegistration) => Promise<ProductProbeResult>;
     /** Recovery action for a product over threshold. Typically
      *  `productService.restart`. Rejection counts as a failed attempt. */
     restart: (name: string) => Promise<void>;
@@ -69,4 +85,4 @@ export declare class ProductHealthMonitor {
  *  (default /healthz) with a short timeout. Any non-2xx, network error, or
  *  timeout reads as unhealthy. Container-only — dev products are externally
  *  owned and never reach here. */
-export declare function probeProductHealth(reg: ProductRegistration, timeoutMs?: number): Promise<boolean>;
+export declare function probeProductHealth(reg: ProductRegistration, timeoutMs?: number): Promise<ProductProbeResult>;

package/product-health-monitor.js

@@ -75,62 +75,83 @@ export class ProductHealthMonitor {
     }
     async checkOne(reg) {
         const prev = this.states.get(reg.name) ?? UNKNOWN;
-        let healthy;
+        let result;
         try {
-            healthy = await this.probe(reg);
+            result = await this.probe(reg);
         }
-        catch {
-            healthy = false;
+        catch (e) {
+            result = { ok: false, url: "", detail: `probe threw: ${e instanceof Error ? e.message : String(e)}` };
         }
-        if (healthy) {
+        if (result.ok) {
             this.set(reg.name, { status: "healthy", consecutiveFailures: 0, restartAttempts: 0 }, prev);
             return;
         }
         const consecutiveFailures = prev.consecutiveFailures + 1;
+        const probed = { lastProbeUrl: result.url, lastProbeDetail: result.detail };
+        // One line per unhealthy episode, not per sweep: enough to diagnose a
+        // restart loop without a warn every interval for a product known bad.
+        if (prev.consecutiveFailures === 0) {
+            logger.warn(`Product '${reg.name}' health probe failed: ${result.url || "not dialled"} — ${result.detail ?? "no detail"}`);
+        }
+        else {
+            logger.debug(`Product '${reg.name}' health probe failed (${consecutiveFailures}): ${result.url || "not dialled"} — ${result.detail ?? "no detail"}`);
+        }
         const belowThreshold = consecutiveFailures < this.failureThreshold;
         const gaveUp = prev.restartAttempts >= this.maxRestarts;
         const backoffElapsed = prev.lastRestartAt === undefined || this.now() - prev.lastRestartAt >= this.restartBackoffMs;
         if (belowThreshold || gaveUp || !backoffElapsed) {
-            this.set(reg.name, { ...prev, status: "unhealthy", consecutiveFailures }, prev);
+            this.set(reg.name, { ...prev, ...probed, status: "unhealthy", consecutiveFailures }, prev);
             return;
         }
         // Threshold reached, attempts left, backoff elapsed: attempt recovery.
         const at = this.now();
         const restartAttempts = prev.restartAttempts + 1;
-        this.set(reg.name, { status: "restarting", consecutiveFailures: 0, restartAttempts, lastRestartAt: at }, prev);
+        this.set(reg.name, { ...probed, status: "restarting", consecutiveFailures: 0, restartAttempts, lastRestartAt: at }, prev);
         try {
             await this.restart(reg.name);
             // Stay "restarting"; the next sweep re-probes to confirm recovery.
         }
         catch (e) {
             logger.warn(`Product '${reg.name}': restart failed — ${e instanceof Error ? e.message : String(e)}`);
-            this.set(reg.name, { status: "unhealthy", consecutiveFailures: 0, restartAttempts, lastRestartAt: at }, prev);
+            this.set(reg.name, { ...probed, status: "unhealthy", consecutiveFailures: 0, restartAttempts, lastRestartAt: at }, prev);
         }
     }
     set(name, next, prev) {
         this.states.set(name, next);
         if (next.status !== prev.status) {
-            logger.info(`Product '${name}' health: ${prev.status} -> ${next.status}`);
+            const why = next.status === "healthy" ? "" : detailSuffix(next);
+            logger.info(`Product '${name}' health: ${prev.status} -> ${next.status}${why}`);
             this.onChange?.(name, next);
         }
     }
 }
+/** Renders the recorded probe failure onto a transition log line, so the CI
+ *  log of a restart loop carries the address and the reason. */
+function detailSuffix(state) {
+    if (state.lastProbeUrl === undefined && state.lastProbeDetail === undefined)
+        return "";
+    return ` (${state.lastProbeUrl || "not dialled"} — ${state.lastProbeDetail ?? "no detail"})`;
+}
 /** Default liveness probe: GET the product's manifest-declared health path
  *  (default /healthz) with a short timeout. Any non-2xx, network error, or
  *  timeout reads as unhealthy. Container-only — dev products are externally
  *  owned and never reach here. */
 export async function probeProductHealth(reg, timeoutMs = 3_000) {
-    if (reg.spec.kind !== "container" || reg.port === undefined)
-        return false;
+    if (reg.spec.kind !== "container") {
+        return { ok: false, url: "", detail: `not a container product (kind '${reg.spec.kind}')` };
+    }
+    if (reg.port === undefined && reg.reachHost === undefined) {
+        return { ok: false, url: "", detail: "registration has no published port and no reachHost" };
+    }
     const path = reg.manifest.api?.healthCheckPath ?? "/healthz";
-    const base = productBaseUrl(reg);
+    const url = `${productBaseUrl(reg)}${path.startsWith("/") ? path : `/${path}`}`;
     try {
-        const r = await fetch(`${base}${path.startsWith("/") ? path : `/${path}`}`, {
-            signal: AbortSignal.timeout(timeoutMs),
-        });
-        return r.ok;
+        const r = await fetch(url, { signal: AbortSignal.timeout(timeoutMs) });
+        if (r.ok)
+            return { ok: true, url };
+        return { ok: false, url, detail: `HTTP ${r.status} ${r.statusText}`.trim() };
     }
-    catch {
-        return false;
+    catch (e) {
+        return { ok: false, url, detail: e instanceof Error ? `${e.name}: ${e.message}` : String(e) };
     }
 }

package/product-service.d.ts

@@ -1,3 +1,4 @@
+import { type ProductPlacement } from "./docker-runner.js";
 import { type LicenseStager } from "./license-registration.js";
 import { type ProductReach } from "./product-reach.js";
 import type { ProductTemplateSource } from "./product-template-record.js";
@@ -80,7 +81,9 @@ export type RefreshProductTemplateBytesFn = ImportProductTemplateBytesFn;
  *  made an injected fake silently inert on add/remove. */
 export interface ProductContainerOps {
     pull(image: string): Promise<void>;
-    run(image: string, hostPort: number): Promise<string>;
+    /** Start the container. The host port comes back from docker rather than
+     *  going in: docker owns the host port space (see docker-runner.ts). */
+    run(image: string): Promise<ProductPlacement>;
     remove(containerId: string): Promise<void>;
     rename(containerId: string, name: string): Promise<void>;
     waitForReady(baseUrl: string): Promise<void>;
@@ -94,7 +97,6 @@ export interface ProductContainerOps {
 export declare const defaultProductContainerOps: ProductContainerOps;
 export interface ProductServiceOptions {
     store: ProductStore;
-    allocatePort: AllocatePortFn;
     /** Optional: called once per `manifest.defaultProductTemplates` entry at
      *  registration time. Omit on hosts that don't store product templates
      *  (defaultProductTemplates is then silently skipped). */
@@ -135,7 +137,6 @@ export declare class ProductService {
      */
     private readonly mutations;
     private readonly store;
-    private readonly allocatePort;
     private readonly importProductTemplateBytes?;
     private readonly refreshProductTemplateBytes?;
     private readonly stageLicense?;

package/product-service.js

@@ -52,7 +52,6 @@ export class ProductService {
      */
     mutations = new Mutex();
     store;
-    allocatePort;
     importProductTemplateBytes;
     refreshProductTemplateBytes;
     stageLicense;
@@ -61,7 +60,6 @@ export class ProductService {
     reach;
     constructor(opts) {
         this.store = opts.store;
-        this.allocatePort = opts.allocatePort;
         this.importProductTemplateBytes = opts.importProductTemplateBytes;
         this.refreshProductTemplateBytes = opts.refreshProductTemplateBytes;
         this.stageLicense = opts.stageLicense;
@@ -117,18 +115,12 @@ export class ProductService {
             baseUrl = specBaseUrl(spec);
         }
         else {
-            const portMap = {};
-            for (const p of existing)
-                if (p.port !== undefined)
-                    portMap[p.name] = { port: p.port };
-            const allocated = this.allocatePort(portMap);
-            if (allocated === null) {
-                throw new ProductError("PORT_EXHAUSTED", `no free port available for new product`);
-            }
-            port = allocated;
             await this.refreshImage(spec.image);
-            logger.info(`Starting product container: ${spec.image} on host port ${port} (reach: ${this.reach})`);
-            containerId = await this.containerOps.run(spec.image, port);
+            logger.info(`Starting product container: ${spec.image} (reach: ${this.reach})`);
+            const placement = await this.containerOps.run(spec.image);
+            containerId = placement.containerId;
+            port = placement.hostPort;
+            logger.info(`Product container ${containerId.slice(0, 12)} published on host port ${port}`);
             reachHost = await this.reachHostFor(containerId);
             baseUrl = specBaseUrl(spec, port, reachHost);
         }
@@ -317,10 +309,10 @@ export class ProductService {
                     // Already gone (crashed / --rm reaped) — nothing to stop.
                 }
             }
-            const containerId = await this.containerOps.run(target.spec.image, target.port);
+            const { containerId, hostPort } = await this.containerOps.run(target.spec.image);
             const reachHost = await this.reachHostFor(containerId);
-            await this.store.update((products) => products.map((p) => (p.name === name ? withContainerId(p, containerId, reachHost) : p)));
-            await this.containerOps.waitForReady(specBaseUrl(target.spec, target.port, reachHost));
+            await this.store.update((products) => products.map((p) => (p.name === name ? withPlacement(p, containerId, hostPort, reachHost) : p)));
+            await this.containerOps.waitForReady(specBaseUrl(target.spec, hostPort, reachHost));
             await this.containerOps.rename(containerId, productContainerName(name));
             containerRestarted = true;
         }
@@ -408,12 +400,12 @@ export class ProductService {
                 return;
             }
             try {
-                const containerId = await this.containerOps.run(reg.spec.image, reg.port);
+                const { containerId, hostPort } = await this.containerOps.run(reg.spec.image);
                 const reachHost = await this.reachHostFor(containerId);
-                await this.containerOps.waitForReady(specBaseUrl(reg.spec, reg.port, reachHost));
+                await this.containerOps.waitForReady(specBaseUrl(reg.spec, hostPort, reachHost));
                 await this.containerOps.rename(containerId, productContainerName(reg.name));
-                restored.set(reg.name, { containerId, reachHost });
-                logger.info(`Product '${reg.name}' container restored on host port ${reg.port}`);
+                restored.set(reg.name, { containerId, hostPort, reachHost });
+                logger.info(`Product '${reg.name}' container restored on host port ${hostPort}`);
             }
             catch (e) {
                 restored.set(reg.name, undefined);
@@ -424,7 +416,9 @@ export class ProductService {
             if (!restored.has(p.name))
                 return p;
             const r = restored.get(p.name);
-            return withContainerId(p, r?.containerId, r?.reachHost);
+            return r === undefined
+                ? withContainerId(p, undefined)
+                : withPlacement(p, r.containerId, r.hostPort, r.reachHost);
         }));
     }
     /** Stop (if still present) and relaunch a single container-kind product,
@@ -455,12 +449,12 @@ export class ProductService {
                 // Already gone (crashed / --rm reaped) — nothing to stop.
             }
         }
-        const containerId = await this.containerOps.run(target.spec.image, target.port);
+        const { containerId, hostPort } = await this.containerOps.run(target.spec.image);
         const reachHost = await this.reachHostFor(containerId);
-        await this.store.update((products) => products.map((p) => (p.name === name ? withContainerId(p, containerId, reachHost) : p)));
-        await this.containerOps.waitForReady(specBaseUrl(target.spec, target.port, reachHost));
+        await this.store.update((products) => products.map((p) => (p.name === name ? withPlacement(p, containerId, hostPort, reachHost) : p)));
+        await this.containerOps.waitForReady(specBaseUrl(target.spec, hostPort, reachHost));
         await this.containerOps.rename(containerId, productContainerName(name));
-        logger.info(`Product '${name}' container restarted on host port ${target.port}`);
+        logger.info(`Product '${name}' container restarted on host port ${hostPort}`);
     }
 }
 /** Return a copy of `reg` with containerId (and the address it was reached at)
@@ -473,3 +467,9 @@ function withContainerId(reg, containerId, reachHost) {
         return rest;
     return { ...rest, containerId, ...(reachHost !== undefined ? { reachHost } : {}) };
 }
+/** As withContainerId, but also records the host port docker assigned THIS
+ *  container. Every (re)start is a new container with a new assignment, so the
+ *  port travels with the id or the next dial goes to the dead one's port. */
+function withPlacement(reg, containerId, hostPort, reachHost) {
+    return { ...withContainerId(reg, containerId, reachHost), port: hostPort };
+}