@omnitron-dev/omnitron 0.1.3 → 0.3.0

package/dist/infrastructure/container-runtime.js

@@ -7,8 +7,66 @@
  *
  * All Omnitron-managed containers are labeled with `omnitron.managed=true`.
  */
+import { createHash } from 'node:crypto';
+import { CONFIG_ROOT_PREFIX } from './shipped-config.js';
+/** Label carrying the desired-spec fingerprint, used for config-drift detection. */
+export const SPEC_HASH_LABEL = 'omnitron.spec-hash';
+/**
+ * Stable fingerprint of a container's *desired* spec. Covers the fields that a
+ * config edit can change and that require a recreate to apply (image, env,
+ * ports, volumes, command/entrypoint, extraHosts). Deliberately EXCLUDES
+ * labels — the spec-hash label is derived from this, and docker-compose adds
+ * label noise — so the hash is stable across recreations of an unchanged spec.
+ *
+ * Stamped as a label at create time and compared on reconcile so that an env
+ * change (e.g. the Tor hidden-service target) triggers a recreate instead of
+ * being silently ignored because the image is unchanged.
+ */
+export function containerSpecHash(config) {
+    const sortedEnv = Object.keys(config.environment)
+        .sort()
+        .map((k) => [k, config.environment[k]]);
+    const normalized = JSON.stringify({
+        image: config.image,
+        env: sortedEnv,
+        // `bindHost` is part of what a port IS. Without it, changing a container
+        // from every-interface to loopback leaves the hash identical and the
+        // container published exactly as before.
+        ports: [...config.ports]
+            .map((p) => ({ h: p.host, c: p.container, b: p.bindHost ?? null }))
+            .sort((a, b) => a.h - b.h || a.c - b.c),
+        volumes: [...config.volumes]
+            .map((v) => ({ s: v.source, t: v.target, ro: !!v.readonly }))
+            .sort((a, b) => (a.t < b.t ? -1 : a.t > b.t ? 1 : 0)),
+        command: config.command ?? null,
+        entrypoint: config.entrypoint ?? null,
+        extraHosts: [...(config.extraHosts ?? [])].sort(),
+        // A health check is baked into the container at creation: docker reads
+        // `HEALTHCHECK` once and never again. Leaving it out of the hash means a
+        // corrected check never reaches anything already running — the fix lands
+        // in the code, the reconciler sees no difference, and the container goes
+        // on answering the old question forever.
+        //
+        // Measured: the gateway's probe was corrected from `curl` (absent from
+        // its image) to one that also tries wget, the node was reprovisioned,
+        // and the container kept the curl-only check and kept reporting
+        // unhealthy.
+        healthCheck: config.healthCheck ?? null,
+        // The network a container is on is not a detail of how it runs but of
+        // what it can reach. A service moved between networks and not recreated
+        // is on the old one, and its name resolves for nobody.
+        network: config.network ?? null,
+        restart: config.restart,
+        // What the mounted config files SAY, which the volume paths above cannot:
+        // an nginx.conf rewritten in place has the same path. Only when present,
+        // so a container with no config files keeps the hash it was created
+        // with and an upgrade of omnitron recreates nothing on its account.
+        ...(config.configDigest ? { configDigest: config.configDigest } : {}),
+    });
+    return createHash('sha1').update(normalized).digest('hex').slice(0, 16);
+}
 // =============================================================================
-// Singleton DockerAdapter (lazy-loaded)
+// Singleton DockerAdapter (lazy-loaded, self-healing)
 // =============================================================================
 let _adapter = null;
 async function getAdapter() {
@@ -18,16 +76,78 @@ async function getAdapter() {
     _adapter = new DockerAdapter();
     return _adapter;
 }
+/**
+ * Patterns that mean "the docker daemon connection is gone" rather than
+ * "this specific operation failed". When we see one, we invalidate the
+ * cached adapter so the next call gets a fresh instance — without this,
+ * a docker-daemon restart would leave us forever trying to talk to a
+ * dead socket.
+ */
+const ADAPTER_DEAD_PATTERNS = /(?:cannot connect to the docker daemon|docker daemon is not running|connect ENOENT|connect ECONNREFUSED.*docker|EOF$|connection reset)/i;
+function isAdapterDeadError(err) {
+    if (!err)
+        return false;
+    const msg = err instanceof Error ? err.message : String(err);
+    return ADAPTER_DEAD_PATTERNS.test(msg);
+}
+/**
+ * Run `op` against the docker adapter. If it fails with an error that
+ * indicates the daemon connection is dead, invalidate the cached adapter
+ * and retry exactly once with a fresh one — recovering automatically
+ * from docker daemon restarts.
+ */
+async function withAdapter(op) {
+    const adapter = await getAdapter();
+    try {
+        return await op(adapter);
+    }
+    catch (err) {
+        if (!isAdapterDeadError(err))
+            throw err;
+        _adapter = null;
+        const fresh = await getAdapter();
+        return await op(fresh);
+    }
+}
 // =============================================================================
 // Public API
 // =============================================================================
+/**
+ * How many containers this host's Docker holds, by state — every container,
+ * not only omnitron's.
+ *
+ * One `docker ps`, the CLI the rest of this file speaks, with a timeout that
+ * ends it. The system snapshot asked Docker through systeminformation's
+ * socket client instead, which sends `GET /info HTTP/1.0` and waits for the
+ * connection to close; OrbStack answers `HTTP/1.1 200` and keeps it open, so
+ * every question hung until the library's own timeout — measured 2026-09-23:
+ * no answer in 20 s where `curl` on the same socket took 0.055 s — and left
+ * its socket open behind it.
+ */
+export async function countContainersByState(timeoutMs = 10_000) {
+    const { execFile } = await import('node:child_process');
+    const { promisify } = await import('node:util');
+    const { stdout } = await promisify(execFile)('docker', ['ps', '-a', '--format', '{{.State}}'], {
+        encoding: 'utf-8',
+        timeout: timeoutMs,
+    });
+    const counts = { running: 0, paused: 0, stopped: 0 };
+    for (const state of String(stdout).split('\n').map((line) => line.trim()).filter(Boolean)) {
+        if (state === 'running')
+            counts.running++;
+        else if (state === 'paused')
+            counts.paused++;
+        else
+            counts.stopped++;
+    }
+    return counts;
+}
 /**
  * Check if Docker daemon is available.
  */
 export async function isDockerAvailable() {
     try {
-        const adapter = await getAdapter();
-        return adapter.isAvailable();
+        return await withAdapter((adapter) => adapter.isAvailable());
     }
     catch {
         return false;
@@ -35,59 +155,272 @@ export async function isDockerAvailable() {
 }
 /**
  * List all Omnitron-managed containers.
+ *
+ * `[]` when Docker cannot be asked, which is right for a listing and wrong for
+ * a health check: «nothing is running» and «Docker did not answer» are
+ * different states. `orThrow` keeps them apart.
  */
-export async function listManagedContainers() {
-    const adapter = await getAdapter();
+export async function listManagedContainers(options = {}) {
     try {
-        // listContainers returns names; filter by inspecting labels
-        const allNames = await adapter.listContainers(true);
+        const names = await withAdapter((adapter) => adapter.listContainers(true));
+        if (names.length === 0)
+            return [];
+        // One `docker inspect` for every container, not one per container.
+        //
+        // The per-container loop this replaces issued a subprocess each: on a host
+        // with 28 containers that is 28 spawns, and the console polls this every
+        // ten seconds. Measured on that host: 0.042s for one inspect, 0.225s for
+        // all 28 in a single call — so the loop cost roughly 1.2s of process
+        // churn per poll when it was working, and under load it stopped returning
+        // at all, leaving the containers page in its loading state indefinitely.
+        const inspected = await inspectContainers(names);
+        // Named and then not described: `inspectContainers` swallows its own
+        // failure, and an empty answer here is Docker going away mid-question.
+        if (options.orThrow && inspected.length === 0) {
+            throw new Error(`docker listed ${names.length} container(s) and then described none of them`);
+        }
         const managed = [];
-        for (const name of allNames) {
-            try {
-                const info = await adapter.inspectContainer(name);
-                const labels = info?.Config?.Labels ?? {};
-                if (labels['omnitron.managed'] === 'true') {
-                    managed.push({
-                        name: (info.Name ?? name).replace(/^\//, ''),
-                        image: info.Config?.Image ?? '',
-                        status: mapInspectStatus(info.State?.Status),
-                        containerId: info.Id?.slice(0, 12),
-                        health: mapInspectHealth(info.State?.Health?.Status),
-                    });
-                }
-            }
-            catch {
-                // Container may have been removed between list and inspect
-            }
+        for (const info of inspected) {
+            if (info?.Config?.Labels?.['omnitron.managed'] !== 'true')
+                continue;
+            const failure = describeContainerFailure(info.State);
+            managed.push({
+                name: (info.Name ?? '').replace(/^\//, ''),
+                image: info.Config?.Image ?? '',
+                status: mapInspectStatus(info.State?.Status),
+                containerId: info.Id?.slice(0, 12),
+                health: mapInspectHealth(info.State?.Health?.Status),
+                startedAt: info.State?.StartedAt,
+                ports: publishedPorts(info),
+                service: info.Config?.Labels?.['omnitron.service'],
+                project: info.Config?.Labels?.['omnitron.project'],
+                stack: info.Config?.Labels?.['omnitron.stack'],
+                app: info.Config?.Labels?.['omnitron.app'],
+                ...(failure && { error: failure }),
+            });
         }
         return managed;
     }
-    catch {
+    catch (err) {
+        if (options.orThrow)
+            throw err;
         return [];
     }
 }
+/**
+ * Published ports, as `"80/tcp" -> 9800`.
+ *
+ * `ContainerState.ports` was declared, documented as a name→port map, and
+ * assigned by nothing — so the console's Ports column showed `--` for every
+ * container on a host where all twelve publish ports. The data was already in
+ * hand: `docker inspect` returns it, and this reads it out of the response
+ * both call sites already have.
+ *
+ * The keys are container ports rather than the logical names a service config
+ * uses (`main`, `rpc`), because that is what Docker knows. A container that
+ * exposes a port without publishing it has a null binding and is left out —
+ * it is not reachable from the host, so listing it would overstate what is
+ * available.
+ */
+function publishedPorts(info) {
+    const raw = info.NetworkSettings?.Ports;
+    if (!raw)
+        return undefined;
+    const out = {};
+    for (const [containerPort, bindings] of Object.entries(raw)) {
+        // Docker lists both the IPv4 and IPv6 binding for the same publish; they
+        // carry the same host port, so the first usable one is the answer.
+        const hostPort = bindings?.find((b) => b?.HostPort)?.HostPort;
+        if (!hostPort)
+            continue;
+        const port = Number(hostPort);
+        if (Number.isInteger(port) && port > 0)
+            out[containerPort] = port;
+    }
+    return Object.keys(out).length > 0 ? out : undefined;
+}
+/**
+ * Inspect many containers in one call.
+ *
+ * `docker inspect` takes any number of names and returns a JSON array. Names
+ * that no longer exist are reported on stderr and simply absent from the
+ * array — which is the behaviour wanted here, since a container can be
+ * removed between listing and inspecting.
+ */
+async function inspectContainers(names) {
+    const { execFile } = await import('node:child_process');
+    const { promisify } = await import('node:util');
+    const execFileAsync = promisify(execFile);
+    try {
+        const { stdout } = await execFileAsync('docker', ['inspect', ...names], {
+            timeout: 15_000,
+            maxBuffer: 32 * 1024 * 1024,
+        });
+        const parsed = JSON.parse(stdout || '[]');
+        return Array.isArray(parsed) ? parsed : [];
+    }
+    catch (err) {
+        // A non-zero exit means SOME name was unknown; docker still prints the
+        // ones it found, so the output is worth parsing rather than discarding.
+        const stdout = err.stdout;
+        if (!stdout)
+            return [];
+        try {
+            const parsed = JSON.parse(stdout);
+            return Array.isArray(parsed) ? parsed : [];
+        }
+        catch {
+            return [];
+        }
+    }
+}
 /**
  * Get state of a specific container by name.
  */
 export async function getContainerState(name) {
-    const adapter = await getAdapter();
     try {
-        const exists = await adapter.containerExists(name);
-        if (!exists)
-            return null;
-        const info = await adapter.inspectContainer(name);
-        return {
-            name: (info.Name ?? name).replace(/^\//, ''),
-            image: info.Config?.Image ?? '',
-            status: mapInspectStatus(info.State?.Status),
-            containerId: info.Id?.slice(0, 12),
-            health: mapInspectHealth(info.State?.Health?.Status),
-        };
+        return await withAdapter(async (adapter) => {
+            const exists = await adapter.containerExists(name);
+            if (!exists)
+                return null;
+            const info = await adapter.inspectContainer(name);
+            // Network attachment: an OrbStack/dockerd restart can leave a container
+            // 'running' but detached from every network (empty Networks) with its
+            // published ports gone — reachable by nothing. Surface it so callers can
+            // recreate instead of trusting the 'running' status.
+            const networkAttached = Object.keys(info.NetworkSettings?.Networks ?? {}).length > 0;
+            const failure = describeContainerFailure(info.State);
+            return {
+                name: (info.Name ?? name).replace(/^\//, ''),
+                image: info.Config?.Image ?? '',
+                status: mapInspectStatus(info.State?.Status),
+                containerId: info.Id?.slice(0, 12),
+                health: mapInspectHealth(info.State?.Health?.Status),
+                startedAt: info.State?.StartedAt,
+                ports: publishedPorts(info),
+                specHash: info.Config?.Labels?.[SPEC_HASH_LABEL],
+                service: info.Config?.Labels?.['omnitron.service'],
+                networkAttached,
+                ...(failure && { error: failure }),
+            };
+        });
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * One `-p` value.
+ *
+ * `9800:80` publishes on every INTERFACE — Docker's default, and Docker's
+ * iptables rules are inserted ahead of the host firewall's, so such a port is
+ * reachable from the network no matter what `ufw status` says.
+ *
+ * An omission therefore used to mean the widest possible answer. This is the
+ * single place a published port becomes a docker argument, and it now reads an
+ * absent `bindHost` as loopback: a container that must be reachable says so,
+ * with `bindHost: '0.0.0.0'`, in a line someone can see in a diff.
+ *
+ * `applyManagedDefaults` fills the same field in the resolver, and should —
+ * that is what puts the binding into `containerSpecHash`, so changing it
+ * recreates the container instead of leaving the old one published. But a
+ * default that every resolver must remember is a default that one of them
+ * will not: `resolveGateway` was the one return of four that did not call it,
+ * and the gateway is the container that faces outward. Measured on the dev
+ * stand: `daos-dev-gateway` published on 0.0.0.0:80 while the other ten
+ * managed containers were all on 127.0.0.1, and a browser on the LAN could
+ * sign in. Defence in depth, because the resolver-side default is the one
+ * that can be forgotten and this one cannot.
+ */
+export function portArg(p) {
+    const bindHost = p.bindHost ?? '127.0.0.1';
+    return `${bindHost}:${p.host}:${p.container}`;
+}
+/**
+ * Docker state left over from a container that is gone.
+ *
+ * An orphaned libnetwork endpoint, or a `Created` husk holding the name.
+ * Both are scrubbed and the run retried.
+ */
+export function isStaleContainerState(message) {
+    return /endpoint with name .* already exists|is already in use by container/i.test(message);
+}
+/**
+ * A port whose previous holder has not let go yet.
+ *
+ * `docker rm` returns before the kernel releases the bind its docker-proxy
+ * held, so removing a container and immediately recreating it races that
+ * release. Measured on a node recreating postgres, redis and minio at once:
+ * all three failed this way, each left in `Created`, and `docker start` by
+ * hand a second later worked — which is what makes it a race rather than a
+ * conflict, and a retry the right answer rather than a louder error.
+ *
+ * Narrow on purpose. A missing image, a bad flag, or a port genuinely held
+ * by something else is permanent, and retrying it turns one clear error into
+ * several identical ones separated by delays.
+ */
+export function isPortNotYetReleased(message) {
+    return /port is already allocated|failed to set up container networking/i.test(message);
+}
+/**
+ * Which container holds a host port, if any.
+ *
+ * `port is already allocated` is true and useless: it names the port and
+ * withholds the only thing an operator can act on. Measured on a node where
+ * three services would not start —
+ *
+ *     Bind for 127.0.0.1:5432 failed: port is already allocated
+ *
+ * — and the holder was `omnitron-postgres`, a container from an earlier
+ * naming of the same stack that the reconciler had stopped recognising and
+ * therefore stopped managing. Nothing in the failure said that, and nothing
+ * else would: the reconciler converges the containers it can name, and a
+ * container it no longer names is invisible to it while remaining very
+ * visible to the kernel.
+ *
+ * Answers null when nothing holds it, which is its own information: the
+ * bind failed for another reason.
+ */
+export async function containerHoldingPort(hostPort) {
+    const { execFile } = await import('node:child_process');
+    const { promisify } = await import('node:util');
+    try {
+        const { stdout } = await promisify(execFile)('docker', ['ps', '-a', '--format', '{{.Names}}\t{{.Ports}}'], { encoding: 'utf-8', timeout: 20_000 });
+        for (const line of String(stdout).split('\n')) {
+            const [name, ports] = line.split('\t');
+            if (!name || !ports)
+                continue;
+            // `127.0.0.1:5432->5432/tcp` and `0.0.0.0:5480->5432/tcp`: the host
+            // port is the one before the arrow, and it is the one that collides.
+            if (new RegExp(`:${hostPort}->`).test(ports))
+                return name;
+        }
+        return null;
     }
     catch {
         return null;
     }
 }
+/**
+ * Explain a bind failure by naming what is holding the port.
+ *
+ * Kept separate from the retry because it answers a different question:
+ * the retry asks "will this resolve on its own", and this asks "if not,
+ * what do I tell the operator".
+ */
+export async function describeBindFailure(config, original, 
+/** The lookup, as a parameter so this is answerable without a docker. */
+lookup = containerHoldingPort) {
+    for (const port of config.ports) {
+        const holder = await lookup(port.host);
+        if (!holder || holder === config.name)
+            continue;
+        return (`${config.name} cannot take ${port.bindHost ?? '0.0.0.0'}:${port.host} — the container ` +
+            `\`${holder}\` is holding it. If that is a leftover from an earlier naming of this stack, ` +
+            `\`docker rm -f ${holder}\` releases the port and keeps its volume.`);
+    }
+    return original;
+}
 /**
  * Create and start a container from a resolved config.
  */
@@ -95,9 +428,19 @@ export async function createContainer(config) {
     const { execFile } = await import('node:child_process');
     const { promisify } = await import('node:util');
     const execFileAsync = promisify(execFile);
-    // Ensure volumes exist
+    // Ensure the managed network exists before we try to attach to it.
+    // Idempotent — `docker network create` errors with "already exists"
+    // when present, which we treat as success. Doing this here (rather
+    // than at daemon start) keeps the provisioner self-sufficient and
+    // means single-container test setups work without a manual prep step.
+    if (config.network) {
+        await ensureNetwork(config.network);
+    }
+    // Ensure volumes exist. A `configroot:` source is a shipped directory this
+    // node has no copy of (`shipped-config.ts`), not a volume to create: it
+    // reaches `docker run`, which refuses it by name.
     for (const vol of config.volumes) {
-        if (!vol.source.startsWith('/') && !vol.source.startsWith('.')) {
+        if (!vol.source.startsWith('/') && !vol.source.startsWith('.') && !vol.source.startsWith(CONFIG_ROOT_PREFIX)) {
             await createVolume(vol.source);
         }
     }
@@ -105,9 +448,16 @@ export async function createContainer(config) {
     await ensureImage(config.image);
     // Build docker run command directly — xec adapter can hang
     const args = ['docker', 'run', '-d', '--name', config.name];
-    // Port mappings
+    // Network — explicit network avoids the default bridge, which
+    // accumulates phantom endpoints after dockerd restarts on macOS and
+    // blocks container recreation with "endpoint with name X already
+    // exists in network bridge". Falls back to docker's default when
+    // unspecified so single-container / no-stack usage still works.
+    if (config.network) {
+        args.push('--network', config.network);
+    }
     for (const p of config.ports) {
-        args.push('-p', `${p.host}:${p.container}`);
+        args.push('-p', portArg(p));
     }
     // Environment variables
     for (const [key, value] of Object.entries(config.environment)) {
@@ -117,8 +467,11 @@ export async function createContainer(config) {
     for (const v of config.volumes) {
         args.push('-v', `${v.source}:${v.target}${v.readonly ? ':ro' : ''}`);
     }
-    // Labels
-    for (const [key, value] of Object.entries(config.labels ?? {})) {
+    // Labels — stamp the desired-spec fingerprint so a later reconcile can
+    // detect config drift (env/ports/volumes/cmd change under the same image)
+    // and recreate instead of silently no-op'ing.
+    const labels = { ...(config.labels ?? {}), [SPEC_HASH_LABEL]: containerSpecHash(config) };
+    for (const [key, value] of Object.entries(labels)) {
         args.push('--label', `${key}=${value}`);
     }
     // Restart policy
@@ -163,12 +516,131 @@ export async function createContainer(config) {
     if (config.command) {
         args.push(...config.command);
     }
-    // Execute docker run — use async execFile to avoid blocking event loop
-    const { stdout } = await execFileAsync('docker', args.slice(1), { encoding: 'utf-8', timeout: 60_000 });
+    // Execute docker run — use async execFile to avoid blocking event loop.
+    //
+    // Self-heal the two stale-state failures that otherwise wedge a managed
+    // container into an unstartable state (both seen in production on
+    // macOS/OrbStack after dockerd restarts under disk pressure):
+    //   1. "endpoint with name X already exists in network N" — an orphaned
+    //      libnetwork endpoint outlived its container; docker run then leaves a
+    //      half-created container with PortBindings set but NO published host
+    //      port, so clients get ECONNREFUSED while inspect says "running".
+    //   2. "Conflict. The container name X is already in use" — a leftover
+    //      Created-state container from a previously failed run.
+    //   3. "Bind for 127.0.0.1:5432 failed: port is already allocated" — the
+    //      docker-proxy of a container `docker rm` has already returned for
+    //      has not released its bind yet. A reconciler that removes and
+    //      immediately recreates races that release. Measured on a node
+    //      recreating three services at once: all three failed this way and
+    //      were left in `Created`, and `docker start` by hand a second later
+    //      worked — which is what makes it a race rather than a conflict.
+    //
+    // The first two are scrubbed; the third only needs a moment. Both are
+    // retried, with a short wait, up to three times.
+    //
+    // Deliberately narrow. A missing image, a bad flag, or a port genuinely
+    // held by something else is permanent, and retrying it turns one clear
+    // error into several identical ones separated by delays.
+    const runDocker = () => execFileAsync('docker', args.slice(1), { encoding: 'utf-8', timeout: 60_000 });
+    let stdout;
+    let lastError;
+    for (let attempt = 1; attempt <= 3; attempt += 1) {
+        try {
+            ({ stdout } = await runDocker());
+            lastError = undefined;
+            break;
+        }
+        catch (err) {
+            lastError = err;
+            const msg = err instanceof Error ? err.message : String(err);
+            if (isStaleContainerState(msg)) {
+                await scrubStaleContainerState(config.name, config.network);
+                continue;
+            }
+            if (isPortNotYetReleased(msg)) {
+                // The failed run leaves the container in `Created` holding the name,
+                // so the next attempt would hit the name conflict instead of the
+                // port. Remove it, then wait for the bind to go.
+                await removeContainer(config.name).catch(() => undefined);
+                await new Promise((resolve) => setTimeout(resolve, attempt * 1_000));
+                continue;
+            }
+            throw err;
+        }
+    }
+    if (lastError || stdout === undefined) {
+        const message = lastError instanceof Error ? lastError.message : String(lastError ?? '');
+        if (isPortNotYetReleased(message)) {
+            // Waiting did not help, so something is holding the port rather than
+            // letting go of it. Name it: the docker error names the port and
+            // withholds the one thing an operator can act on.
+            throw new Error(await describeBindFailure(config, message));
+        }
+        throw lastError ?? new Error(`docker run produced no container id for ${config.name}`);
+    }
+    // Post-create guard: a container can report "running" yet have its declared
+    // host ports UNpublished when it silently reused an orphaned endpoint
+    // (PortBindings present but never wired by dockerd) — the exact failure that
+    // took Redis offline and cascaded to a login outage. Detect the mismatch and
+    // rebuild once from clean state so a successful return always means the
+    // declared ports are actually reachable.
+    if (config.ports.length > 0 && !(await arePortsPublished(config.name, config.ports))) {
+        await scrubStaleContainerState(config.name, config.network);
+        ({ stdout } = await runDocker());
+    }
     const output = (stdout ?? '').trim();
     // Return container ID from docker run output (first line is container ID)
     return (output || config.name).slice(0, 12);
 }
+/**
+ * Clear stale Docker state that blocks (re)creating a managed container: a
+ * leftover container holding the name, and an orphaned libnetwork endpoint
+ * holding the name on the managed network. Best-effort — every step may
+ * legitimately find nothing to remove.
+ */
+async function scrubStaleContainerState(name, network) {
+    const { execFile } = await import('node:child_process');
+    const { promisify } = await import('node:util');
+    const execFileAsync = promisify(execFile);
+    try {
+        await execFileAsync('docker', ['rm', '-f', name], { encoding: 'utf-8', timeout: 15_000 });
+    }
+    catch {
+        // No leftover container.
+    }
+    if (network) {
+        try {
+            await execFileAsync('docker', ['network', 'disconnect', '-f', network, name], {
+                encoding: 'utf-8',
+                timeout: 15_000,
+            });
+        }
+        catch {
+            // No orphaned endpoint for this name.
+        }
+    }
+}
+/**
+ * True if every declared host port shows up in `docker port <name>` (i.e.
+ * dockerd actually published it). Returns true when the check itself fails,
+ * so a transient docker hiccup never triggers a needless rebuild of a
+ * container that is in fact fine.
+ */
+async function arePortsPublished(name, ports) {
+    const { execFile } = await import('node:child_process');
+    const { promisify } = await import('node:util');
+    const execFileAsync = promisify(execFile);
+    try {
+        // `docker port` prints one line per published port, e.g.
+        // "6379/tcp -> 0.0.0.0:6379"; an unpublished container prints nothing.
+        const { stdout } = await execFileAsync('docker', ['port', name], { encoding: 'utf-8', timeout: 10_000 });
+        const mapped = stdout ?? '';
+        return ports.every((p) => mapped.includes(`:${p.host}`));
+    }
+    catch {
+        return true;
+    }
+}
 /**
  * Start an existing stopped container.
  */
@@ -241,6 +713,32 @@ export async function waitForHealthy(name, timeoutMs = 60_000) {
         return false;
     }
 }
+/**
+ * Create a Docker bridge network if it doesn't exist.
+ *
+ * Idempotent — `docker network create` exits non-zero with a recognisable
+ * "already exists" message when the network is present; we swallow that
+ * specific case and propagate everything else. Done via execFile rather
+ * than the adapter because the adapter doesn't expose network APIs and
+ * the call is rare (once per daemon startup per network).
+ */
+export async function ensureNetwork(name) {
+    const { execFile } = await import('node:child_process');
+    const { promisify } = await import('node:util');
+    const execFileAsync = promisify(execFile);
+    try {
+        await execFileAsync('docker', ['network', 'create', name, '--driver', 'bridge'], {
+            encoding: 'utf-8',
+            timeout: 15_000,
+        });
+    }
+    catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        if (/already exists/i.test(msg))
+            return; // idempotent
+        throw err;
+    }
+}
 /**
  * Create a Docker volume if it doesn't exist.
  */
@@ -253,6 +751,38 @@ export async function createVolume(name) {
         // Volume may already exist
     }
 }
+/**
+ * Does a Docker volume exist?
+ *
+ * The question behind this one is "does this service already hold state
+ * initialised with some other password" — a Postgres data directory keeps
+ * the password it was created with and ignores `POSTGRES_PASSWORD`
+ * thereafter, so a volume is the difference between a first provision and
+ * one that would lock an application out of its own database.
+ *
+ * `docker volume inspect` rather than listing and filtering: the answer is
+ * about one name, and a list is a different question that happens to
+ * contain it.
+ */
+export async function volumeExists(name) {
+    const { execFile } = await import('node:child_process');
+    const { promisify } = await import('node:util');
+    try {
+        await promisify(execFile)('docker', ['volume', 'inspect', name], { timeout: 15_000 });
+        return true;
+    }
+    catch (err) {
+        // "No such volume" is an answer. Anything else — docker unreachable, a
+        // timeout, a permission error — is NOT, and the two must not be
+        // collapsed: the caller generates a credential when this says false, and
+        // generating one for a volume that does exist locks an application out
+        // of its own database. So only the message that means absence answers
+        // absence; everything else answers "assume there is state", which
+        // leaves a service on a default loudly rather than breaking it quietly.
+        const message = err instanceof Error ? err.message : String(err);
+        return !/no such volume/i.test(message);
+    }
+}
 /**
  * Get container logs.
  */
@@ -272,6 +802,35 @@ export async function disposeEngine() {
 // =============================================================================
 // Helpers
 // =============================================================================
+/**
+ * Why a container is not running, in the words Docker already has.
+ *
+ * `ContainerState.error` has been part of the type all along and nothing ever
+ * filled it, while `docker inspect` carries both the exit code and a
+ * human-readable reason. So `omnitron doctor` could say "Container
+ * omnitron-nginx is created" and no more, when the answer was sitting one
+ * field away: "Bind for 0.0.0.0:9800 failed: port is already allocated".
+ *
+ * A container that is running, or that exited cleanly because it was asked
+ * to, has nothing to report — returning something there would turn a normal
+ * stop into a finding.
+ */
+function describeContainerFailure(state) {
+    if (!state)
+        return undefined;
+    if (state.Status?.toLowerCase() === 'running')
+        return undefined;
+    const parts = [];
+    if (state.Error)
+        parts.push(state.Error);
+    if (state.OOMKilled)
+        parts.push('killed by the OOM killer');
+    // Exit 0 is a clean stop; only a non-zero code is worth carrying.
+    if (typeof state.ExitCode === 'number' && state.ExitCode !== 0) {
+        parts.push(`exit code ${state.ExitCode}`);
+    }
+    return parts.length > 0 ? parts.join(' — ') : undefined;
+}
 function mapInspectStatus(status) {
     if (!status)
         return 'not_found';
@@ -301,4 +860,12 @@ function mapInspectHealth(health) {
         return 'starting';
     return 'none';
 }
+/**
+ * Exported for tests only.
+ *
+ * `describeContainerFailure` is a pure function over an inspect payload, and
+ * the payloads worth testing are the ones a live Docker will not produce on
+ * demand — a refused port bind, an OOM kill.
+ */
+export const __test__ = { describeContainerFailure, portArg };
 //# sourceMappingURL=container-runtime.js.map