@omnitron-dev/omnitron 0.1.3 → 0.3.0

package/dist/infrastructure/infrastructure.service.js

@@ -15,39 +15,178 @@
  *   ResolvedContainer[] → ContainerRuntime → Docker CLI
  *   Health Loop → detect failure → auto-restart (exponential backoff)
  */
-import { resolveInfrastructure, resolveOmnitronPg } from './service-resolver.js';
-import { isDockerAvailable, getContainerState, createContainer, startContainer, removeContainer, ensureImage, waitForHealthy, createVolume, } from './container-runtime.js';
+import { computeBackoff } from '@omnitron-dev/titan/utils';
+import { resolveInfrastructure, resolveOmnitronPg, getManagedNetwork } from './service-resolver.js';
+import { PhantomEndpointJanitor } from './phantom-endpoint-janitor.js';
+import { LOCAL_INFRA_HOST } from '../shared/local-infra-host.js';
+import { isDockerAvailable, getContainerState, createContainer, startContainer, removeContainer, ensureImage, waitForHealthy, createVolume, containerSpecHash, } from './container-runtime.js';
+import { summariseProvisioning, describeProvisioning } from './provisioning-outcome.js';
+import { OMNITRON_PG_PORT, OMNITRON_PG_USER, OMNITRON_PG_PASSWORD, OMNITRON_PG_DATABASE, } from '../database/connection.js';
 const RECONCILE_INTERVAL = 30_000; // 30s health sweep
 const STARTUP_TIMEOUT = 120_000; // 2min max per service
-/** Omnitron internal PG connection defaults */
-const OMNITRON_PG_PORT = 5480;
-const OMNITRON_PG_USER = 'omnitron';
-const OMNITRON_PG_PASSWORD = 'omnitron';
-const OMNITRON_PG_DATABASE = 'omnitron';
+/** Consecutive 'unhealthy' ticks before recreate. 2 ticks @ 30s = 60s grace. */
+const UNHEALTHY_RESTART_THRESHOLD = 2;
+/** Restart circuit-breaker: first failed-restart backoff, doubled each failure. */
+const RESTART_BACKOFF_BASE = 30_000; // 30s
+/** Restart circuit-breaker: backoff ceiling (30s → 1m → 2m … capped here). */
+const RESTART_BACKOFF_MAX = 15 * 60_000; // 15min
+/**
+ * Restart-circuit-breaker backoff schedule: the delay (ms) to wait before the
+ * next restart attempt after `failures` consecutive failed (re)creates.
+ * Exponential from RESTART_BACKOFF_BASE, capped at RESTART_BACKOFF_MAX so a
+ * permanently-broken service settles into an occasional retry instead of
+ * hammering the supervisor every 30s.
+ *
+ * Delegates to the shared titan `computeBackoff` (the same helper the downstream
+ * backends and the titan packages use) so the exponential shape lives in ONE
+ * place. Exported for tests.
+ */
+export function restartBackoffMs(failures) {
+    if (failures < 1)
+        return 0;
+    return computeBackoff({ attempt: failures - 1, baseMs: RESTART_BACKOFF_BASE, maxMs: RESTART_BACKOFF_MAX });
+}
+// Omnitron internal PG connection defaults live in `database/connection.ts`
+// so the container spec below and every client that connects to it are
+// derived from the same values (see the import above).
+/**
+ * What to do about the control plane's own database.
+ *
+ * Three cases that look alike and are not:
+ *
+ *   - a node's daemon keeps its state in SQLite and needs no database at all;
+ *   - one is running, so it gets the same reconcile — and so the same
+ *     `containerSpecHash` drift check — as every app service;
+ *   - nothing is running, so provision.
+ *
+ * The middle case is the one that was missing. "Already running" was read as
+ * "configured as declared", and the database was adopted exactly as it stood.
+ * Measured: `omnitron-pg`, created before published ports were bound to
+ * loopback, survived on 0.0.0.0:5480 through the fix that reached all eleven
+ * other managed containers, and still answered `nc -vz <lan-address> 5480`
+ * from another machine months later, while `daos-dev-postgres` beside it in
+ * `docker ps` was correctly on 127.0.0.1.
+ *
+ * `reconcileName` is the subtlety. The control-plane database is ONE
+ * container per host — it owns port 5480 — but each daemon resolves a name
+ * carrying its own project prefix, so a daemon on the `daos/dev` stack
+ * computes `daos-dev-pg` while the container that exists is `omnitron-pg`.
+ * Reconciling the running container against a spec under the OTHER name
+ * creates a second one and collides on the port; the first version of this
+ * did exactly that and corrected nothing. The name to reconcile is therefore
+ * the name of the container that is actually there, and the spec is
+ * everything else. `containerSpecHash` does not cover the name, so the drift
+ * comparison is unaffected either way.
+ *
+ * A differing name is not evidence of another owner: only omnitron creates
+ * this container, and it carries omnitron's own `omnitron.internal` label.
+ */
+export function decideControlPlaneDatabase(input) {
+    if (!input.needsControlPlaneDatabase)
+        return { action: 'skip' };
+    if (input.running?.status !== 'running')
+        return { action: 'provision' };
+    return { action: 'reconcile', reconcileName: input.running.name ?? input.desiredName };
+}
 export class InfrastructureService {
     logger;
     config;
     presetRegistry;
+    serviceOverrides;
+    needsControlPlaneDatabase;
     desiredContainers = [];
+    /** What the most recent reconcile did to each service. */
+    lastActions = new Map();
     omnitronPgContainer;
     usingGlobalOmnitronPg = false;
     healthTimer = null;
     state = { services: {}, ready: false };
     normalizedServices;
-    constructor(logger, config, normalizedServices, presetRegistry) {
+    /**
+     * Per-service consecutive-unhealthy counter. Restart only fires after
+     * UNHEALTHY_RESTART_THRESHOLD ticks in a row to avoid flapping during
+     * cold-start / transient health-check timeouts. Reset to 0 on a healthy
+     * observation. Without this, a service that briefly fails one health
+     * probe (network blip, slow query) would be torn down and recreated,
+     * making transient pressure look like outages.
+     */
+    unhealthyTicks = new Map();
+    /**
+     * Per-service restart circuit-breaker. A container whose (re)create keeps
+     * failing — an image that won't pull, or infra stuck on an unrecoverable
+     * error — must NOT be retried every 30s forever: that churn (docker calls +
+     * error logging) is what wedged the daemon's event loop and let an external
+     * "socket unreachable → SIGTERM" kill the whole daemon during the 2026-05
+     * outage. After each failed restart we back off exponentially (30s → 1m →
+     * 2m … capped at 15m); a successful restart or an externally-recovered
+     * 'running' observation resets it.
+     */
+    restartBackoff = new Map();
+    /**
+     * T#75: phantom-endpoint janitor instance. Created in `provision()`
+     * once we know Docker is reachable; torn down in `teardown()`.
+     */
+    phantomJanitor = null;
+    constructor(logger, config, normalizedServices, presetRegistry, 
+    /**
+     * Stack overrides. A service marked `disabled` here is not provisioned,
+     * and one pointed at an `external` address is not provisioned either —
+     * there is nothing to create, and the address reaches the application
+     * through its environment instead.
+     */
+    serviceOverrides, 
+    /**
+     * Whether this daemon stores its own state in Postgres.
+     *
+     * True for a master, which is where projects, nodes, metrics and logs
+     * live. False for a node: its daemon keeps that in SQLite and reads a
+     * Postgres never.
+     */
+    needsControlPlaneDatabase = true) {
         this.logger = logger;
         this.config = config;
         this.presetRegistry = presetRegistry;
+        this.serviceOverrides = serviceOverrides;
+        this.needsControlPlaneDatabase = needsControlPlaneDatabase;
         this.normalizedServices = normalizedServices ?? {};
-        this.desiredContainers = resolveInfrastructure(config, normalizedServices);
+        this.desiredContainers = resolveInfrastructure(config, normalizedServices, serviceOverrides);
         this.omnitronPgContainer = resolveOmnitronPg();
     }
     /**
      * Add app-declared infrastructure containers to the provisioning list.
      * Called by StackInfrastructureManager after resolving app requirements.
+     *
+     * T#76: dedupe by container name. The `${CONTAINER_PREFIX}-${service}`
+     * naming convention means two apps that BOTH declare a `postgres`
+     * requirement produce two identical-named entries. Pre-T#76, the
+     * provisioning loop tried to create the container twice; Docker's
+     * name-uniqueness check failed the second create with "name already
+     * in use", leaving the second app's perspective on infrastructure
+     * broken (it saw a failure that, from the cluster's view, was a
+     * benign collision — the shared service was actually fine).
+     *
+     * The intended semantics for shared infrastructure is "all apps that
+     * declare `postgres` use the same container", so the first
+     * declaration wins. When subsequent declarations differ in shape
+     * (image / env / ports), log a structured warning so the operator
+     * can reconcile intent vs. config.
      */
     addAppContainers(containers) {
-        this.desiredContainers.push(...containers);
+        for (const c of containers) {
+            const existing = this.desiredContainers.find((d) => d.name === c.name);
+            if (!existing) {
+                this.desiredContainers.push(c);
+                continue;
+            }
+            if (!shallowContainerEqual(existing, c)) {
+                this.logger.warn({
+                    container: c.name,
+                    existingImage: existing.image,
+                    duplicateImage: c.image,
+                }, 'Duplicate container declaration with differing shape — keeping first; check for conflicting app infrastructure requirements (T#76)');
+            }
+            // else: identical declaration, silent dedupe — this is the common case
+        }
     }
     /**
      * Provision all infrastructure services.
@@ -65,26 +204,52 @@ export class InfrastructureService {
         //    The stack-prefixed container (e.g. omni-dev-pg) shares the same port,
         //    so we must not create/start it when the global one is active.
         const globalOmnitronPg = await getContainerState('omnitron-pg');
-        const stackPgName = this.omnitronPgContainer.name;
-        const isGlobalPgRunning = globalOmnitronPg?.status === 'running';
-        const isStackPgSameAsGlobal = stackPgName === 'omnitron-pg';
-        if (isGlobalPgRunning && !isStackPgSameAsGlobal) {
-            // Global omnitron-pg already owns port 5480 — reuse it and skip stack-prefixed container.
+        const decision = decideControlPlaneDatabase({
+            needsControlPlaneDatabase: this.needsControlPlaneDatabase,
+            running: globalOmnitronPg,
+            desiredName: this.omnitronPgContainer.name,
+        });
+        if (decision.action === 'skip') {
+            // A node's daemon keeps its own state in SQLite — `SlaveStorageService`,
+            // `~/.omnitron/data/slave.db` — and reads this database never. Creating
+            // it anyway gave every provisioned node a Postgres nobody queries, on
+            // default credentials, and until published ports were bound to
+            // loopback it was reachable from the internet: measured answering on
+            // 0.0.0.0:5480 on a host whose firewall allows only SSH.
+            //
+            // The control plane's database belongs to the control plane.
+            this.logger.debug('This daemon keeps its own state in SQLite — not provisioning a control-plane database');
+        }
+        else if (decision.action === 'reconcile') {
             this.usingGlobalOmnitronPg = true;
-            this.logger.info({ global: 'omnitron-pg', stackPg: stackPgName }, 'Using existing omnitron-pg (already running) — skipping stack-prefixed PG');
-            // Remove stale stack-prefixed container if it exists (leftover from a previous attempt)
-            const stalePg = await getContainerState(stackPgName);
-            if (stalePg) {
-                try {
-                    await removeContainer(stackPgName);
-                    this.logger.info({ service: stackPgName }, 'Removed stale stack-prefixed PG container');
+            // A stack-prefixed leftover cannot run beside the global one — they
+            // share port 5480 — so it goes before anything else touches that port.
+            if (this.omnitronPgContainer.name !== decision.reconcileName) {
+                const stalePg = await getContainerState(this.omnitronPgContainer.name);
+                if (stalePg) {
+                    try {
+                        await removeContainer(this.omnitronPgContainer.name);
+                        this.logger.info({ service: this.omnitronPgContainer.name }, 'Removed stale stack-prefixed PG container');
+                    }
+                    catch { /* already gone */ }
                 }
-                catch { /* already gone */ }
             }
-        }
-        else if (isGlobalPgRunning) {
-            this.usingGlobalOmnitronPg = true;
-            this.logger.info('Using existing omnitron-pg (already running)');
+            // The same reconcile, and so the same drift check, as every app
+            // service — under the name the container actually has. Its data lives
+            // in the `omnitron-pg-data` volume and outlives the container it is
+            // recreated into.
+            // Name AND volume, not one of them. `resolveOmnitronPg` derives both
+            // from the container prefix, so overriding only the name produced a
+            // container called `omnitron-pg` mounting `daos-dev-pg-data` — the
+            // global identity on a stack-scoped volume, and the stack's own
+            // `omnitron-pg-data` left dangling where a `docker volume prune` would
+            // take it silently. The two names are one decision and must move
+            // together.
+            const reconciled = { ...this.omnitronPgContainer, name: decision.reconcileName };
+            if (decision.reconcileName !== this.omnitronPgContainer.name) {
+                reconciled.volumes = this.omnitronPgContainer.volumes.map((v) => v.target === '/var/lib/postgresql/data' ? { ...v, source: `${decision.reconcileName}-data` } : v);
+            }
+            await this.reconcileService(reconciled);
         }
         else {
             try {
@@ -101,8 +266,25 @@ export class InfrastructureService {
             }
         }
         this.logger.info({ services: this.desiredContainers.map((c) => c.name) }, 'Provisioning app infrastructure');
-        // 3. Reconcile each app service
-        for (const desired of this.desiredContainers) {
+        // 3. Reconcile every app service in parallel.
+        //
+        // Pre-fix this was a serial `for await` loop over up to a dozen
+        // containers. For an already-healthy stack (the common cold-boot
+        // case where Docker preserved containers across a daemon
+        // restart) each reconcileService is a single `docker inspect`
+        // round-trip — but doing nine of them serially still costs
+        // multiple seconds of latency on the critical path before
+        // apps can even start spawning.
+        //
+        // Safety: `reconcileService` holds a per-name lock
+        // (`this.reconciling`), so parallel calls for the SAME
+        // container still serialise. Cross-container ordering (e.g.
+        // monero-wallet-rpc must talk to monero-daemon) is enforced
+        // by each container's own healthcheck loop — the wallet's
+        // health probe retries until the daemon's RPC accepts
+        // connections, so launching them in parallel just extends the
+        // wallet's first-try wait, not its end state.
+        await Promise.all(this.desiredContainers.map(async (desired) => {
             try {
                 await this.reconcileService(desired);
             }
@@ -115,26 +297,81 @@ export class InfrastructureService {
                     error: err.message,
                 };
             }
-        }
+        }));
         // 4. Post-provisioning setup (create databases, buckets, etc.)
         await this.postProvision();
         // 4. Start health monitor
         this.startHealthMonitor();
-        this.state.ready = true;
+        // T#75: start the phantom-endpoint janitor. It periodically
+        // cleans up stale endpoint records on the managed network so
+        // a Docker daemon restart / host suspend doesn't leave us
+        // unable to recreate containers ("endpoint with name X
+        // already exists in network ...").
+        this.phantomJanitor = new PhantomEndpointJanitor({
+            networks: [getManagedNetwork()],
+            intervalMs: 60_000,
+            logger: this.logger,
+        });
+        this.phantomJanitor.start();
+        // Readiness is measured, not announced.
+        //
+        // These two lines were `ready = true` and "Infrastructure provisioned
+        // and healthy", unconditionally — and every service reconciles inside a
+        // `Promise.all` whose per-service catch records `exited` and continues.
+        // So a run where every container failed ended exactly like a run where
+        // all of them came up, and the gate below this waits on a flag that
+        // could not say no. See `summariseProvisioning`.
+        const outcome = summariseProvisioning(this.desiredContainers, this.state.services);
+        this.state.ready = outcome.ready;
         this.state.lastReconciled = new Date().toISOString();
-        this.logger.info('Infrastructure provisioned and healthy');
+        if (outcome.ready) {
+            this.logger.info(describeProvisioning(outcome));
+        }
+        else {
+            this.logger.error({
+                missing: outcome.missing.map((s) => s.name),
+                failed: outcome.failed.map((s) => ({ name: s.name, status: s.status, error: s.error })),
+                running: outcome.running.length,
+            }, describeProvisioning(outcome));
+        }
         return this.state;
     }
+    /**
+     * What this service was asked to provide.
+     *
+     * Exposed because a caller cannot tell "present and fine" from "never
+     * attempted" by reading `state.services` alone: the second leaves no entry,
+     * and an absent entry is indistinguishable from an absent problem.
+     */
+    getDesiredServices() {
+        return this.desiredContainers.map((c) => ({ name: c.name, image: c.image }));
+    }
+    /**
+     * Services whose socket is new since this reconcile began.
+     *
+     * `noop` is the only action that leaves an existing connection usable.
+     * Everything else means anything already connected is holding a dead pool.
+     */
+    getRestartedServices() {
+        return [...this.lastActions.entries()]
+            .filter(([, action]) => action === 'create' || action === 'start' || action === 'recreate')
+            .map(([name]) => name);
+    }
     /**
      * Stop all managed infrastructure containers.
      */
     async teardown() {
         this.stopHealthMonitor();
-        // Remove all containers in parallel for fast shutdown.
-        // Skip the stack-prefixed PG when using the global omnitron-pg (it's shared).
-        const allContainers = this.usingGlobalOmnitronPg
-            ? [...this.desiredContainers].reverse()
-            : [...[...this.desiredContainers].reverse(), this.omnitronPgContainer];
+        // T#75: stop the janitor BEFORE removing containers so a final
+        // tick can't race the teardown's `docker rm` and produce noisy
+        // disconnect-of-already-disconnected errors.
+        if (this.phantomJanitor) {
+            this.phantomJanitor.stop();
+            this.phantomJanitor = null;
+        }
+        // Remove all containers in parallel for fast shutdown — the control-plane
+        // database only where this service keeps one (`controlPlaneContainers`).
+        const allContainers = [...[...this.desiredContainers].reverse(), ...this.controlPlaneContainers()];
         await Promise.allSettled(allContainers.map(async (c) => {
             try {
                 await removeContainer(c.name);
@@ -152,6 +389,25 @@ export class InfrastructureService {
     getState() {
         return this.state;
     }
+    /**
+     * Serve the stack's definition as it is now.
+     *
+     * A node's daemon builds this service on its first `provisionStack` and
+     * reuses it for every later one — rightly: a second service would be a
+     * second janitor and a second health monitor over one set of containers.
+     * But the definition it was built with stayed its definition. A later
+     * deployment that enabled a service, changed an image or dropped a
+     * `disabled` reached the node's host services — `reconcileHostServices`
+     * reads each call's own — and not its containers, until something restarted
+     * the daemon. Replaced whole, as the constructor sets it; `setConfigRoots`
+     * and `addAppContainers` then work on this set, as on a first call.
+     */
+    redefine(config, normalizedServices, serviceOverrides) {
+        this.config = config;
+        this.normalizedServices = normalizedServices;
+        this.serviceOverrides = serviceOverrides;
+        this.desiredContainers = resolveInfrastructure(config, normalizedServices, serviceOverrides);
+    }
     /**
      * Get normalized services (from preset system).
      * Used by config-resolver for generic address resolution.
@@ -159,6 +415,28 @@ export class InfrastructureService {
     getNormalizedServices() {
         return this.normalizedServices;
     }
+    /**
+     * Databases declared for this stack's postgres (e.g. main, storage, geo).
+     * Source of truth for "which DBs must be backed up" — read straight from
+     * the same `infrastructure.postgres.databases` block omnitron provisions
+     * from, so the backup set can never drift from the provisioned set.
+     */
+    getPostgresDatabases() {
+        const pg = this.config.postgres;
+        if (!pg || !pg.databases)
+            return [];
+        return Object.keys(pg.databases);
+    }
+    /**
+     * Resolved container name for an infra service (e.g. 'postgres' →
+     * 'acme-dev-postgres'). Taken from the already-resolved desired containers
+     * so it reflects the real container prefix rather than re-deriving it from
+     * the mutable module-global. Returns null if the stack has no such service.
+     */
+    getResolvedContainerName(service) {
+        const match = this.desiredContainers.find((d) => d.name.endsWith(`-${service}`));
+        return match?.name ?? null;
+    }
     /**
      * Get connection info for an infrastructure service.
      * Used by config generator to resolve { infra: 'postgres', database: 'main' }.
@@ -170,7 +448,7 @@ export class InfrastructureService {
                 if (!pg)
                     return null;
                 return {
-                    host: 'localhost',
+                    host: LOCAL_INFRA_HOST,
                     port: pg.port ?? 5432,
                     user: pg.user ?? 'postgres',
                     password: typeof pg.password === 'string' ? pg.password : 'postgres',
@@ -181,7 +459,7 @@ export class InfrastructureService {
                 if (!redis)
                     return null;
                 return {
-                    host: 'localhost',
+                    host: LOCAL_INFRA_HOST,
                     port: redis.port ?? 6379,
                     password: typeof redis.password === 'string' ? redis.password : undefined,
                     databases: redis.databases ?? {},
@@ -192,7 +470,7 @@ export class InfrastructureService {
                 if (!minio)
                     return null;
                 return {
-                    endpoint: `http://localhost:${minio.ports?.api ?? 9000}`,
+                    endpoint: `http://${LOCAL_INFRA_HOST}:${minio.ports?.api ?? 9000}`,
                     accessKey: minio.accessKey ?? 'minioadmin',
                     secretKey: typeof minio.secretKey === 'string' ? minio.secretKey : 'minioadmin',
                     forcePathStyle: true,
@@ -200,7 +478,7 @@ export class InfrastructureService {
             }
             case 'omnitron-pg':
                 return {
-                    host: 'localhost',
+                    host: LOCAL_INFRA_HOST,
                     port: OMNITRON_PG_PORT,
                     user: OMNITRON_PG_USER,
                     password: OMNITRON_PG_PASSWORD,
@@ -213,9 +491,73 @@ export class InfrastructureService {
     // ===========================================================================
     // Private — Reconciliation
     // ===========================================================================
+    /**
+     * Per-name in-flight reconcile lock (T#58). A health sweep that
+     * overlaps a previous slow sweep — or a reconcile triggered by
+     * `ensureXxx` calls happening concurrently — used to enter
+     * `reconcileService` twice for the same container. The first did
+     * `removeContainer` then `createAndStart`; the second saw the
+     * container missing mid-recreate and tried to recreate it too,
+     * producing name conflicts, partial state, and (with Docker's
+     * own restart-policy in the mix) occasional duplicate containers.
+     */
+    reconciling = new Set();
+    /**
+     * Where this node keeps each service's config files, once a master sent them.
+     *
+     * The gateway is not an ordinary preset container — it needs four bind
+     * mounts, an entrypoint and fifteen upstream variables, and `resolveGateway`
+     * is the only thing that produces them. The constructor resolved everything
+     * through the generic preset path, because at construction time this node
+     * had no files and no local path to mount from.
+     *
+     * So this RE-RESOLVES rather than patching the volumes of a container built
+     * without them. A spec assembled by the wrong resolver is missing more than
+     * paths, and rewriting one field of it would produce a container that looks
+     * configured and is not — which is the failure that was already shipping,
+     * one level down.
+     *
+     * Called before `provision()`, so the recreation the changed spec hash
+     * implies happens on the first pass rather than the second.
+     */
+    setConfigRoots(roots, staticRoots, redis, 
+    /**
+     * The static ROOT per service, when the delivery made a
+     * `current-<project>-<stack>` link inside it. Given, the gateway mounts
+     * the root and serves through the link — a path that does not change from
+     * one release to the next, so the container is not recreated and the
+     * second of 000 a deployment cost goes with it. Empty, everything below
+     * resolves exactly as before.
+     */
+    staticLinkRoots) {
+        if (roots.size === 0)
+            return;
+        this.desiredContainers = resolveInfrastructure(this.config, this.normalizedServices, this.serviceOverrides, roots, { redis, staticRoots, ...(staticLinkRoots && { staticLinkRoots }) });
+    }
     async reconcileService(desired) {
+        // T#58: per-name lock around the whole inspect→decide→act loop.
+        if (this.reconciling.has(desired.name)) {
+            this.logger.debug({ service: desired.name }, 'Reconcile already in flight — skipping');
+            return;
+        }
+        this.reconciling.add(desired.name);
+        try {
+            await this.reconcileServiceLocked(desired);
+        }
+        finally {
+            this.reconciling.delete(desired.name);
+        }
+    }
+    async reconcileServiceLocked(desired) {
         const actual = await getContainerState(desired.name);
         const action = this.computeAction(desired, actual);
+        // A container that was created, started or recreated is listening on a
+        // NEW socket. Applications already running hold pools pointing at the
+        // old one, and a titan liveness probe does not notice — the app answers
+        // `/health` 200 while every request it serves fails with "Connection
+        // terminated unexpectedly". Whoever asked for this reconcile is the only
+        // one in a position to restart them, so it has to be told.
+        this.lastActions.set(desired.name, action.type);
         switch (action.type) {
             case 'noop':
                 this.logger.debug({ service: desired.name }, 'Service already running');
@@ -251,6 +593,17 @@ export class InfrastructureService {
             return { type: 'create', service: desired.name, config: desired };
         }
         if (actual.status === 'running') {
+            // Running but detached from all networks (OrbStack/dockerd-restart
+            // artifact): the container is unreachable and its published ports are
+            // gone. 'running' is a lie here — recreate to re-attach + re-publish.
+            if (actual.networkAttached === false) {
+                return {
+                    type: 'recreate',
+                    service: desired.name,
+                    config: desired,
+                    reason: 'network-detached (running but on no network)',
+                };
+            }
             // Check if image changed
             if (actual.image !== desired.image && !actual.image.startsWith(desired.image)) {
                 return {
@@ -260,9 +613,55 @@ export class InfrastructureService {
                     reason: `image changed: ${actual.image} → ${desired.image}`,
                 };
             }
+            // Config drift: image unchanged but the desired spec (env, ports,
+            // volumes, command, extraHosts) differs from what the running container
+            // was created with. Without this, an omnitron.config.ts edit that only
+            // touches env — e.g. the Tor hidden-service target — is silently ignored
+            // because the container is "running and healthy". Only fires when the
+            // running container carries a spec-hash label; pre-upgrade containers
+            // (no label) are left alone until they recreate for another reason, so
+            // upgrading omnitron never churns an otherwise-healthy stack.
+            if (actual.specHash) {
+                const desiredHash = containerSpecHash(desired);
+                if (actual.specHash !== desiredHash) {
+                    return {
+                        type: 'recreate',
+                        service: desired.name,
+                        config: desired,
+                        reason: `config drift (spec ${actual.specHash} → ${desiredHash})`,
+                    };
+                }
+            }
             return { type: 'noop', service: desired.name };
         }
         if (actual.status === 'exited' || actual.status === 'created') {
+            // A container that is not running still carries the spec it was created
+            // with, and `start` runs THAT spec. The drift checks above were only
+            // asked of a running container, so one created wrong was started wrong
+            // for as long as it existed: Nominatim on daos/test, created at 05:37
+            // with a mount the master had not rewritten, was «started» by every
+            // reconcile after the fix that corrected its desired spec — «exec:
+            // /opt/nominatim-tools/cis-entrypoint.sh: no such file» each time
+            // (2026-09-25). Same questions, same answers, before it is started.
+            if (actual.image !== desired.image && !actual.image.startsWith(desired.image)) {
+                return {
+                    type: 'recreate',
+                    service: desired.name,
+                    config: desired,
+                    reason: `image changed while stopped: ${actual.image} → ${desired.image}`,
+                };
+            }
+            if (actual.specHash) {
+                const desiredHash = containerSpecHash(desired);
+                if (actual.specHash !== desiredHash) {
+                    return {
+                        type: 'recreate',
+                        service: desired.name,
+                        config: desired,
+                        reason: `config drift while ${actual.status} (spec ${actual.specHash} → ${desiredHash})`,
+                    };
+                }
+            }
             return { type: 'start', service: desired.name, containerId: actual.containerId ?? desired.name };
         }
         // Dead or other — recreate
@@ -343,6 +742,26 @@ export class InfrastructureService {
         const { execFile } = await import('node:child_process');
         const { promisify } = await import('node:util');
         const execFileAsync = promisify(execFile);
+        // A preset hook is what creates the databases inside Postgres and the
+        // buckets inside MinIO — the containers are running either way, so a
+        // failure here is invisible until an application cannot find its
+        // database.
+        //
+        // `this.presetRegistry!` asserted a dependency the constructor declares
+        // OPTIONAL, so a caller that did not pass one — which the type permits —
+        // got `Cannot read properties of undefined (reading 'get')` from a line
+        // that reads as internal, after every container was already up.
+        // Measured on a node provisioning a stack, where the containers stayed
+        // and the databases were never made.
+        if (!this.presetRegistry) {
+            const withPresets = Object.entries(this.normalizedServices)
+                .filter(([, r]) => r._preset)
+                .map(([name]) => name);
+            if (withPresets.length > 0) {
+                this.logger.error({ services: withPresets }, 'No preset registry, so post-provision steps were skipped — these services are running but their databases and buckets were not created');
+            }
+            return;
+        }
         for (const [serviceName, requirement] of Object.entries(this.normalizedServices)) {
             if (!requirement._preset)
                 continue;
@@ -400,51 +819,20 @@ export class InfrastructureService {
         this.logger.info('Omnitron internal database provisioned and migrated');
     }
     /**
-     * Connect to Omnitron's PG and run migrations via Kysely's built-in Migrator.
-     * Uses dynamic import to keep pg/kysely optional until actually needed.
+     * Apply the internal-database schema through the shared runner.
+     *
+     * This used to be a second, independently-maintained copy of the daemon's
+     * migration code that only knew migrations 001–002 — and, like the
+     * daemon's copy, it constructed `Migrator` from the `kysely` package root
+     * where that export does not exist, so it threw on every run. Both are now
+     * one call into `database/migration-runner.ts`.
+     *
+     * Errors propagate to `provisionOmnitronDatabase`'s caller: provisioning
+     * that ends with an unmigrated database has not succeeded.
      */
     async runOmnitronMigrations() {
-        const { Kysely, PostgresDialect, Migrator } = await import('kysely');
-        const pg = await import('pg');
-        const m001 = await import('../database/migrations/001_initial_schema.js');
-        const m002 = await import('../database/migrations/002_metrics_raw.js');
-        const pool = new pg.default.Pool({
-            host: 'localhost',
-            port: OMNITRON_PG_PORT,
-            database: OMNITRON_PG_DATABASE,
-            user: OMNITRON_PG_USER,
-            password: OMNITRON_PG_PASSWORD,
-        });
-        const db = new Kysely({
-            dialect: new PostgresDialect({ pool }),
-        });
-        try {
-            const migrator = new Migrator({
-                db,
-                provider: {
-                    async getMigrations() {
-                        return {
-                            '001_initial_schema': { up: m001.up, down: m001.down },
-                            '002_metrics_raw': { up: m002.up, down: m002.down },
-                        };
-                    },
-                },
-            });
-            const { results, error } = await migrator.migrateToLatest();
-            const applied = results?.filter((r) => r.status === 'Success') ?? [];
-            if (applied.length > 0) {
-                this.logger.info({ migrations: applied.map((r) => r.migrationName) }, `Applied ${applied.length} Omnitron migration(s)`);
-            }
-            else {
-                this.logger.debug('Omnitron database schema is up to date');
-            }
-            if (error) {
-                this.logger.error({ error: String(error) }, 'Migration error');
-            }
-        }
-        finally {
-            await db.destroy();
-        }
+        const { runOmnitronMigrations } = await import('../database/migration-runner.js');
+        await runOmnitronMigrations(this.logger);
     }
     // ===========================================================================
     // Private — Health Monitor (Self-Healing)
@@ -463,33 +851,84 @@ export class InfrastructureService {
             this.healthTimer = null;
         }
     }
+    /**
+     * The control-plane database container this service keeps, if any: none on
+     * a node — it keeps its own state in SQLite (`decideControlPlaneDatabase`) —
+     * and none where the global `omnitron-pg` serves.
+     *
+     * `provision()` asked `needsControlPlaneDatabase`; the health sweep and the
+     * teardown asked only `usingGlobalOmnitronPg`. So on a node the sweep found
+     * `<prefix>-pg` «not running» every 30 s and created it: daos/test got one on
+     * 2026-09-20, and again on 2026-09-25 forty seconds after it was removed —
+     * on the default credentials, for a database nothing reads.
+     */
+    controlPlaneContainers() {
+        return this.needsControlPlaneDatabase && !this.usingGlobalOmnitronPg ? [this.omnitronPgContainer] : [];
+    }
     async healthSweep() {
-        // Include omnitron-pg in health sweep alongside app containers,
-        // but skip the stack-prefixed PG when using the global omnitron-pg
-        const allContainers = this.usingGlobalOmnitronPg
-            ? [...this.desiredContainers]
-            : [this.omnitronPgContainer, ...this.desiredContainers];
+        // The control-plane database alongside the app containers, where this
+        // service keeps one.
+        const allContainers = [...this.controlPlaneContainers(), ...this.desiredContainers];
         for (const desired of allContainers) {
             const actual = await getContainerState(desired.name);
+            // T#58: Docker's own `--restart=always` / `--restart=on-failure`
+            // policy puts the container into the 'restarting' state for the
+            // brief window between exit and re-entry. If we recreate during
+            // that window we race Docker's restart — `removeContainer` mid-
+            // restart leaves the daemon confused, and the recreated container
+            // may end up duplicated under a renamed transient. Let Docker
+            // finish; the next 30s tick will see 'running' and skip.
+            if (actual?.status === 'restarting') {
+                this.logger.debug({ service: desired.name }, "Service is restarting via Docker's own policy — yielding to it (T#58)");
+                continue;
+            }
             if (!actual || actual.status !== 'running') {
+                this.unhealthyTicks.delete(desired.name);
+                // Circuit-breaker: while a service is inside its backoff window, skip
+                // the restart so a permanently-failing container can't hammer the
+                // supervisor every tick and starve the daemon's event loop.
+                const backoff = this.restartBackoff.get(desired.name);
+                if (backoff && Date.now() < backoff.nextAttemptAt) {
+                    this.logger.debug({ service: desired.name, failures: backoff.failures, retryInMs: backoff.nextAttemptAt - Date.now() }, 'Service not running — in restart backoff, skipping tick');
+                    continue;
+                }
                 this.logger.warn({ service: desired.name, status: actual?.status ?? 'not_found' }, 'Service not running — auto-restarting');
                 try {
                     await this.reconcileService(desired);
+                    this.restartBackoff.delete(desired.name);
                 }
                 catch (err) {
-                    this.logger.error({ service: desired.name, error: err.message }, 'Auto-restart failed');
+                    const failures = (this.restartBackoff.get(desired.name)?.failures ?? 0) + 1;
+                    const backoffMs = restartBackoffMs(failures);
+                    this.restartBackoff.set(desired.name, { failures, nextAttemptAt: Date.now() + backoffMs });
+                    this.logger.error({ service: desired.name, error: err.message, failures, backoffMs }, 'Auto-restart failed');
                 }
             }
             else if (actual.health === 'unhealthy') {
-                this.logger.warn({ service: desired.name }, 'Service unhealthy — restarting container');
-                try {
-                    await removeContainer(desired.name);
-                    await this.createAndStart(desired);
+                const ticks = (this.unhealthyTicks.get(desired.name) ?? 0) + 1;
+                this.unhealthyTicks.set(desired.name, ticks);
+                if (ticks < UNHEALTHY_RESTART_THRESHOLD) {
+                    this.logger.warn({ service: desired.name, consecutiveUnhealthy: ticks, threshold: UNHEALTHY_RESTART_THRESHOLD }, 'Service unhealthy — waiting for confirmation before restart');
                 }
-                catch (err) {
-                    this.logger.error({ service: desired.name, error: err.message }, 'Health restart failed');
+                else {
+                    this.unhealthyTicks.delete(desired.name);
+                    this.logger.warn({ service: desired.name, consecutiveUnhealthy: ticks }, 'Service unhealthy past threshold — restarting container');
+                    try {
+                        await removeContainer(desired.name);
+                        await this.createAndStart(desired);
+                    }
+                    catch (err) {
+                        this.logger.error({ service: desired.name, error: err.message }, 'Health restart failed');
+                    }
                 }
             }
+            else {
+                // Healthy or starting — clear any accumulated unhealthy ticks and
+                // reset the restart circuit-breaker (it may have recovered on its own
+                // via Docker's restart policy).
+                this.unhealthyTicks.delete(desired.name);
+                this.restartBackoff.delete(desired.name);
+            }
             // Update state
             const newState = await getContainerState(desired.name);
             if (newState) {
@@ -499,4 +938,40 @@ export class InfrastructureService {
         this.state.lastReconciled = new Date().toISOString();
     }
 }
+/**
+ * T#76: shallow equality on the fields that drive container
+ * identity. Two declarations that differ only in incidental fields
+ * (label maps, restart policy adornments) should still be treated
+ * as the same shared container; differences in image / env / port
+ * mapping legitimately deserve an operator-facing warning.
+ */
+function shallowContainerEqual(a, b) {
+    if (a.image !== b.image)
+        return false;
+    if (!sameRecord(a.environment, b.environment))
+        return false;
+    if (a.ports.length !== b.ports.length)
+        return false;
+    for (let i = 0; i < a.ports.length; i++) {
+        if (a.ports[i].host !== b.ports[i].host || a.ports[i].container !== b.ports[i].container)
+            return false;
+    }
+    if (a.volumes.length !== b.volumes.length)
+        return false;
+    for (let i = 0; i < a.volumes.length; i++) {
+        if (a.volumes[i].source !== b.volumes[i].source || a.volumes[i].target !== b.volumes[i].target)
+            return false;
+    }
+    return true;
+}
+function sameRecord(a, b) {
+    const ka = Object.keys(a ?? {});
+    const kb = Object.keys(b ?? {});
+    if (ka.length !== kb.length)
+        return false;
+    for (const k of ka)
+        if (a?.[k] !== b?.[k])
+            return false;
+    return true;
+}
 //# sourceMappingURL=infrastructure.service.js.map