@omnitron-dev/omnitron 0.2.0 → 0.3.1

package/dist/infrastructure/infrastructure.service.js

@@ -15,23 +15,88 @@
  *   ResolvedContainer[] → ContainerRuntime → Docker CLI
  *   Health Loop → detect failure → auto-restart (exponential backoff)
  */
+import { computeBackoff } from '@omnitron-dev/titan/utils';
 import { resolveInfrastructure, resolveOmnitronPg, getManagedNetwork } from './service-resolver.js';
 import { PhantomEndpointJanitor } from './phantom-endpoint-janitor.js';
-import { isDockerAvailable, getContainerState, createContainer, startContainer, removeContainer, ensureImage, waitForHealthy, createVolume, } from './container-runtime.js';
+import { LOCAL_INFRA_HOST } from '../shared/local-infra-host.js';
+import { isDockerAvailable, getContainerState, createContainer, startContainer, removeContainer, ensureImage, waitForHealthy, createVolume, containerSpecHash, } from './container-runtime.js';
+import { summariseProvisioning, describeProvisioning } from './provisioning-outcome.js';
+import { OMNITRON_PG_PORT, OMNITRON_PG_USER, OMNITRON_PG_PASSWORD, OMNITRON_PG_DATABASE, } from '../database/connection.js';
 const RECONCILE_INTERVAL = 30_000; // 30s health sweep
 const STARTUP_TIMEOUT = 120_000; // 2min max per service
 /** Consecutive 'unhealthy' ticks before recreate. 2 ticks @ 30s = 60s grace. */
 const UNHEALTHY_RESTART_THRESHOLD = 2;
-/** Omnitron internal PG connection defaults */
-const OMNITRON_PG_PORT = 5480;
-const OMNITRON_PG_USER = 'omnitron';
-const OMNITRON_PG_PASSWORD = 'omnitron';
-const OMNITRON_PG_DATABASE = 'omnitron';
+/** Restart circuit-breaker: first failed-restart backoff, doubled each failure. */
+const RESTART_BACKOFF_BASE = 30_000; // 30s
+/** Restart circuit-breaker: backoff ceiling (30s → 1m → 2m … capped here). */
+const RESTART_BACKOFF_MAX = 15 * 60_000; // 15min
+/**
+ * Restart-circuit-breaker backoff schedule: the delay (ms) to wait before the
+ * next restart attempt after `failures` consecutive failed (re)creates.
+ * Exponential from RESTART_BACKOFF_BASE, capped at RESTART_BACKOFF_MAX so a
+ * permanently-broken service settles into an occasional retry instead of
+ * hammering the supervisor every 30s.
+ *
+ * Delegates to the shared titan `computeBackoff` (the same helper the downstream
+ * backends and the titan packages use) so the exponential shape lives in ONE
+ * place. Exported for tests.
+ */
+export function restartBackoffMs(failures) {
+    if (failures < 1)
+        return 0;
+    return computeBackoff({ attempt: failures - 1, baseMs: RESTART_BACKOFF_BASE, maxMs: RESTART_BACKOFF_MAX });
+}
+// Omnitron internal PG connection defaults live in `database/connection.ts`
+// so the container spec below and every client that connects to it are
+// derived from the same values (see the import above).
+/**
+ * What to do about the control plane's own database.
+ *
+ * Three cases that look alike and are not:
+ *
+ *   - a node's daemon keeps its state in SQLite and needs no database at all;
+ *   - one is running, so it gets the same reconcile — and so the same
+ *     `containerSpecHash` drift check — as every app service;
+ *   - nothing is running, so provision.
+ *
+ * The middle case is the one that was missing. "Already running" was read as
+ * "configured as declared", and the database was adopted exactly as it stood.
+ * Measured: `omnitron-pg`, created before published ports were bound to
+ * loopback, survived on 0.0.0.0:5480 through the fix that reached all eleven
+ * other managed containers, and still answered `nc -vz <lan-address> 5480`
+ * from another machine months later, while `sample-dev-postgres` beside it in
+ * `docker ps` was correctly on 127.0.0.1.
+ *
+ * `reconcileName` is the subtlety. The control-plane database is ONE
+ * container per host — it owns port 5480 — but each daemon resolves a name
+ * carrying its own project prefix, so a daemon on the `sample/dev` stack
+ * computes `sample-dev-pg` while the container that exists is `omnitron-pg`.
+ * Reconciling the running container against a spec under the OTHER name
+ * creates a second one and collides on the port; the first version of this
+ * did exactly that and corrected nothing. The name to reconcile is therefore
+ * the name of the container that is actually there, and the spec is
+ * everything else. `containerSpecHash` does not cover the name, so the drift
+ * comparison is unaffected either way.
+ *
+ * A differing name is not evidence of another owner: only omnitron creates
+ * this container, and it carries omnitron's own `omnitron.internal` label.
+ */
+export function decideControlPlaneDatabase(input) {
+    if (!input.needsControlPlaneDatabase)
+        return { action: 'skip' };
+    if (input.running?.status !== 'running')
+        return { action: 'provision' };
+    return { action: 'reconcile', reconcileName: input.running.name ?? input.desiredName };
+}
 export class InfrastructureService {
     logger;
     config;
     presetRegistry;
+    serviceOverrides;
+    needsControlPlaneDatabase;
     desiredContainers = [];
+    /** What the most recent reconcile did to each service. */
+    lastActions = new Map();
     omnitronPgContainer;
     usingGlobalOmnitronPg = false;
     healthTimer = null;
@@ -46,17 +111,45 @@ export class InfrastructureService {
      * making transient pressure look like outages.
      */
     unhealthyTicks = new Map();
+    /**
+     * Per-service restart circuit-breaker. A container whose (re)create keeps
+     * failing — an image that won't pull, or infra stuck on an unrecoverable
+     * error — must NOT be retried every 30s forever: that churn (docker calls +
+     * error logging) is what wedged the daemon's event loop and let an external
+     * "socket unreachable → SIGTERM" kill the whole daemon during the 2026-05
+     * outage. After each failed restart we back off exponentially (30s → 1m →
+     * 2m … capped at 15m); a successful restart or an externally-recovered
+     * 'running' observation resets it.
+     */
+    restartBackoff = new Map();
     /**
      * T#75: phantom-endpoint janitor instance. Created in `provision()`
      * once we know Docker is reachable; torn down in `teardown()`.
      */
     phantomJanitor = null;
-    constructor(logger, config, normalizedServices, presetRegistry) {
+    constructor(logger, config, normalizedServices, presetRegistry, 
+    /**
+     * Stack overrides. A service marked `disabled` here is not provisioned,
+     * and one pointed at an `external` address is not provisioned either —
+     * there is nothing to create, and the address reaches the application
+     * through its environment instead.
+     */
+    serviceOverrides, 
+    /**
+     * Whether this daemon stores its own state in Postgres.
+     *
+     * True for a master, which is where projects, nodes, metrics and logs
+     * live. False for a node: its daemon keeps that in SQLite and reads a
+     * Postgres never.
+     */
+    needsControlPlaneDatabase = true) {
         this.logger = logger;
         this.config = config;
         this.presetRegistry = presetRegistry;
+        this.serviceOverrides = serviceOverrides;
+        this.needsControlPlaneDatabase = needsControlPlaneDatabase;
         this.normalizedServices = normalizedServices ?? {};
-        this.desiredContainers = resolveInfrastructure(config, normalizedServices);
+        this.desiredContainers = resolveInfrastructure(config, normalizedServices, serviceOverrides);
         this.omnitronPgContainer = resolveOmnitronPg();
     }
     /**
@@ -111,26 +204,52 @@ export class InfrastructureService {
         //    The stack-prefixed container (e.g. omni-dev-pg) shares the same port,
         //    so we must not create/start it when the global one is active.
         const globalOmnitronPg = await getContainerState('omnitron-pg');
-        const stackPgName = this.omnitronPgContainer.name;
-        const isGlobalPgRunning = globalOmnitronPg?.status === 'running';
-        const isStackPgSameAsGlobal = stackPgName === 'omnitron-pg';
-        if (isGlobalPgRunning && !isStackPgSameAsGlobal) {
-            // Global omnitron-pg already owns port 5480 — reuse it and skip stack-prefixed container.
+        const decision = decideControlPlaneDatabase({
+            needsControlPlaneDatabase: this.needsControlPlaneDatabase,
+            running: globalOmnitronPg,
+            desiredName: this.omnitronPgContainer.name,
+        });
+        if (decision.action === 'skip') {
+            // A node's daemon keeps its own state in SQLite — `SlaveStorageService`,
+            // `~/.omnitron/data/slave.db` — and reads this database never. Creating
+            // it anyway gave every provisioned node a Postgres nobody queries, on
+            // default credentials, and until published ports were bound to
+            // loopback it was reachable from the internet: measured answering on
+            // 0.0.0.0:5480 on a host whose firewall allows only SSH.
+            //
+            // The control plane's database belongs to the control plane.
+            this.logger.debug('This daemon keeps its own state in SQLite — not provisioning a control-plane database');
+        }
+        else if (decision.action === 'reconcile') {
             this.usingGlobalOmnitronPg = true;
-            this.logger.info({ global: 'omnitron-pg', stackPg: stackPgName }, 'Using existing omnitron-pg (already running) — skipping stack-prefixed PG');
-            // Remove stale stack-prefixed container if it exists (leftover from a previous attempt)
-            const stalePg = await getContainerState(stackPgName);
-            if (stalePg) {
-                try {
-                    await removeContainer(stackPgName);
-                    this.logger.info({ service: stackPgName }, 'Removed stale stack-prefixed PG container');
+            // A stack-prefixed leftover cannot run beside the global one — they
+            // share port 5480 — so it goes before anything else touches that port.
+            if (this.omnitronPgContainer.name !== decision.reconcileName) {
+                const stalePg = await getContainerState(this.omnitronPgContainer.name);
+                if (stalePg) {
+                    try {
+                        await removeContainer(this.omnitronPgContainer.name);
+                        this.logger.info({ service: this.omnitronPgContainer.name }, 'Removed stale stack-prefixed PG container');
+                    }
+                    catch { /* already gone */ }
                 }
-                catch { /* already gone */ }
             }
-        }
-        else if (isGlobalPgRunning) {
-            this.usingGlobalOmnitronPg = true;
-            this.logger.info('Using existing omnitron-pg (already running)');
+            // The same reconcile, and so the same drift check, as every app
+            // service — under the name the container actually has. Its data lives
+            // in the `omnitron-pg-data` volume and outlives the container it is
+            // recreated into.
+            // Name AND volume, not one of them. `resolveOmnitronPg` derives both
+            // from the container prefix, so overriding only the name produced a
+            // container called `omnitron-pg` mounting `sample-dev-pg-data` — the
+            // global identity on a stack-scoped volume, and the stack's own
+            // `omnitron-pg-data` left dangling where a `docker volume prune` would
+            // take it silently. The two names are one decision and must move
+            // together.
+            const reconciled = { ...this.omnitronPgContainer, name: decision.reconcileName };
+            if (decision.reconcileName !== this.omnitronPgContainer.name) {
+                reconciled.volumes = this.omnitronPgContainer.volumes.map((v) => v.target === '/var/lib/postgresql/data' ? { ...v, source: `${decision.reconcileName}-data` } : v);
+            }
+            await this.reconcileService(reconciled);
         }
         else {
             try {
@@ -147,8 +266,25 @@ export class InfrastructureService {
             }
         }
         this.logger.info({ services: this.desiredContainers.map((c) => c.name) }, 'Provisioning app infrastructure');
-        // 3. Reconcile each app service
-        for (const desired of this.desiredContainers) {
+        // 3. Reconcile every app service in parallel.
+        //
+        // Pre-fix this was a serial `for await` loop over up to a dozen
+        // containers. For an already-healthy stack (the common cold-boot
+        // case where Docker preserved containers across a daemon
+        // restart) each reconcileService is a single `docker inspect`
+        // round-trip — but doing nine of them serially still costs
+        // multiple seconds of latency on the critical path before
+        // apps can even start spawning.
+        //
+        // Safety: `reconcileService` holds a per-name lock
+        // (`this.reconciling`), so parallel calls for the SAME
+        // container still serialise. Cross-container ordering (e.g.
+        // monero-wallet-rpc must talk to monero-daemon) is enforced
+        // by each container's own healthcheck loop — the wallet's
+        // health probe retries until the daemon's RPC accepts
+        // connections, so launching them in parallel just extends the
+        // wallet's first-try wait, not its end state.
+        await Promise.all(this.desiredContainers.map(async (desired) => {
             try {
                 await this.reconcileService(desired);
             }
@@ -161,7 +297,7 @@ export class InfrastructureService {
                     error: err.message,
                 };
             }
-        }
+        }));
         // 4. Post-provisioning setup (create databases, buckets, etc.)
         await this.postProvision();
         // 4. Start health monitor
@@ -177,15 +313,107 @@ export class InfrastructureService {
             logger: this.logger,
         });
         this.phantomJanitor.start();
-        this.state.ready = true;
+        // Readiness is measured, not announced.
+        //
+        // These two lines were `ready = true` and "Infrastructure provisioned
+        // and healthy", unconditionally — and every service reconciles inside a
+        // `Promise.all` whose per-service catch records `exited` and continues.
+        // So a run where every container failed ended exactly like a run where
+        // all of them came up, and the gate below this waits on a flag that
+        // could not say no. See `summariseProvisioning`.
+        const outcome = summariseProvisioning(this.desiredContainers, this.state.services);
+        this.state.ready = outcome.ready;
         this.state.lastReconciled = new Date().toISOString();
-        this.logger.info('Infrastructure provisioned and healthy');
+        if (outcome.ready) {
+            this.logger.info(describeProvisioning(outcome));
+        }
+        else {
+            this.logger.error({
+                missing: outcome.missing.map((s) => s.name),
+                failed: outcome.failed.map((s) => ({ name: s.name, status: s.status, error: s.error })),
+                running: outcome.running.length,
+            }, describeProvisioning(outcome));
+        }
         return this.state;
     }
     /**
-     * Stop all managed infrastructure containers.
+     * What this service was asked to provide.
+     *
+     * Exposed because a caller cannot tell "present and fine" from "never
+     * attempted" by reading `state.services` alone: the second leaves no entry,
+     * and an absent entry is indistinguishable from an absent problem.
      */
-    async teardown() {
+    getDesiredServices() {
+        return this.desiredContainers.map((c) => ({ name: c.name, image: c.image }));
+    }
+    /**
+     * Transfer monitoring ownership to a freshly resolved declaration. This never
+     * provisions containers: retained specs must match; additions must already be
+     * running with exact native ownership and spec labels. Freeze and drain all
+     * earlier monitor/reconcile work before acknowledging a handoff.
+     */
+    async refreshOwnershipFrom(candidate) {
+        if (this.ownershipFrozen)
+            throw new Error('Infrastructure ownership refresh already in progress');
+        this.ownershipFrozen = true;
+        try {
+            const drained = await Promise.allSettled([...this.activeSweeps, ...this.reconcileTasks.values()]);
+            if (drained.some(result => result.status === 'rejected'))
+                throw new Error('In-flight infrastructure reconciliation failed; retry ownership refresh');
+            const current = new Map(this.desiredContainers.map(c => [c.name, c]));
+            const next = new Map(candidate.desiredContainers.map(c => [c.name, c]));
+            if (next.size !== candidate.desiredContainers.length)
+                throw new Error('Duplicate desired container identity');
+            const adopted = [];
+            for (const [name, desired] of next) {
+                const prior = current.get(name);
+                if (prior) {
+                    if (containerSpecHash(prior) !== containerSpecHash(desired) || !sameRecord(prior.labels, desired.labels)) {
+                        throw new Error(`Ownership refresh cannot change retained service spec: ${name}`);
+                    }
+                    continue;
+                }
+                const actual = await getContainerState(name);
+                const labels = desired.labels;
+                if (!labels['omnitron.project'] || !labels['omnitron.stack'] || labels['omnitron.managed'] !== 'true' ||
+                    actual?.status !== 'running' || actual.networkAttached === false || actual.specHash !== containerSpecHash(desired) ||
+                    Object.entries(labels).some(([key, value]) => actual.labels?.[key] !== value)) {
+                    throw new Error(`Cannot adopt service without matching running native ownership and spec: ${name}`);
+                }
+                adopted.push(name);
+            }
+            const released = [...current.keys()].filter(name => !next.has(name));
+            this.desiredContainers = [...candidate.desiredContainers];
+            this.config = candidate.config;
+            this.normalizedServices = candidate.normalizedServices;
+            this.serviceOverrides = candidate.serviceOverrides;
+            for (const name of released) {
+                delete this.state.services[name];
+                this.unhealthyTicks.delete(name);
+                this.restartBackoff.delete(name);
+                this.lastActions.delete(name);
+            }
+            return { released, adopted };
+        }
+        finally {
+            this.ownershipFrozen = false;
+        }
+    }
+    /**
+     * Services whose socket is new since this reconcile began.
+     *
+     * `noop` is the only action that leaves an existing connection usable.
+     * Everything else means anything already connected is holding a dead pool.
+     */
+    getRestartedServices() {
+        return [...this.lastActions.entries()]
+            .filter(([, action]) => action === 'create' || action === 'start' || action === 'recreate')
+            .map(([name]) => name);
+    }
+    /**
+     * Release process-owned monitoring resources without changing infrastructure.
+     */
+    dispose() {
         this.stopHealthMonitor();
         // T#75: stop the janitor BEFORE removing containers so a final
         // tick can't race the teardown's `docker rm` and produce noisy
@@ -194,11 +422,13 @@ export class InfrastructureService {
             this.phantomJanitor.stop();
             this.phantomJanitor = null;
         }
-        // Remove all containers in parallel for fast shutdown.
-        // Skip the stack-prefixed PG when using the global omnitron-pg (it's shared).
-        const allContainers = this.usingGlobalOmnitronPg
-            ? [...this.desiredContainers].reverse()
-            : [...[...this.desiredContainers].reverse(), this.omnitronPgContainer];
+    }
+    /** Explicit infrastructure removal; process disposal alone preserves containers. */
+    async teardown() {
+        this.dispose();
+        // Remove all containers in parallel for fast shutdown — the control-plane
+        // database only where this service keeps one (`controlPlaneContainers`).
+        const allContainers = [...[...this.desiredContainers].reverse(), ...this.controlPlaneContainers()];
         await Promise.allSettled(allContainers.map(async (c) => {
             try {
                 await removeContainer(c.name);
@@ -216,6 +446,27 @@ export class InfrastructureService {
     getState() {
         return this.state;
     }
+    /**
+     * Serve the stack's definition as it is now.
+     *
+     * A node's daemon builds this service on its first `provisionStack` and
+     * reuses it for every later one — rightly: a second service would be a
+     * second janitor and a second health monitor over one set of containers.
+     * But the definition it was built with stayed its definition. A later
+     * deployment that enabled a service, changed an image or dropped a
+     * `disabled` reached the node's host services — `reconcileHostServices`
+     * reads each call's own — and not its containers, until something restarted
+     * the daemon. Replaced whole, as the constructor sets it; `setConfigRoots`
+     * and `addAppContainers` then work on this set, as on a first call.
+     */
+    redefine(config, normalizedServices, serviceOverrides) {
+        if (this.ownershipFrozen)
+            throw new Error('Infrastructure ownership refresh in progress');
+        this.config = config;
+        this.normalizedServices = normalizedServices;
+        this.serviceOverrides = serviceOverrides;
+        this.desiredContainers = resolveInfrastructure(config, normalizedServices, serviceOverrides);
+    }
     /**
      * Get normalized services (from preset system).
      * Used by config-resolver for generic address resolution.
@@ -223,6 +474,28 @@ export class InfrastructureService {
     getNormalizedServices() {
         return this.normalizedServices;
     }
+    /**
+     * Databases declared for this stack's postgres (e.g. main, storage, geo).
+     * Source of truth for "which DBs must be backed up" — read straight from
+     * the same `infrastructure.postgres.databases` block omnitron provisions
+     * from, so the backup set can never drift from the provisioned set.
+     */
+    getPostgresDatabases() {
+        const pg = this.config.postgres;
+        if (!pg || !pg.databases)
+            return [];
+        return Object.keys(pg.databases);
+    }
+    /**
+     * Resolved container name for an infra service (e.g. 'postgres' →
+     * 'acme-dev-postgres'). Taken from the already-resolved desired containers
+     * so it reflects the real container prefix rather than re-deriving it from
+     * the mutable module-global. Returns null if the stack has no such service.
+     */
+    getResolvedContainerName(service) {
+        const match = this.desiredContainers.find((d) => d.name.endsWith(`-${service}`));
+        return match?.name ?? null;
+    }
     /**
      * Get connection info for an infrastructure service.
      * Used by config generator to resolve { infra: 'postgres', database: 'main' }.
@@ -234,7 +507,7 @@ export class InfrastructureService {
                 if (!pg)
                     return null;
                 return {
-                    host: 'localhost',
+                    host: LOCAL_INFRA_HOST,
                     port: pg.port ?? 5432,
                     user: pg.user ?? 'postgres',
                     password: typeof pg.password === 'string' ? pg.password : 'postgres',
@@ -245,7 +518,7 @@ export class InfrastructureService {
                 if (!redis)
                     return null;
                 return {
-                    host: 'localhost',
+                    host: LOCAL_INFRA_HOST,
                     port: redis.port ?? 6379,
                     password: typeof redis.password === 'string' ? redis.password : undefined,
                     databases: redis.databases ?? {},
@@ -256,7 +529,7 @@ export class InfrastructureService {
                 if (!minio)
                     return null;
                 return {
-                    endpoint: `http://localhost:${minio.ports?.api ?? 9000}`,
+                    endpoint: `http://${LOCAL_INFRA_HOST}:${minio.ports?.api ?? 9000}`,
                     accessKey: minio.accessKey ?? 'minioadmin',
                     secretKey: typeof minio.secretKey === 'string' ? minio.secretKey : 'minioadmin',
                     forcePathStyle: true,
@@ -264,7 +537,7 @@ export class InfrastructureService {
             }
             case 'omnitron-pg':
                 return {
-                    host: 'localhost',
+                    host: LOCAL_INFRA_HOST,
                     port: OMNITRON_PG_PORT,
                     user: OMNITRON_PG_USER,
                     password: OMNITRON_PG_PASSWORD,
@@ -288,23 +561,70 @@ export class InfrastructureService {
      * own restart-policy in the mix) occasional duplicate containers.
      */
     reconciling = new Set();
+    reconcileTasks = new Map();
+    activeSweeps = new Set();
+    ownershipFrozen = false;
+    /**
+     * Where this node keeps each service's config files, once a master sent them.
+     *
+     * The gateway is not an ordinary preset container — it needs four bind
+     * mounts, an entrypoint and fifteen upstream variables, and `resolveGateway`
+     * is the only thing that produces them. The constructor resolved everything
+     * through the generic preset path, because at construction time this node
+     * had no files and no local path to mount from.
+     *
+     * So this RE-RESOLVES rather than patching the volumes of a container built
+     * without them. A spec assembled by the wrong resolver is missing more than
+     * paths, and rewriting one field of it would produce a container that looks
+     * configured and is not — which is the failure that was already shipping,
+     * one level down.
+     *
+     * Called before `provision()`, so the recreation the changed spec hash
+     * implies happens on the first pass rather than the second.
+     */
+    setConfigRoots(roots, staticRoots, redis, 
+    /**
+     * The static ROOT per service, when the delivery made a
+     * `current-<project>-<stack>` link inside it. Given, the gateway mounts
+     * the root and serves through the link — a path that does not change from
+     * one release to the next, so the container is not recreated and the
+     * second of 000 a deployment cost goes with it. Empty, everything below
+     * resolves exactly as before.
+     */
+    staticLinkRoots) {
+        if (roots.size === 0)
+            return;
+        this.desiredContainers = resolveInfrastructure(this.config, this.normalizedServices, this.serviceOverrides, roots, { redis, staticRoots, ...(staticLinkRoots && { staticLinkRoots }) });
+    }
     async reconcileService(desired) {
         // T#58: per-name lock around the whole inspect→decide→act loop.
-        if (this.reconciling.has(desired.name)) {
+        if (this.ownershipFrozen || this.reconciling.has(desired.name)) {
             this.logger.debug({ service: desired.name }, 'Reconcile already in flight — skipping');
             return;
         }
         this.reconciling.add(desired.name);
+        const task = this.reconcileServiceLocked(desired);
+        this.reconcileTasks.set(desired.name, task);
         try {
-            await this.reconcileServiceLocked(desired);
+            await task;
         }
         finally {
             this.reconciling.delete(desired.name);
+            this.reconcileTasks.delete(desired.name);
         }
     }
     async reconcileServiceLocked(desired) {
         const actual = await getContainerState(desired.name);
+        if (this.ownershipFrozen)
+            return;
         const action = this.computeAction(desired, actual);
+        // A container that was created, started or recreated is listening on a
+        // NEW socket. Applications already running hold pools pointing at the
+        // old one, and a titan liveness probe does not notice — the app answers
+        // `/health` 200 while every request it serves fails with "Connection
+        // terminated unexpectedly". Whoever asked for this reconcile is the only
+        // one in a position to restart them, so it has to be told.
+        this.lastActions.set(desired.name, action.type);
         switch (action.type) {
             case 'noop':
                 this.logger.debug({ service: desired.name }, 'Service already running');
@@ -340,6 +660,17 @@ export class InfrastructureService {
             return { type: 'create', service: desired.name, config: desired };
         }
         if (actual.status === 'running') {
+            // Running but detached from all networks (OrbStack/dockerd-restart
+            // artifact): the container is unreachable and its published ports are
+            // gone. 'running' is a lie here — recreate to re-attach + re-publish.
+            if (actual.networkAttached === false) {
+                return {
+                    type: 'recreate',
+                    service: desired.name,
+                    config: desired,
+                    reason: 'network-detached (running but on no network)',
+                };
+            }
             // Check if image changed
             if (actual.image !== desired.image && !actual.image.startsWith(desired.image)) {
                 return {
@@ -349,9 +680,55 @@ export class InfrastructureService {
                     reason: `image changed: ${actual.image} → ${desired.image}`,
                 };
             }
+            // Config drift: image unchanged but the desired spec (env, ports,
+            // volumes, command, extraHosts) differs from what the running container
+            // was created with. Without this, an omnitron.config.ts edit that only
+            // touches env — e.g. the Tor hidden-service target — is silently ignored
+            // because the container is "running and healthy". Only fires when the
+            // running container carries a spec-hash label; pre-upgrade containers
+            // (no label) are left alone until they recreate for another reason, so
+            // upgrading omnitron never churns an otherwise-healthy stack.
+            if (actual.specHash) {
+                const desiredHash = containerSpecHash(desired);
+                if (actual.specHash !== desiredHash) {
+                    return {
+                        type: 'recreate',
+                        service: desired.name,
+                        config: desired,
+                        reason: `config drift (spec ${actual.specHash} → ${desiredHash})`,
+                    };
+                }
+            }
             return { type: 'noop', service: desired.name };
         }
         if (actual.status === 'exited' || actual.status === 'created') {
+            // A container that is not running still carries the spec it was created
+            // with, and `start` runs THAT spec. The drift checks above were only
+            // asked of a running container, so one created wrong was started wrong
+            // for as long as it existed: Nominatim on sample/test, created at 05:37
+            // with a mount the master had not rewritten, was «started» by every
+            // reconcile after the fix that corrected its desired spec — «exec:
+            // /opt/nominatim-tools/cis-entrypoint.sh: no such file» each time
+            // (2026-09-25). Same questions, same answers, before it is started.
+            if (actual.image !== desired.image && !actual.image.startsWith(desired.image)) {
+                return {
+                    type: 'recreate',
+                    service: desired.name,
+                    config: desired,
+                    reason: `image changed while stopped: ${actual.image} → ${desired.image}`,
+                };
+            }
+            if (actual.specHash) {
+                const desiredHash = containerSpecHash(desired);
+                if (actual.specHash !== desiredHash) {
+                    return {
+                        type: 'recreate',
+                        service: desired.name,
+                        config: desired,
+                        reason: `config drift while ${actual.status} (spec ${actual.specHash} → ${desiredHash})`,
+                    };
+                }
+            }
             return { type: 'start', service: desired.name, containerId: actual.containerId ?? desired.name };
         }
         // Dead or other — recreate
@@ -432,6 +809,26 @@ export class InfrastructureService {
         const { execFile } = await import('node:child_process');
         const { promisify } = await import('node:util');
         const execFileAsync = promisify(execFile);
+        // A preset hook is what creates the databases inside Postgres and the
+        // buckets inside MinIO — the containers are running either way, so a
+        // failure here is invisible until an application cannot find its
+        // database.
+        //
+        // `this.presetRegistry!` asserted a dependency the constructor declares
+        // OPTIONAL, so a caller that did not pass one — which the type permits —
+        // got `Cannot read properties of undefined (reading 'get')` from a line
+        // that reads as internal, after every container was already up.
+        // Measured on a node provisioning a stack, where the containers stayed
+        // and the databases were never made.
+        if (!this.presetRegistry) {
+            const withPresets = Object.entries(this.normalizedServices)
+                .filter(([, r]) => r._preset)
+                .map(([name]) => name);
+            if (withPresets.length > 0) {
+                this.logger.error({ services: withPresets }, 'No preset registry, so post-provision steps were skipped — these services are running but their databases and buckets were not created');
+            }
+            return;
+        }
         for (const [serviceName, requirement] of Object.entries(this.normalizedServices)) {
             if (!requirement._preset)
                 continue;
@@ -489,51 +886,20 @@ export class InfrastructureService {
         this.logger.info('Omnitron internal database provisioned and migrated');
     }
     /**
-     * Connect to Omnitron's PG and run migrations via Kysely's built-in Migrator.
-     * Uses dynamic import to keep pg/kysely optional until actually needed.
+     * Apply the internal-database schema through the shared runner.
+     *
+     * This used to be a second, independently-maintained copy of the daemon's
+     * migration code that only knew migrations 001–002 — and, like the
+     * daemon's copy, it constructed `Migrator` from the `kysely` package root
+     * where that export does not exist, so it threw on every run. Both are now
+     * one call into `database/migration-runner.ts`.
+     *
+     * Errors propagate to `provisionOmnitronDatabase`'s caller: provisioning
+     * that ends with an unmigrated database has not succeeded.
      */
     async runOmnitronMigrations() {
-        const { Kysely, PostgresDialect, Migrator } = await import('kysely');
-        const pg = await import('pg');
-        const m001 = await import('../database/migrations/001_initial_schema.js');
-        const m002 = await import('../database/migrations/002_metrics_raw.js');
-        const pool = new pg.default.Pool({
-            host: 'localhost',
-            port: OMNITRON_PG_PORT,
-            database: OMNITRON_PG_DATABASE,
-            user: OMNITRON_PG_USER,
-            password: OMNITRON_PG_PASSWORD,
-        });
-        const db = new Kysely({
-            dialect: new PostgresDialect({ pool }),
-        });
-        try {
-            const migrator = new Migrator({
-                db,
-                provider: {
-                    async getMigrations() {
-                        return {
-                            '001_initial_schema': { up: m001.up, down: m001.down },
-                            '002_metrics_raw': { up: m002.up, down: m002.down },
-                        };
-                    },
-                },
-            });
-            const { results, error } = await migrator.migrateToLatest();
-            const applied = results?.filter((r) => r.status === 'Success') ?? [];
-            if (applied.length > 0) {
-                this.logger.info({ migrations: applied.map((r) => r.migrationName) }, `Applied ${applied.length} Omnitron migration(s)`);
-            }
-            else {
-                this.logger.debug('Omnitron database schema is up to date');
-            }
-            if (error) {
-                this.logger.error({ error: String(error) }, 'Migration error');
-            }
-        }
-        finally {
-            await db.destroy();
-        }
+        const { runOmnitronMigrations } = await import('../database/migration-runner.js');
+        await runOmnitronMigrations(this.logger);
     }
     // ===========================================================================
     // Private — Health Monitor (Self-Healing)
@@ -552,14 +918,40 @@ export class InfrastructureService {
             this.healthTimer = null;
         }
     }
+    /**
+     * The control-plane database container this service keeps, if any: none on
+     * a node — it keeps its own state in SQLite (`decideControlPlaneDatabase`) —
+     * and none where the global `omnitron-pg` serves.
+     *
+     * `provision()` asked `needsControlPlaneDatabase`; the health sweep and the
+     * teardown asked only `usingGlobalOmnitronPg`. So on a node the sweep found
+     * `<prefix>-pg` «not running» every 30 s and created it: sample/test got one on
+     * 2026-09-20, and again on 2026-09-25 forty seconds after it was removed —
+     * on the default credentials, for a database nothing reads.
+     */
+    controlPlaneContainers() {
+        return this.needsControlPlaneDatabase && !this.usingGlobalOmnitronPg ? [this.omnitronPgContainer] : [];
+    }
     async healthSweep() {
-        // Include omnitron-pg in health sweep alongside app containers,
-        // but skip the stack-prefixed PG when using the global omnitron-pg
-        const allContainers = this.usingGlobalOmnitronPg
-            ? [...this.desiredContainers]
-            : [this.omnitronPgContainer, ...this.desiredContainers];
+        if (this.ownershipFrozen)
+            return;
+        const task = this.healthSweepOwned();
+        this.activeSweeps.add(task);
+        try {
+            await task;
+        }
+        finally {
+            this.activeSweeps.delete(task);
+        }
+    }
+    async healthSweepOwned() {
+        // The control-plane database alongside the app containers, where this
+        // service keeps one.
+        const allContainers = [...this.controlPlaneContainers(), ...this.desiredContainers];
         for (const desired of allContainers) {
             const actual = await getContainerState(desired.name);
+            if (this.ownershipFrozen)
+                return;
             // T#58: Docker's own `--restart=always` / `--restart=on-failure`
             // policy puts the container into the 'restarting' state for the
             // brief window between exit and re-entry. If we recreate during
@@ -573,12 +965,24 @@ export class InfrastructureService {
             }
             if (!actual || actual.status !== 'running') {
                 this.unhealthyTicks.delete(desired.name);
+                // Circuit-breaker: while a service is inside its backoff window, skip
+                // the restart so a permanently-failing container can't hammer the
+                // supervisor every tick and starve the daemon's event loop.
+                const backoff = this.restartBackoff.get(desired.name);
+                if (backoff && Date.now() < backoff.nextAttemptAt) {
+                    this.logger.debug({ service: desired.name, failures: backoff.failures, retryInMs: backoff.nextAttemptAt - Date.now() }, 'Service not running — in restart backoff, skipping tick');
+                    continue;
+                }
                 this.logger.warn({ service: desired.name, status: actual?.status ?? 'not_found' }, 'Service not running — auto-restarting');
                 try {
                     await this.reconcileService(desired);
+                    this.restartBackoff.delete(desired.name);
                 }
                 catch (err) {
-                    this.logger.error({ service: desired.name, error: err.message }, 'Auto-restart failed');
+                    const failures = (this.restartBackoff.get(desired.name)?.failures ?? 0) + 1;
+                    const backoffMs = restartBackoffMs(failures);
+                    this.restartBackoff.set(desired.name, { failures, nextAttemptAt: Date.now() + backoffMs });
+                    this.logger.error({ service: desired.name, error: err.message, failures, backoffMs }, 'Auto-restart failed');
                 }
             }
             else if (actual.health === 'unhealthy') {
@@ -600,8 +1004,11 @@ export class InfrastructureService {
                 }
             }
             else {
-                // Healthy or starting — clear any accumulated unhealthy ticks
+                // Healthy or starting — clear any accumulated unhealthy ticks and
+                // reset the restart circuit-breaker (it may have recovered on its own
+                // via Docker's restart policy).
                 this.unhealthyTicks.delete(desired.name);
+                this.restartBackoff.delete(desired.name);
             }
             // Update state
             const newState = await getContainerState(desired.name);