@omnitron-dev/omnitron 0.3.0 → 0.3.2

package/dist/infrastructure/infrastructure.service.js

@@ -1,31 +1,28 @@
-/**
- * InfrastructureService — Self-healing infrastructure provisioner
- *
- * Manages the complete lifecycle of infrastructure containers:
- *   1. Desired state declared in omnitron.config.ts
- *   2. Reconciliation loop diffs desired vs actual state
- *   3. Creates/starts/recreates containers as needed
- *   4. Continuous health monitoring with auto-restart
- *   5. Post-provisioning setup (create databases, buckets, etc.)
- *
- * Replaces: docker-compose.dev.yml + dev.sh + .env.dev
- *
- * Architecture:
- *   InfrastructureConfig → ServiceResolver → ResolvedContainer[]
- *   ResolvedContainer[] → ContainerRuntime → Docker CLI
- *   Health Loop → detect failure → auto-restart (exponential backoff)
- */
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+import { RecoveryMutation, RecoveryBusyError } from '../daemon/recovery-state.js';
 import { computeBackoff } from '@omnitron-dev/titan/utils';
+import { describeError } from '@omnitron-dev/titan/diagnostics';
+import { isDeepStrictEqual } from 'node:util';
 import { resolveInfrastructure, resolveOmnitronPg, getManagedNetwork } from './service-resolver.js';
+import { ownedContainerId } from './owned-container.js';
+import { observeUnhealthy } from './health-recovery.js';
+import { normalizeInfraConfig } from './config-normalizer.js';
+import { createDefaultRegistry } from './presets/index.js';
 import { PhantomEndpointJanitor } from './phantom-endpoint-janitor.js';
 import { LOCAL_INFRA_HOST } from '../shared/local-infra-host.js';
-import { isDockerAvailable, getContainerState, createContainer, startContainer, removeContainer, ensureImage, waitForHealthy, createVolume, containerSpecHash, } from './container-runtime.js';
-import { summariseProvisioning, describeProvisioning } from './provisioning-outcome.js';
+import { isDockerAvailable, getContainerState, createContainer, startContainer, removeContainer, ensureImage, waitForHealthy, createVolume, containerSpecHash, containerResources, containerResourcesMatch, } from './container-runtime.js';
+import { summariseProvisioning, describeProvisioning, retainProvisioningObservation } from './provisioning-outcome.js';
 import { OMNITRON_PG_PORT, OMNITRON_PG_USER, OMNITRON_PG_PASSWORD, OMNITRON_PG_DATABASE, } from '../database/connection.js';
 const RECONCILE_INTERVAL = 30_000; // 30s health sweep
 const STARTUP_TIMEOUT = 120_000; // 2min max per service
-/** Consecutive 'unhealthy' ticks before recreate. 2 ticks @ 30s = 60s grace. */
-const UNHEALTHY_RESTART_THRESHOLD = 2;
 /** Restart circuit-breaker: first failed-restart backoff, doubled each failure. */
 const RESTART_BACKOFF_BASE = 30_000; // 30s
 /** Restart circuit-breaker: backoff ceiling (30s → 1m → 2m … capped here). */
@@ -64,13 +61,13 @@ export function restartBackoffMs(failures) {
  * Measured: `omnitron-pg`, created before published ports were bound to
  * loopback, survived on 0.0.0.0:5480 through the fix that reached all eleven
  * other managed containers, and still answered `nc -vz <lan-address> 5480`
- * from another machine months later, while `daos-dev-postgres` beside it in
+ * from another machine months later, while `sample-dev-postgres` beside it in
  * `docker ps` was correctly on 127.0.0.1.
  *
  * `reconcileName` is the subtlety. The control-plane database is ONE
  * container per host — it owns port 5480 — but each daemon resolves a name
- * carrying its own project prefix, so a daemon on the `daos/dev` stack
- * computes `daos-dev-pg` while the container that exists is `omnitron-pg`.
+ * carrying its own project prefix, so a daemon on the `sample/dev` stack
+ * computes `sample-dev-pg` while the container that exists is `omnitron-pg`.
  * Reconciling the running container against a spec under the OTHER name
  * creates a second one and collides on the port; the first version of this
  * did exactly that and corrected nothing. The name to reconcile is therefore
@@ -94,6 +91,7 @@ export class InfrastructureService {
     presetRegistry;
     serviceOverrides;
     needsControlPlaneDatabase;
+    retiredStacks = new Set();
     desiredContainers = [];
     /** What the most recent reconcile did to each service. */
     lastActions = new Map();
@@ -102,15 +100,8 @@ export class InfrastructureService {
     healthTimer = null;
     state = { services: {}, ready: false };
     normalizedServices;
-    /**
-     * Per-service consecutive-unhealthy counter. Restart only fires after
-     * UNHEALTHY_RESTART_THRESHOLD ticks in a row to avoid flapping during
-     * cold-start / transient health-check timeouts. Reset to 0 on a healthy
-     * observation. Without this, a service that briefly fails one health
-     * probe (network blip, slow query) would be torn down and recreated,
-     * making transient pressure look like outages.
-     */
-    unhealthyTicks = new Map();
+    /** Distinct completed probes and elapsed grace, bound to one native incarnation. */
+    unhealthyEvidence = new Map();
     /**
      * Per-service restart circuit-breaker. A container whose (re)create keeps
      * failing — an image that won't pull, or infra stuck on an unrecoverable
@@ -127,6 +118,8 @@ export class InfrastructureService {
      * once we know Docker is reachable; torn down in `teardown()`.
      */
     phantomJanitor = null;
+    handoffMonitoring = false;
+    janitorNetworks = [];
     constructor(logger, config, normalizedServices, presetRegistry, 
     /**
      * Stack overrides. A service marked `disabled` here is not provisioned,
@@ -148,8 +141,14 @@ export class InfrastructureService {
         this.presetRegistry = presetRegistry;
         this.serviceOverrides = serviceOverrides;
         this.needsControlPlaneDatabase = needsControlPlaneDatabase;
-        this.normalizedServices = normalizedServices ?? {};
+        this.presetRegistry ??= createDefaultRegistry();
+        if (!normalizedServices) {
+            normalizedServices = normalizeInfraConfig(config, this.presetRegistry);
+        }
+        this.normalizedServices = normalizedServices;
         this.desiredContainers = resolveInfrastructure(config, normalizedServices, serviceOverrides);
+        for (const desired of this.desiredContainers)
+            containerResources(desired);
         this.omnitronPgContainer = resolveOmnitronPg();
     }
     /**
@@ -166,24 +165,25 @@ export class InfrastructureService {
      * benign collision — the shared service was actually fine).
      *
      * The intended semantics for shared infrastructure is "all apps that
-     * declare `postgres` use the same container", so the first
-     * declaration wins. When subsequent declarations differ in shape
-     * (image / env / ports), log a structured warning so the operator
-     * can reconcile intent vs. config.
+     * declare `postgres` use the same container", so
+     * identical declarations are shared. Conflicting specifications refuse
+     * the plan before anything is provisioned.
      */
     addAppContainers(containers) {
+        for (const desired of containers)
+            containerResources(desired);
+        if (this.monitoringRelinquished)
+            throw new Error('Infrastructure monitoring ownership has been relinquished');
+        if (this.ownershipFrozen)
+            throw new Error('Infrastructure ownership is frozen');
         for (const c of containers) {
             const existing = this.desiredContainers.find((d) => d.name === c.name);
             if (!existing) {
                 this.desiredContainers.push(c);
                 continue;
             }
-            if (!shallowContainerEqual(existing, c)) {
-                this.logger.warn({
-                    container: c.name,
-                    existingImage: existing.image,
-                    duplicateImage: c.image,
-                }, 'Duplicate container declaration with differing shape — keeping first; check for conflicting app infrastructure requirements (T#76)');
+            if (containerSpecHash(existing) !== containerSpecHash(c)) {
+                throw new Error(`Conflicting infrastructure declarations for ${c.name}; use one shared specification or distinct service names`);
             }
             // else: identical declaration, silent dedupe — this is the common case
         }
@@ -193,7 +193,62 @@ export class InfrastructureService {
      * Ensures Docker is available, pulls images, creates/starts containers,
      * runs post-provisioning setup, and starts the health monitor.
      */
-    async provision() {
+    async provision(definition) {
+        if (this.monitoringRelinquished)
+            throw new Error('Infrastructure monitoring ownership has been relinquished');
+        if (this.ownershipFrozen)
+            throw new Error('Infrastructure ownership is frozen');
+        if (this.provisioning) {
+            if (definition)
+                throw new Error('Infrastructure provisioning is already in progress; retry the new declaration after it finishes');
+            return this.provisioning;
+        }
+        const task = this.provisionDefinition(definition);
+        this.provisioning = task;
+        try {
+            return await task;
+        }
+        finally {
+            if (this.provisioning === task)
+                this.provisioning = undefined;
+        }
+    }
+    /** Resolve a plan without installing another monitor or changing the current definition. */
+    resolveDefinition(config, services, overrides) {
+        return new InfrastructureService(this.logger, config, services, this.presetRegistry, overrides, this.needsControlPlaneDatabase);
+    }
+    async provisionDefinition(definition) {
+        const monitored = this.healthTimer !== null;
+        this.definitionChanging = true;
+        try {
+            try {
+                // An old sweep must finish before fresh declarations can authorize native work.
+                const drained = await Promise.allSettled([...this.activeSweeps, ...this.reconcileTasks.values()]);
+                if (drained.some(result => result.status === 'rejected'))
+                    throw new Error('In-flight infrastructure reconciliation failed; inspect it before retrying provisioning');
+                await this.phantomJanitor?.drain();
+                if (definition && this.ownershipFrozen)
+                    throw new Error('Infrastructure ownership changed before the new declaration was applied');
+                if (definition)
+                    this.replaceDefinitionFrom(definition);
+                this.state.ready = false;
+            }
+            finally {
+                this.definitionChanging = false;
+            }
+            return await this.provisionOwned();
+        }
+        finally {
+            // Failed Docker admission must not silently abandon previously owned services.
+            if (monitored && !this.ownershipFrozen && !this.monitoringRelinquished) {
+                this.startHealthMonitor();
+                this.phantomJanitor?.start();
+            }
+        }
+    }
+    async provisionOwned() {
+        for (const desired of [...this.desiredContainers, ...this.controlPlaneContainers()])
+            containerResources(desired);
         // 1. Verify Docker is available
         if (!(await isDockerAvailable())) {
             throw new Error('Docker is not available. Omnitron requires Docker to manage infrastructure.\n' +
@@ -218,20 +273,17 @@ export class InfrastructureService {
             // 0.0.0.0:5480 on a host whose firewall allows only SSH.
             //
             // The control plane's database belongs to the control plane.
-            this.logger.debug('This daemon keeps its own state in SQLite — not provisioning a control-plane database');
+            this.logger.debug('Control-plane database provisioning is outside this infrastructure allocation');
         }
         else if (decision.action === 'reconcile') {
             this.usingGlobalOmnitronPg = true;
             // A stack-prefixed leftover cannot run beside the global one — they
             // share port 5480 — so it goes before anything else touches that port.
             if (this.omnitronPgContainer.name !== decision.reconcileName) {
-                const stalePg = await getContainerState(this.omnitronPgContainer.name);
+                const stalePg = await getContainerState(this.omnitronPgContainer.name, { orThrow: true, fullId: true });
                 if (stalePg) {
-                    try {
-                        await removeContainer(this.omnitronPgContainer.name);
-                        this.logger.info({ service: this.omnitronPgContainer.name }, 'Removed stale stack-prefixed PG container');
-                    }
-                    catch { /* already gone */ }
+                    await removeContainer(ownedContainerId(this.omnitronPgContainer, stalePg));
+                    this.logger.info({ service: this.omnitronPgContainer.name }, 'Removed stale stack-prefixed PG container');
                 }
             }
             // The same reconcile, and so the same drift check, as every app
@@ -240,7 +292,7 @@ export class InfrastructureService {
             // recreated into.
             // Name AND volume, not one of them. `resolveOmnitronPg` derives both
             // from the container prefix, so overriding only the name produced a
-            // container called `omnitron-pg` mounting `daos-dev-pg-data` — the
+            // container called `omnitron-pg` mounting `sample-dev-pg-data` — the
             // global identity on a stack-scoped volume, and the stack's own
             // `omnitron-pg-data` left dangling where a `docker volume prune` would
             // take it silently. The two names are one decision and must move
@@ -307,8 +359,9 @@ export class InfrastructureService {
         // a Docker daemon restart / host suspend doesn't leave us
         // unable to recreate containers ("endpoint with name X
         // already exists in network ...").
+        this.janitorNetworks = [getManagedNetwork()];
         this.phantomJanitor = new PhantomEndpointJanitor({
-            networks: [getManagedNetwork()],
+            networks: this.janitorNetworks,
             intervalMs: 60_000,
             logger: this.logger,
         });
@@ -343,9 +396,166 @@ export class InfrastructureService {
      * attempted" by reading `state.services` alone: the second leaves no entry,
      * and an absent entry is indistinguishable from an absent problem.
      */
+    /** Internal immutable plan. Never expose environment or command values through RPC. */
+    getDesiredContainers() {
+        return structuredClone(this.desiredContainers);
+    }
     getDesiredServices() {
         return this.desiredContainers.map((c) => ({ name: c.name, image: c.image }));
     }
+    /**
+     * Transfer monitoring ownership to a freshly resolved declaration. This never
+     * provisions containers: retained specs must match; additions must already be
+     * running with exact native ownership and spec labels. Freeze and drain all
+     * earlier monitor/reconcile work before acknowledging a handoff.
+     */
+    async refreshOwnershipFrom(candidate) {
+        if (this.ownershipFrozen || this.provisioning)
+            throw new Error('Infrastructure ownership refresh or provisioning already in progress');
+        this.ownershipFrozen = true;
+        try {
+            const drained = await Promise.allSettled([...this.activeSweeps, ...this.reconcileTasks.values()]);
+            if (drained.some(result => result.status === 'rejected'))
+                throw new Error('In-flight infrastructure reconciliation failed; retry ownership refresh');
+            const current = new Map(this.desiredContainers.map(c => [c.name, c]));
+            const next = new Map(candidate.desiredContainers.map(c => [c.name, c]));
+            if (next.size !== candidate.desiredContainers.length)
+                throw new Error('Duplicate desired container identity');
+            const adopted = [];
+            for (const [name, desired] of next) {
+                const prior = current.get(name);
+                if (prior) {
+                    if (containerSpecHash(prior) !== containerSpecHash(desired) || !sameRecord(prior.labels, desired.labels)) {
+                        throw new Error(`Ownership refresh cannot change retained service spec: ${name}`);
+                    }
+                    continue;
+                }
+                const actual = await getContainerState(name);
+                const labels = desired.labels;
+                if (!labels['omnitron.project'] || !labels['omnitron.stack'] || labels['omnitron.managed'] !== 'true' ||
+                    actual?.status !== 'running' || actual.networkAttached === false || actual.specHash !== containerSpecHash(desired) || !containerResourcesMatch(desired, actual) ||
+                    Object.entries(labels).some(([key, value]) => actual.labels?.[key] !== value)) {
+                    throw new Error(`Cannot adopt service without matching running native ownership and spec: ${name}`);
+                }
+                adopted.push(name);
+            }
+            const released = [...current.keys()].filter(name => !next.has(name));
+            this.replaceDefinitionFrom(candidate);
+            return { released, adopted };
+        }
+        finally {
+            this.ownershipFrozen = false;
+        }
+    }
+    /** Freeze and drain Docker mutations before another backend takes monitoring ownership. */
+    async prepareHandoff() {
+        if (this.ownershipFrozen)
+            throw new Error('Infrastructure ownership is already frozen');
+        this.ownershipFrozen = true;
+        this.handoffMonitoring = this.healthTimer !== null;
+        try {
+            const settled = await Promise.allSettled([...this.activeSweeps, ...this.reconcileTasks.values(), ...(this.provisioning ? [this.provisioning] : [])]);
+            if (settled.some(result => result.status === 'rejected'))
+                throw new Error('Infrastructure reconciliation failed while draining');
+            await this.phantomJanitor?.drain();
+            const observations = await InfrastructureService.validateHandoffContainers(this.desiredContainers, this.needsControlPlaneDatabase && !this.usingGlobalOmnitronPg ? this.omnitronPgContainer : undefined, this.state, { protocol: 1 });
+            this.dispose();
+            const snapshot = this.ownershipSnapshot(this.handoffMonitoring);
+            snapshot.nativeState = { protocol: 1, observations };
+            return snapshot;
+        }
+        catch (error) {
+            this.resumeAfterHandoff();
+            throw error;
+        }
+    }
+    captureRecoveryState() {
+        if (this.ownershipFrozen || this.provisioning || this.activeSweeps.size || this.reconcileTasks.size) {
+            throw new RecoveryBusyError('Infrastructure ownership is changing; a recovery snapshot is not ready');
+        }
+        return this.ownershipSnapshot(this.healthTimer !== null);
+    }
+    ownershipSnapshot(monitoring) {
+        return structuredClone({
+            nativeState: { protocol: 1 },
+            retiredStacks: [...this.retiredStacks],
+            config: this.config, containers: this.desiredContainers, normalizedServices: this.normalizedServices,
+            ...(this.serviceOverrides && { serviceOverrides: this.serviceOverrides }),
+            state: this.state, usingGlobalOmnitronPg: this.usingGlobalOmnitronPg,
+            needsControlPlaneDatabase: this.needsControlPlaneDatabase, monitoring,
+            controlPlaneContainer: this.omnitronPgContainer, janitorNetworks: this.janitorNetworks,
+            // Protocol1 rollback readers retain the old field but restart their
+            // confirmation window; poll counts cannot encode native observations.
+            unhealthyTicks: [], unhealthyEvidence: [...this.unhealthyEvidence], restartBackoff: [...this.restartBackoff],
+        });
+    }
+    resumeAfterHandoff() {
+        if (this.monitoringRelinquished)
+            return;
+        this.ownershipFrozen = false;
+        if (this.handoffMonitoring)
+            this.startHealthMonitor();
+        if (!this.phantomJanitor && this.janitorNetworks.length) {
+            this.phantomJanitor = new PhantomEndpointJanitor({ networks: this.janitorNetworks, intervalMs: 60_000, logger: this.logger });
+            this.phantomJanitor.start();
+        }
+        this.handoffMonitoring = false;
+    }
+    /** Read and validate existing containers. Never provision, start or recreate during adoption. */
+    static async adoptHandoff(logger, snapshot) {
+        await InfrastructureService.validateHandoffContainers(snapshot.containers, snapshot.needsControlPlaneDatabase && !snapshot.usingGlobalOmnitronPg ? snapshot.controlPlaneContainer : undefined, snapshot.state, snapshot.nativeState);
+        const service = new InfrastructureService(logger, snapshot.config, snapshot.normalizedServices, undefined, snapshot.serviceOverrides, snapshot.needsControlPlaneDatabase);
+        service.desiredContainers = structuredClone(snapshot.containers);
+        for (const key of snapshot.retiredStacks ?? [])
+            service.retiredStacks.add(key);
+        service.omnitronPgContainer = structuredClone(snapshot.controlPlaneContainer);
+        service.janitorNetworks = [...snapshot.janitorNetworks];
+        service.unhealthyEvidence = new Map(snapshot.unhealthyEvidence ?? []);
+        service.restartBackoff = new Map(snapshot.restartBackoff);
+        Object.assign(service.state, structuredClone(snapshot.state));
+        service.usingGlobalOmnitronPg = snapshot.usingGlobalOmnitronPg;
+        service.handoffMonitoring = snapshot.monitoring;
+        service.ownershipFrozen = true;
+        return service;
+    }
+    static async validateHandoffContainers(containers, controlPlane, state, nativeState) {
+        if (nativeState && nativeState.protocol !== 1)
+            throw new Error('Unsupported infrastructure native observation protocol');
+        const desiredContainers = [...containers, ...(controlPlane ? [controlPlane] : [])];
+        const expected = nativeState?.observations && new Map(nativeState.observations.map(entry => [entry.name, entry.containerId]));
+        if (expected && (expected.size !== nativeState.observations.length || expected.size !== desiredContainers.length ||
+            desiredContainers.some(desired => !expected.has(desired.name))))
+            throw new Error('Incomplete infrastructure native observations');
+        const observations = [];
+        for (const desired of desiredContainers) {
+            const actual = await getContainerState(desired.name, { orThrow: true, fullId: true });
+            const previous = state.services[desired.name];
+            // A failed creation owns no native container. Preserve that failure and
+            // its desired declaration; it must not make every backend upgrade fail.
+            // Missing previously observed or supposedly running resources still fail
+            // closed, as do legacy snapshots without the explicit protocol.
+            const failedCreation = nativeState?.protocol === 1 && previous?.status === 'exited' &&
+                !previous.containerId && typeof previous.error === 'string' && previous.error.length > 0;
+            if (!actual) {
+                if (!failedCreation || (expected && expected.get(desired.name) !== null))
+                    throw new Error(`Infrastructure identity or specification changed during handoff: ${desired.name}`);
+                observations.push({ name: desired.name, containerId: null });
+                continue;
+            }
+            // Protocol 1 exported Docker's 12-character display ID. Keep its wire
+            // shape for rollback, while comparing a full prior state ID when present.
+            const matches = (id) => id?.length === 12
+                ? actual.containerId?.startsWith(id) : id === actual.containerId;
+            if (!actual.containerId || actual.specHash !== containerSpecHash(desired) || !containerResourcesMatch(desired, actual) ||
+                (expected && !matches(expected.get(desired.name))) ||
+                (previous?.containerId && !matches(previous.containerId)) ||
+                Object.entries(desired.labels).some(([key, value]) => actual.labels?.[key] !== value)) {
+                throw new Error(`Infrastructure identity or specification changed during handoff: ${desired.name}`);
+            }
+            observations.push({ name: desired.name, containerId: actual.containerId.slice(0, 12) });
+        }
+        return observations;
+    }
     /**
      * Services whose socket is new since this reconcile began.
      *
@@ -358,9 +568,9 @@ export class InfrastructureService {
             .map(([name]) => name);
     }
     /**
-     * Stop all managed infrastructure containers.
+     * Release process-owned monitoring resources without changing infrastructure.
      */
-    async teardown() {
+    dispose() {
         this.stopHealthMonitor();
         // T#75: stop the janitor BEFORE removing containers so a final
         // tick can't race the teardown's `docker rm` and produce noisy
@@ -369,19 +579,89 @@ export class InfrastructureService {
             this.phantomJanitor.stop();
             this.phantomJanitor = null;
         }
+    }
+    /** Permanently release this entire observer, preserving every native resource. */
+    async relinquishMonitoring() {
+        if (this.monitoringRelinquished)
+            return [...this.relinquishedNames];
+        if (this.ownershipFrozen)
+            throw new Error('Infrastructure ownership is already frozen');
+        this.ownershipFrozen = true;
+        this.handoffMonitoring = false;
+        this.stopHealthMonitor();
+        const settled = await Promise.allSettled([
+            ...this.activeSweeps, ...this.reconcileTasks.values(), ...(this.provisioning ? [this.provisioning] : []),
+        ]);
+        // An already admitted provision may have created its janitor while being
+        // drained. Take the current reference only after that actor has completed.
+        await this.phantomJanitor?.drain();
+        // Rejected tasks are also finished. Their diagnostic remains in the state;
+        // relinquishing authority neither retries nor reverses their native work.
+        const failed = settled.filter(result => result.status === 'rejected').length;
+        if (failed)
+            this.logger.warn({ failed }, 'Infrastructure monitoring released after earlier reconciliation failures');
+        this.relinquishedNames = [...new Set([...this.desiredContainers, ...this.controlPlaneContainers()].map(item => item.name))];
+        this.monitoringRelinquished = true;
+        this.dispose();
+        this.janitorNetworks = [];
+        return [...this.relinquishedNames];
+    }
+    /** Stop reconciliation for exactly one allocation, retaining every other scope. */
+    async retireStack(scope) {
+        if (this.monitoringRelinquished)
+            throw new Error('Infrastructure monitoring ownership has been relinquished');
+        if (this.ownershipFrozen)
+            throw new Error('Infrastructure ownership is frozen');
+        const key = `${scope.project}/${scope.stack}`;
+        this.retiredStacks.add(key);
+        await Promise.allSettled([...this.activeSweeps, ...this.reconcileTasks.values()]);
+        this.desiredContainers = this.desiredContainers.filter(container => this.containerScope(container) !== key);
+    }
+    resumeStack(scope) {
+        if (this.monitoringRelinquished)
+            throw new Error('Infrastructure monitoring ownership has been relinquished');
+        if (this.ownershipFrozen)
+            throw new Error('Infrastructure ownership is frozen');
+        this.retiredStacks.delete(`${scope.project}/${scope.stack}`);
+    }
+    containerScope(container) {
+        return `${container.labels?.['omnitron.project']}/${container.labels?.['omnitron.stack']}`;
+    }
+    /** Exact names this local infrastructure owner would remove, without exposing credentials in container specs. */
+    getTeardownNames() {
+        return [...[...this.desiredContainers].reverse(), ...this.controlPlaneContainers()].map(container => container.name);
+    }
+    /** Explicit infrastructure removal; process disposal alone preserves containers. */
+    async teardown() {
+        if (this.monitoringRelinquished)
+            throw new Error('Infrastructure monitoring ownership has been relinquished');
+        if (this.ownershipFrozen)
+            throw new Error('Infrastructure ownership is frozen');
+        this.ownershipFrozen = true;
+        this.stopHealthMonitor();
+        await Promise.allSettled([...this.activeSweeps, ...this.reconcileTasks.values(), ...(this.provisioning ? [this.provisioning] : [])]);
+        await this.phantomJanitor?.drain();
+        this.dispose();
         // Remove all containers in parallel for fast shutdown — the control-plane
         // database only where this service keeps one (`controlPlaneContainers`).
-        const allContainers = [...[...this.desiredContainers].reverse(), ...this.controlPlaneContainers()];
-        await Promise.allSettled(allContainers.map(async (c) => {
-            try {
-                await removeContainer(c.name);
-                this.logger.info({ service: c.name }, 'Container removed');
-            }
-            catch {
-                // Already removed
-            }
+        const allContainers = [...this.desiredContainers, ...this.controlPlaneContainers()];
+        // Validate every target before removing the first. Absence is an idempotent success.
+        const reviewed = await Promise.all(allContainers.map(async (desired) => {
+            const actual = await getContainerState(desired.name, { orThrow: true, fullId: true });
+            return actual ? ownedContainerId(desired, actual) : null;
+        }));
+        const results = await Promise.allSettled(reviewed.map(async (id, index) => {
+            if (!id)
+                return;
+            await removeContainer(id, { graceful: true });
+            this.logger.info({ service: allContainers[index].name }, 'Container removed');
         }));
         this.state.ready = false;
+        const failures = results.flatMap((result, index) => result.status === 'rejected'
+            ? [new Error(`${allContainers[index].name}: ${String(result.reason)}`, { cause: result.reason })] : []);
+        if (failures.length) {
+            throw new AggregateError(failures, `Infrastructure stop incomplete: ${failures.map(error => error.message).join('; ')}`);
+        }
     }
     /**
      * Get current infrastructure state.
@@ -389,24 +669,34 @@ export class InfrastructureService {
     getState() {
         return this.state;
     }
-    /**
-     * Serve the stack's definition as it is now.
-     *
-     * A node's daemon builds this service on its first `provisionStack` and
-     * reuses it for every later one — rightly: a second service would be a
-     * second janitor and a second health monitor over one set of containers.
-     * But the definition it was built with stayed its definition. A later
-     * deployment that enabled a service, changed an image or dropped a
-     * `disabled` reached the node's host services — `reconcileHostServices`
-     * reads each call's own — and not its containers, until something restarted
-     * the daemon. Replaced whole, as the constructor sets it; `setConfigRoots`
-     * and `addAppContainers` then work on this set, as on a first call.
-     */
-    redefine(config, normalizedServices, serviceOverrides) {
-        this.config = config;
-        this.normalizedServices = normalizedServices;
-        this.serviceOverrides = serviceOverrides;
-        this.desiredContainers = resolveInfrastructure(config, normalizedServices, serviceOverrides);
+    /** Called only after older native work has drained and new observations are held. */
+    replaceDefinitionFrom(candidate) {
+        if (candidate.needsControlPlaneDatabase !== this.needsControlPlaneDatabase)
+            throw new Error('Infrastructure declaration cannot change control-plane database ownership');
+        const next = new Map(candidate.desiredContainers.map(container => [container.name, container]));
+        if (next.size !== candidate.desiredContainers.length)
+            throw new Error('Duplicate desired container identity');
+        for (const container of this.desiredContainers) {
+            const replacement = next.get(container.name);
+            if (replacement && !sameRecord(container.labels, replacement.labels))
+                throw new Error(`Infrastructure declaration cannot change retained ownership: ${container.name}`);
+        }
+        const previousServices = this.normalizedServices;
+        for (const container of this.desiredContainers) {
+            const replacement = next.get(container.name);
+            if (!replacement || containerSpecHash(container) !== containerSpecHash(replacement)) {
+                delete this.state.services[container.name];
+                this.unhealthyEvidence.delete(container.name);
+                this.restartBackoff.delete(container.name);
+                this.lastActions.delete(container.name);
+            }
+        }
+        this.config = candidate.config;
+        this.normalizedServices = candidate.normalizedServices;
+        this.serviceOverrides = candidate.serviceOverrides;
+        this.desiredContainers = [...candidate.desiredContainers];
+        this.invalidateChangedSetup(previousServices);
+        this.state.ready = summariseProvisioning(this.desiredContainers, this.state.services).ready;
     }
     /**
      * Get normalized services (from preset system).
@@ -427,6 +717,15 @@ export class InfrastructureService {
             return [];
         return Object.keys(pg.databases);
     }
+    /** The bucket inventory declared to the same MinIO provisioner. */
+    getMinioBuckets() {
+        const buckets = this.normalizedServices['minio']?._presetConfig?.['buckets'] ?? this.config.minio?.buckets;
+        if (buckets === undefined)
+            return [];
+        if (!Array.isArray(buckets) || buckets.some(bucket => typeof bucket !== 'string'))
+            throw new Error('Invalid declared MinIO bucket inventory');
+        return [...new Set(buckets)];
+    }
     /**
      * Resolved container name for an infra service (e.g. 'postgres' →
      * 'acme-dev-postgres'). Taken from the already-resolved desired containers
@@ -467,12 +766,16 @@ export class InfrastructureService {
             }
             case 'minio': {
                 const minio = this.config.minio;
-                if (!minio)
+                const name = this.getResolvedContainerName('minio');
+                const resolved = this.desiredContainers.find(container => container.name === name);
+                if (!resolved && !minio)
                     return null;
+                // Use the provisioner's resolved specification for both declaration forms.
+                // Reading only legacy `minio` loses credentials when `services.minio` overrides it.
                 return {
-                    endpoint: `http://${LOCAL_INFRA_HOST}:${minio.ports?.api ?? 9000}`,
-                    accessKey: minio.accessKey ?? 'minioadmin',
-                    secretKey: typeof minio.secretKey === 'string' ? minio.secretKey : 'minioadmin',
+                    endpoint: `http://${LOCAL_INFRA_HOST}:${resolved?.ports.find(port => port.container === 9000)?.host ?? minio?.ports?.api ?? 9000}`,
+                    accessKey: resolved?.environment['MINIO_ROOT_USER'] ?? minio?.accessKey ?? 'minioadmin',
+                    secretKey: resolved?.environment['MINIO_ROOT_PASSWORD'] ?? (typeof minio?.secretKey === 'string' ? minio.secretKey : 'minioadmin'),
                     forcePathStyle: true,
                 };
             }
@@ -502,6 +805,13 @@ export class InfrastructureService {
      * own restart-policy in the mix) occasional duplicate containers.
      */
     reconciling = new Set();
+    reconcileTasks = new Map();
+    activeSweeps = new Set();
+    ownershipFrozen = false;
+    definitionChanging = false;
+    monitoringRelinquished = false;
+    provisioning;
+    relinquishedNames = [];
     /**
      * Where this node keeps each service's config files, once a master sent them.
      *
@@ -534,22 +844,31 @@ export class InfrastructureService {
             return;
         this.desiredContainers = resolveInfrastructure(this.config, this.normalizedServices, this.serviceOverrides, roots, { redis, staticRoots, ...(staticLinkRoots && { staticLinkRoots }) });
     }
-    async reconcileService(desired) {
+    reconcileService(desired) {
+        return this.withContainerReconciliation(desired, () => this.reconcileServiceLocked(desired));
+    }
+    async withContainerReconciliation(desired, operation) {
         // T#58: per-name lock around the whole inspect→decide→act loop.
-        if (this.reconciling.has(desired.name)) {
+        if (this.ownershipFrozen || this.definitionChanging || this.retiredStacks.has(this.containerScope(desired)) || this.reconciling.has(desired.name)) {
             this.logger.debug({ service: desired.name }, 'Reconcile already in flight — skipping');
             return;
         }
         this.reconciling.add(desired.name);
         try {
-            await this.reconcileServiceLocked(desired);
+            const task = operation();
+            this.reconcileTasks.set(desired.name, task);
+            await task;
         }
         finally {
             this.reconciling.delete(desired.name);
+            this.reconcileTasks.delete(desired.name);
         }
     }
     async reconcileServiceLocked(desired) {
-        const actual = await getContainerState(desired.name);
+        const actual = await getContainerState(desired.name, { orThrow: true, fullId: true });
+        if (this.ownershipFrozen || this.definitionChanging || this.retiredStacks.has(this.containerScope(desired)))
+            return;
+        const id = actual && ownedContainerId(desired, actual);
         const action = this.computeAction(desired, actual);
         // A container that was created, started or recreated is listening on a
         // NEW socket. Applications already running hold pools pointing at the
@@ -568,12 +887,12 @@ export class InfrastructureService {
                 break;
             case 'start':
                 this.logger.info({ service: desired.name }, 'Starting existing service');
-                await startContainer(action.containerId);
-                await this.waitHealthy(desired.name);
+                await startContainer(id);
+                await this.waitHealthy(id);
                 break;
             case 'recreate':
                 this.logger.info({ service: desired.name, reason: action.reason }, 'Recreating service');
-                await removeContainer(desired.name);
+                await removeContainer(id);
                 await this.createAndStart(desired);
                 break;
             case 'remove':
@@ -583,15 +902,48 @@ export class InfrastructureService {
                 break;
         }
         // Update state
-        const newState = await getContainerState(desired.name);
+        const newState = await getContainerState(desired.name, { orThrow: true, fullId: true });
         if (newState) {
-            this.state.services[desired.name] = newState;
+            this.recordContainerObservation(desired, newState);
+        }
+    }
+    async recoverUnhealthyLocked(desired, actual, previous) {
+        // Recheck immediately before destruction: queued polls, recovery and
+        // replacement while inspecting must not authorize a stale decision.
+        const confirmed = await getContainerState(desired.name, { orThrow: true, fullId: true });
+        if (this.ownershipFrozen || this.definitionChanging || this.retiredStacks.has(this.containerScope(desired)))
+            return;
+        if (!confirmed || confirmed.containerId !== actual.containerId || confirmed.startedAt !== actual.startedAt ||
+            Date.parse(confirmed.healthCheck?.completedAt ?? '') < Date.parse(actual.healthCheck.completedAt) ||
+            !observeUnhealthy(previous, confirmed, Date.now()).restart) {
+            this.unhealthyEvidence.delete(desired.name);
+            this.logger.info({ service: desired.name }, 'Health recovery decision changed before restart; keeping the container');
+        }
+        else {
+            this.unhealthyEvidence.delete(desired.name);
+            const failures = (this.restartBackoff.get(desired.name)?.failures ?? 0) + 1;
+            const backoffMs = restartBackoffMs(failures);
+            // Count attempts until an observed healthy container recovers.
+            // A successful create alone cannot erase repeated health failure.
+            this.restartBackoff.set(desired.name, { failures, nextAttemptAt: Date.now() + backoffMs });
+            this.logger.warn({ service: desired.name, nativeCheck: confirmed.healthCheck, failures, backoffMs }, 'Confirmed sustained native health failure — restarting container');
+            try {
+                await removeContainer(ownedContainerId(desired, confirmed));
+                await this.createAndStart(desired);
+            }
+            catch (err) {
+                this.logger.error({ service: desired.name, error: describeError(err, 0, Object.values(desired.environment)), failures, backoffMs }, 'Health restart failed; recovery backoff retained');
+            }
         }
     }
     computeAction(desired, actual) {
+        containerResources(desired);
         if (!actual) {
             return { type: 'create', service: desired.name, config: desired };
         }
+        if (!containerResourcesMatch(desired, actual)) {
+            return { type: 'recreate', service: desired.name, config: desired, reason: 'resource limits differ from Docker inspection' };
+        }
         if (actual.status === 'running') {
             // Running but detached from all networks (OrbStack/dockerd-restart
             // artifact): the container is unreachable and its published ports are
@@ -638,7 +990,7 @@ export class InfrastructureService {
             // A container that is not running still carries the spec it was created
             // with, and `start` runs THAT spec. The drift checks above were only
             // asked of a running container, so one created wrong was started wrong
-            // for as long as it existed: Nominatim on daos/test, created at 05:37
+            // for as long as it existed: Nominatim on sample/test, created at 05:37
             // with a mount the master had not rewritten, was «started» by every
             // reconcile after the fix that corrected its desired spec — «exec:
             // /opt/nominatim-tools/cis-entrypoint.sh: no such file» each time
@@ -673,6 +1025,7 @@ export class InfrastructureService {
         };
     }
     async createAndStart(desired) {
+        containerResources(desired);
         // Ensure volumes exist
         for (const vol of desired.volumes) {
             if (!vol.source.startsWith('/') && !vol.source.startsWith('.')) {
@@ -690,41 +1043,30 @@ export class InfrastructureService {
             this.logger.info({ service: desired.name }, 'Non-critical service — skipping health wait');
         }
         else {
-            await this.waitHealthy(desired.name);
+            await this.waitHealthy(id);
         }
     }
     async waitHealthy(name) {
         // Poll Docker health via async CLI — avoids blocking event loop
-        const { exec } = await import('node:child_process');
+        const { execFile } = await import('node:child_process');
         const { promisify } = await import('node:util');
-        const execAsync = promisify(exec);
+        const inspect = promisify(execFile);
         const startTime = Date.now();
-        const pollInterval = 2000;
         while (Date.now() - startTime < STARTUP_TIMEOUT) {
             try {
-                const { stdout } = await execAsync(`docker inspect --format='{{.State.Health.Status}}' ${name} 2>/dev/null || echo "none"`, { encoding: 'utf-8', timeout: 5000 });
-                const output = (stdout ?? '').trim().replace(/'/g, '');
-                if (output === 'healthy') {
-                    this.logger.info({ service: name, elapsed: Date.now() - startTime }, 'Service healthy');
+                const { stdout } = await inspect('docker', ['inspect', '--format', '{{json .State}}', '--', name], { encoding: 'utf8', timeout: 5000 });
+                const state = JSON.parse(stdout);
+                if (state.Health?.Status === 'healthy' || (state.Status === 'running' && (!state.Health || state.Health.Status === 'none')))
+                    return;
+                if (state.Status === 'exited' || state.Status === 'dead') {
+                    this.logger.warn({ service: name, status: state.Status }, 'Container not running');
                     return;
-                }
-                if (output === 'none' || output === '') {
-                    const { stdout: statusOut } = await execAsync(`docker inspect --format='{{.State.Status}}' ${name} 2>/dev/null || echo "missing"`, { encoding: 'utf-8', timeout: 5000 });
-                    const status = (statusOut ?? '').trim().replace(/'/g, '');
-                    if (status === 'running') {
-                        this.logger.info({ service: name, elapsed: Date.now() - startTime }, 'Service running (no health check)');
-                        return;
-                    }
-                    if (status === 'exited' || status === 'dead' || status === 'missing') {
-                        this.logger.warn({ service: name, status }, 'Container not running');
-                        return;
-                    }
                 }
             }
             catch {
                 // docker CLI failed — retry
             }
-            await new Promise((r) => setTimeout(r, pollInterval));
+            await new Promise((r) => setTimeout(r, 2000));
         }
         this.logger.warn({ service: name, timeout: STARTUP_TIMEOUT }, 'Service did not become healthy within timeout');
     }
@@ -738,41 +1080,40 @@ export class InfrastructureService {
      * Generic post-provision: iterate all services, call preset hooks.
      * Replaces hardcoded createPostgresDatabases() / createMinioBuckets().
      */
-    async runPresetPostProvisionHooks() {
+    requiresPostProvision(desired) {
+        const requirement = this.normalizedServices[desired.labels['omnitron.service'] ?? ''];
+        if (!requirement?._preset)
+            return false;
+        const preset = this.presetRegistry?.get(requirement._preset);
+        return !preset || Boolean(preset.postProvision);
+    }
+    recordContainerObservation(desired, actual) {
+        this.state.services[desired.name] = retainProvisioningObservation(actual, this.state.services[desired.name], this.requiresPostProvision(desired));
+    }
+    invalidateChangedSetup(previous) {
+        for (const container of this.desiredContainers) {
+            const name = container.labels['omnitron.service'] ?? '';
+            const state = this.state.services[container.name];
+            if (state && this.requiresPostProvision(container) && !isDeepStrictEqual(previous[name], this.normalizedServices[name])) {
+                state.provisioning = 'pending';
+                state.error = 'Declared service setup changed and must be verified';
+                this.state.ready = false;
+            }
+        }
+    }
+    async runPresetPostProvisionHooks(only) {
         const { execFile } = await import('node:child_process');
         const { promisify } = await import('node:util');
         const execFileAsync = promisify(execFile);
-        // A preset hook is what creates the databases inside Postgres and the
-        // buckets inside MinIO — the containers are running either way, so a
-        // failure here is invisible until an application cannot find its
-        // database.
-        //
-        // `this.presetRegistry!` asserted a dependency the constructor declares
-        // OPTIONAL, so a caller that did not pass one — which the type permits —
-        // got `Cannot read properties of undefined (reading 'get')` from a line
-        // that reads as internal, after every container was already up.
-        // Measured on a node provisioning a stack, where the containers stayed
-        // and the databases were never made.
-        if (!this.presetRegistry) {
-            const withPresets = Object.entries(this.normalizedServices)
-                .filter(([, r]) => r._preset)
-                .map(([name]) => name);
-            if (withPresets.length > 0) {
-                this.logger.error({ services: withPresets }, 'No preset registry, so post-provision steps were skipped — these services are running but their databases and buckets were not created');
-            }
-            return;
-        }
         for (const [serviceName, requirement] of Object.entries(this.normalizedServices)) {
             if (!requirement._preset)
                 continue;
-            const preset = this.presetRegistry.get(requirement._preset);
-            if (!preset?.postProvision)
+            const preset = this.presetRegistry?.get(requirement._preset);
+            if (preset && !preset.postProvision)
                 continue;
-            const container = this.desiredContainers.find((c) => c.name.endsWith(`-${serviceName}`));
-            if (!container)
+            const container = this.desiredContainers.find((c) => c.labels['omnitron.service'] === serviceName);
+            if (!container || (only && container.name !== only.name))
                 continue;
-            const cName = container.name;
-            await waitForHealthy(cName, 30_000);
             // Resolve secrets from requirement
             const secrets = {};
             if (requirement.secrets) {
@@ -780,24 +1121,47 @@ export class InfrastructureService {
                     secrets[k] = typeof v === 'string' ? v : '';
                 }
             }
-            const execInCont = async (command) => {
-                const { stdout } = await execFileAsync('docker', ['exec', cName, ...command], {
-                    encoding: 'utf-8',
-                    timeout: 10_000,
-                });
-                return (stdout ?? '').trim();
-            };
+            const safeError = (error) => describeError(error, 0, Object.values(secrets));
             try {
+                const actual = await getContainerState(container.name, { orThrow: true, fullId: true });
+                if (!actual)
+                    throw new Error('Declared service container is absent');
+                const cName = ownedContainerId(container, actual);
+                this.state.services[container.name] = { ...actual, provisioning: 'pending', error: 'Declared service setup is in progress' };
+                if (!preset)
+                    throw new Error(`Preset '${requirement._preset}' is unavailable; declared service setup cannot be verified`);
+                if (!await waitForHealthy(cName, 30_000))
+                    throw new Error('Container did not become healthy before declared service setup');
                 await preset.postProvision({
-                    containerName: cName,
+                    containerName: container.name,
                     userConfig: requirement._presetConfig ?? {},
                     secrets,
-                    execInContainer: execInCont,
+                    execInContainer: async (command) => {
+                        try {
+                            const { stdout } = await execFileAsync('docker', ['exec', cName, ...command], { encoding: 'utf-8', timeout: 10_000 });
+                            return (stdout ?? '').trim();
+                        }
+                        catch (error) {
+                            // Raw subprocess causes include credential arguments. Preserve only the sanitized cause.
+                            // eslint-disable-next-line preserve-caught-error
+                            throw new Error('Declared service setup command failed', { cause: safeError(error) });
+                        }
+                    },
                     logger: this.logger,
                 });
+                const confirmed = await getContainerState(container.name, { orThrow: true, fullId: true });
+                if (!confirmed || confirmed.status !== 'running' || confirmed.health === 'unhealthy' ||
+                    confirmed.containerId !== actual.containerId || confirmed.startedAt !== actual.startedAt)
+                    throw new Error('Container generation changed during declared service setup');
+                this.state.services[container.name] = { ...confirmed, provisioning: 'ready' };
             }
             catch (err) {
-                this.logger.error({ service: serviceName, preset: requirement._preset, error: err.message }, 'Post-provision hook failed');
+                const error = `Post-provisioning '${serviceName}': ${safeError(err)}`;
+                this.state.services[container.name] = {
+                    ...(this.state.services[container.name] ?? { name: container.name, image: container.image, status: 'not_found' }),
+                    provisioning: 'failed', error,
+                };
+                this.logger.error({ service: serviceName, preset: requirement._preset, error }, 'Post-provision hook failed');
             }
         }
     }
@@ -838,6 +1202,8 @@ export class InfrastructureService {
     // Private — Health Monitor (Self-Healing)
     // ===========================================================================
     startHealthMonitor() {
+        if (this.monitoringRelinquished || this.ownershipFrozen || this.healthTimer)
+            return;
         this.healthTimer = setInterval(() => {
             this.healthSweep().catch((err) => {
                 this.logger.error({ error: err.message }, 'Health sweep failed');
@@ -858,7 +1224,7 @@ export class InfrastructureService {
      *
      * `provision()` asked `needsControlPlaneDatabase`; the health sweep and the
      * teardown asked only `usingGlobalOmnitronPg`. So on a node the sweep found
-     * `<prefix>-pg` «not running» every 30 s and created it: daos/test got one on
+     * `<prefix>-pg` «not running» every 30 s and created it: sample/test got one on
      * 2026-09-20, and again on 2026-09-25 forty seconds after it was removed —
      * on the default credentials, for a database nothing reads.
      */
@@ -866,11 +1232,27 @@ export class InfrastructureService {
         return this.needsControlPlaneDatabase && !this.usingGlobalOmnitronPg ? [this.omnitronPgContainer] : [];
     }
     async healthSweep() {
+        if (this.ownershipFrozen || this.provisioning || this.activeSweeps.size)
+            return;
+        const task = this.healthSweepOwned();
+        this.activeSweeps.add(task);
+        try {
+            await task;
+        }
+        finally {
+            this.activeSweeps.delete(task);
+        }
+    }
+    async healthSweepOwned() {
         // The control-plane database alongside the app containers, where this
         // service keeps one.
         const allContainers = [...this.controlPlaneContainers(), ...this.desiredContainers];
         for (const desired of allContainers) {
-            const actual = await getContainerState(desired.name);
+            const actual = await getContainerState(desired.name, { orThrow: true, fullId: true });
+            if (this.ownershipFrozen || this.definitionChanging)
+                return;
+            if (this.retiredStacks.has(this.containerScope(desired)))
+                continue;
             // T#58: Docker's own `--restart=always` / `--restart=on-failure`
             // policy puts the container into the 'restarting' state for the
             // brief window between exit and re-entry. If we recreate during
@@ -879,11 +1261,12 @@ export class InfrastructureService {
             // may end up duplicated under a renamed transient. Let Docker
             // finish; the next 30s tick will see 'running' and skip.
             if (actual?.status === 'restarting') {
+                this.unhealthyEvidence.delete(desired.name);
                 this.logger.debug({ service: desired.name }, "Service is restarting via Docker's own policy — yielding to it (T#58)");
                 continue;
             }
             if (!actual || actual.status !== 'running') {
-                this.unhealthyTicks.delete(desired.name);
+                this.unhealthyEvidence.delete(desired.name);
                 // Circuit-breaker: while a service is inside its backoff window, skip
                 // the restart so a permanently-failing container can't hammer the
                 // supervisor every tick and starve the daemon's event loop.
@@ -895,7 +1278,6 @@ export class InfrastructureService {
                 this.logger.warn({ service: desired.name, status: actual?.status ?? 'not_found' }, 'Service not running — auto-restarting');
                 try {
                     await this.reconcileService(desired);
-                    this.restartBackoff.delete(desired.name);
                 }
                 catch (err) {
                     const failures = (this.restartBackoff.get(desired.name)?.failures ?? 0) + 1;
@@ -905,65 +1287,109 @@ export class InfrastructureService {
                 }
             }
             else if (actual.health === 'unhealthy') {
-                const ticks = (this.unhealthyTicks.get(desired.name) ?? 0) + 1;
-                this.unhealthyTicks.set(desired.name, ticks);
-                if (ticks < UNHEALTHY_RESTART_THRESHOLD) {
-                    this.logger.warn({ service: desired.name, consecutiveUnhealthy: ticks, threshold: UNHEALTHY_RESTART_THRESHOLD }, 'Service unhealthy — waiting for confirmation before restart');
+                const previous = this.unhealthyEvidence.get(desired.name);
+                const observation = observeUnhealthy(previous, actual, Date.now());
+                if (observation.evidence)
+                    this.unhealthyEvidence.set(desired.name, observation.evidence);
+                else
+                    this.unhealthyEvidence.delete(desired.name);
+                const backoff = this.restartBackoff.get(desired.name);
+                if (backoff && Date.now() < backoff.nextAttemptAt) {
+                    this.logger.debug({ service: desired.name, retryInMs: backoff.nextAttemptAt - Date.now() }, 'Service unhealthy — recovery backoff remains active');
+                }
+                else if (!observation.restart) {
+                    this.logger.warn({ service: desired.name, reason: observation.reason, nativeCheck: actual.healthCheck }, 'Service unhealthy — waiting for confirmed native failure');
                 }
                 else {
-                    this.unhealthyTicks.delete(desired.name);
-                    this.logger.warn({ service: desired.name, consecutiveUnhealthy: ticks }, 'Service unhealthy past threshold — restarting container');
-                    try {
-                        await removeContainer(desired.name);
-                        await this.createAndStart(desired);
-                    }
-                    catch (err) {
-                        this.logger.error({ service: desired.name, error: err.message }, 'Health restart failed');
-                    }
+                    await this.withContainerReconciliation(desired, () => this.recoverUnhealthyLocked(desired, actual, previous));
                 }
             }
             else {
-                // Healthy or starting — clear any accumulated unhealthy ticks and
-                // reset the restart circuit-breaker (it may have recovered on its own
-                // via Docker's restart policy).
-                this.unhealthyTicks.delete(desired.name);
-                this.restartBackoff.delete(desired.name);
+                // A starting container is not evidence of recovery. Retain backoff
+                // until a running container is healthy (or declares no native probe).
+                this.unhealthyEvidence.delete(desired.name);
+                if ((actual.health === 'healthy' || actual.health === 'none') && this.state.services[desired.name]?.provisioning !== 'failed')
+                    this.restartBackoff.delete(desired.name);
             }
             // Update state
-            const newState = await getContainerState(desired.name);
+            const newState = await getContainerState(desired.name, { orThrow: true, fullId: true });
             if (newState) {
-                this.state.services[desired.name] = newState;
+                this.recordContainerObservation(desired, newState);
+                const state = this.state.services[desired.name];
+                if (state.status === 'running' && state.health !== 'unhealthy' &&
+                    (state.provisioning === 'pending' || state.provisioning === 'failed') &&
+                    Date.now() >= (this.restartBackoff.get(desired.name)?.nextAttemptAt ?? 0)) {
+                    await this.runPresetPostProvisionHooks(desired);
+                    if (this.state.services[desired.name]?.provisioning === 'ready') {
+                        if (this.state.services[desired.name]?.health === 'healthy' || this.state.services[desired.name]?.health === 'none')
+                            this.restartBackoff.delete(desired.name);
+                    }
+                    else {
+                        const failures = (this.restartBackoff.get(desired.name)?.failures ?? 0) + 1;
+                        this.restartBackoff.set(desired.name, { failures, nextAttemptAt: Date.now() + restartBackoffMs(failures) });
+                    }
+                }
             }
         }
+        this.state.ready = summariseProvisioning(this.desiredContainers, this.state.services).ready;
         this.state.lastReconciled = new Date().toISOString();
     }
 }
-/**
- * T#76: shallow equality on the fields that drive container
- * identity. Two declarations that differ only in incidental fields
- * (label maps, restart policy adornments) should still be treated
- * as the same shared container; differences in image / env / port
- * mapping legitimately deserve an operator-facing warning.
- */
-function shallowContainerEqual(a, b) {
-    if (a.image !== b.image)
-        return false;
-    if (!sameRecord(a.environment, b.environment))
-        return false;
-    if (a.ports.length !== b.ports.length)
-        return false;
-    for (let i = 0; i < a.ports.length; i++) {
-        if (a.ports[i].host !== b.ports[i].host || a.ports[i].container !== b.ports[i].container)
-            return false;
-    }
-    if (a.volumes.length !== b.volumes.length)
-        return false;
-    for (let i = 0; i < a.volumes.length; i++) {
-        if (a.volumes[i].source !== b.volumes[i].source || a.volumes[i].target !== b.volumes[i].target)
-            return false;
-    }
-    return true;
-}
+__decorate([
+    RecoveryMutation(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Array]),
+    __metadata("design:returntype", void 0)
+], InfrastructureService.prototype, "addAppContainers", null);
+__decorate([
+    RecoveryMutation(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [InfrastructureService]),
+    __metadata("design:returntype", Promise)
+], InfrastructureService.prototype, "provision", null);
+__decorate([
+    RecoveryMutation(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [InfrastructureService]),
+    __metadata("design:returntype", Promise)
+], InfrastructureService.prototype, "refreshOwnershipFrom", null);
+__decorate([
+    RecoveryMutation(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", []),
+    __metadata("design:returntype", Promise)
+], InfrastructureService.prototype, "relinquishMonitoring", null);
+__decorate([
+    RecoveryMutation(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], InfrastructureService.prototype, "retireStack", null);
+__decorate([
+    RecoveryMutation(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", []),
+    __metadata("design:returntype", Promise)
+], InfrastructureService.prototype, "teardown", null);
+__decorate([
+    RecoveryMutation(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Map,
+        Map, Object, Map]),
+    __metadata("design:returntype", void 0)
+], InfrastructureService.prototype, "setConfigRoots", null);
+__decorate([
+    RecoveryMutation(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], InfrastructureService.prototype, "reconcileServiceLocked", null);
+__decorate([
+    RecoveryMutation(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object, Object, Object]),
+    __metadata("design:returntype", Promise)
+], InfrastructureService.prototype, "recoverUnhealthyLocked", null);
 function sameRecord(a, b) {
     const ka = Object.keys(a ?? {});
     const kb = Object.keys(b ?? {});