@omnitron-dev/omnitron 0.1.3 → 0.3.0

package/dist/infrastructure/infrastructure.service.d.ts

@@ -17,20 +17,150 @@
  */
 import type { ILogger } from '@omnitron-dev/titan/module/logger';
 import type { InfrastructureConfig, InfrastructureState, ResolvedContainer } from './types.js';
+/**
+ * Restart-circuit-breaker backoff schedule: the delay (ms) to wait before the
+ * next restart attempt after `failures` consecutive failed (re)creates.
+ * Exponential from RESTART_BACKOFF_BASE, capped at RESTART_BACKOFF_MAX so a
+ * permanently-broken service settles into an occasional retry instead of
+ * hammering the supervisor every 30s.
+ *
+ * Delegates to the shared titan `computeBackoff` (the same helper the downstream
+ * backends and the titan packages use) so the exponential shape lives in ONE
+ * place. Exported for tests.
+ */
+export declare function restartBackoffMs(failures: number): number;
+/**
+ * What to do about the control plane's own database.
+ *
+ * Three cases that look alike and are not:
+ *
+ *   - a node's daemon keeps its state in SQLite and needs no database at all;
+ *   - one is running, so it gets the same reconcile — and so the same
+ *     `containerSpecHash` drift check — as every app service;
+ *   - nothing is running, so provision.
+ *
+ * The middle case is the one that was missing. "Already running" was read as
+ * "configured as declared", and the database was adopted exactly as it stood.
+ * Measured: `omnitron-pg`, created before published ports were bound to
+ * loopback, survived on 0.0.0.0:5480 through the fix that reached all eleven
+ * other managed containers, and still answered `nc -vz <lan-address> 5480`
+ * from another machine months later, while `daos-dev-postgres` beside it in
+ * `docker ps` was correctly on 127.0.0.1.
+ *
+ * `reconcileName` is the subtlety. The control-plane database is ONE
+ * container per host — it owns port 5480 — but each daemon resolves a name
+ * carrying its own project prefix, so a daemon on the `daos/dev` stack
+ * computes `daos-dev-pg` while the container that exists is `omnitron-pg`.
+ * Reconciling the running container against a spec under the OTHER name
+ * creates a second one and collides on the port; the first version of this
+ * did exactly that and corrected nothing. The name to reconcile is therefore
+ * the name of the container that is actually there, and the spec is
+ * everything else. `containerSpecHash` does not cover the name, so the drift
+ * comparison is unaffected either way.
+ *
+ * A differing name is not evidence of another owner: only omnitron creates
+ * this container, and it carries omnitron's own `omnitron.internal` label.
+ */
+export declare function decideControlPlaneDatabase(input: {
+    needsControlPlaneDatabase: boolean;
+    running: {
+        name?: string;
+        status?: string;
+        specHash?: string | undefined;
+    } | null;
+    desiredName: string;
+}): {
+    action: 'skip' | 'provision';
+} | {
+    action: 'reconcile';
+    reconcileName: string;
+};
 export declare class InfrastructureService {
     private readonly logger;
-    private readonly config;
+    private config;
     private readonly presetRegistry?;
-    private readonly desiredContainers;
+    /**
+     * Stack overrides. A service marked `disabled` here is not provisioned,
+     * and one pointed at an `external` address is not provisioned either —
+     * there is nothing to create, and the address reaches the application
+     * through its environment instead.
+     */
+    private serviceOverrides?;
+    /**
+     * Whether this daemon stores its own state in Postgres.
+     *
+     * True for a master, which is where projects, nodes, metrics and logs
+     * live. False for a node: its daemon keeps that in SQLite and reads a
+     * Postgres never.
+     */
+    private readonly needsControlPlaneDatabase;
+    private desiredContainers;
+    /** What the most recent reconcile did to each service. */
+    private readonly lastActions;
     private readonly omnitronPgContainer;
     private usingGlobalOmnitronPg;
     private healthTimer;
     private readonly state;
-    private readonly normalizedServices;
-    constructor(logger: ILogger, config: InfrastructureConfig, normalizedServices?: Record<string, import('./types.js').IServiceRequirement>, presetRegistry?: import("./presets/registry.js").PresetRegistry | undefined);
+    private normalizedServices;
+    /**
+     * Per-service consecutive-unhealthy counter. Restart only fires after
+     * UNHEALTHY_RESTART_THRESHOLD ticks in a row to avoid flapping during
+     * cold-start / transient health-check timeouts. Reset to 0 on a healthy
+     * observation. Without this, a service that briefly fails one health
+     * probe (network blip, slow query) would be torn down and recreated,
+     * making transient pressure look like outages.
+     */
+    private unhealthyTicks;
+    /**
+     * Per-service restart circuit-breaker. A container whose (re)create keeps
+     * failing — an image that won't pull, or infra stuck on an unrecoverable
+     * error — must NOT be retried every 30s forever: that churn (docker calls +
+     * error logging) is what wedged the daemon's event loop and let an external
+     * "socket unreachable → SIGTERM" kill the whole daemon during the 2026-05
+     * outage. After each failed restart we back off exponentially (30s → 1m →
+     * 2m … capped at 15m); a successful restart or an externally-recovered
+     * 'running' observation resets it.
+     */
+    private restartBackoff;
+    /**
+     * T#75: phantom-endpoint janitor instance. Created in `provision()`
+     * once we know Docker is reachable; torn down in `teardown()`.
+     */
+    private phantomJanitor;
+    constructor(logger: ILogger, config: InfrastructureConfig, normalizedServices?: Record<string, import('./types.js').IServiceRequirement>, presetRegistry?: import('./presets/registry.js').PresetRegistry | undefined, 
+    /**
+     * Stack overrides. A service marked `disabled` here is not provisioned,
+     * and one pointed at an `external` address is not provisioned either —
+     * there is nothing to create, and the address reaches the application
+     * through its environment instead.
+     */
+    serviceOverrides?: Record<string, import('./types.js').IServiceOverride> | undefined, 
+    /**
+     * Whether this daemon stores its own state in Postgres.
+     *
+     * True for a master, which is where projects, nodes, metrics and logs
+     * live. False for a node: its daemon keeps that in SQLite and reads a
+     * Postgres never.
+     */
+    needsControlPlaneDatabase?: boolean);
     /**
      * Add app-declared infrastructure containers to the provisioning list.
      * Called by StackInfrastructureManager after resolving app requirements.
+     *
+     * T#76: dedupe by container name. The `${CONTAINER_PREFIX}-${service}`
+     * naming convention means two apps that BOTH declare a `postgres`
+     * requirement produce two identical-named entries. Pre-T#76, the
+     * provisioning loop tried to create the container twice; Docker's
+     * name-uniqueness check failed the second create with "name already
+     * in use", leaving the second app's perspective on infrastructure
+     * broken (it saw a failure that, from the cluster's view, was a
+     * benign collision — the shared service was actually fine).
+     *
+     * The intended semantics for shared infrastructure is "all apps that
+     * declare `postgres` use the same container", so the first
+     * declaration wins. When subsequent declarations differ in shape
+     * (image / env / ports), log a structured warning so the operator
+     * can reconcile intent vs. config.
      */
     addAppContainers(containers: ResolvedContainer[]): void;
     /**
@@ -39,6 +169,24 @@ export declare class InfrastructureService {
      * runs post-provisioning setup, and starts the health monitor.
      */
     provision(): Promise<InfrastructureState>;
+    /**
+     * What this service was asked to provide.
+     *
+     * Exposed because a caller cannot tell "present and fine" from "never
+     * attempted" by reading `state.services` alone: the second leaves no entry,
+     * and an absent entry is indistinguishable from an absent problem.
+     */
+    getDesiredServices(): ReadonlyArray<{
+        name: string;
+        image?: string | undefined;
+    }>;
+    /**
+     * Services whose socket is new since this reconcile began.
+     *
+     * `noop` is the only action that leaves an existing connection usable.
+     * Everything else means anything already connected is holding a dead pool.
+     */
+    getRestartedServices(): string[];
     /**
      * Stop all managed infrastructure containers.
      */
@@ -47,17 +195,90 @@ export declare class InfrastructureService {
      * Get current infrastructure state.
      */
     getState(): InfrastructureState;
+    /**
+     * Serve the stack's definition as it is now.
+     *
+     * A node's daemon builds this service on its first `provisionStack` and
+     * reuses it for every later one — rightly: a second service would be a
+     * second janitor and a second health monitor over one set of containers.
+     * But the definition it was built with stayed its definition. A later
+     * deployment that enabled a service, changed an image or dropped a
+     * `disabled` reached the node's host services — `reconcileHostServices`
+     * reads each call's own — and not its containers, until something restarted
+     * the daemon. Replaced whole, as the constructor sets it; `setConfigRoots`
+     * and `addAppContainers` then work on this set, as on a first call.
+     */
+    redefine(config: InfrastructureConfig, normalizedServices: Record<string, import('./types.js').IServiceRequirement>, serviceOverrides?: Record<string, import('./types.js').IServiceOverride>): void;
     /**
      * Get normalized services (from preset system).
      * Used by config-resolver for generic address resolution.
      */
     getNormalizedServices(): Record<string, import('./types.js').IServiceRequirement>;
+    /**
+     * Databases declared for this stack's postgres (e.g. main, storage, geo).
+     * Source of truth for "which DBs must be backed up" — read straight from
+     * the same `infrastructure.postgres.databases` block omnitron provisions
+     * from, so the backup set can never drift from the provisioned set.
+     */
+    getPostgresDatabases(): string[];
+    /**
+     * Resolved container name for an infra service (e.g. 'postgres' →
+     * 'acme-dev-postgres'). Taken from the already-resolved desired containers
+     * so it reflects the real container prefix rather than re-deriving it from
+     * the mutable module-global. Returns null if the stack has no such service.
+     */
+    getResolvedContainerName(service: string): string | null;
     /**
      * Get connection info for an infrastructure service.
      * Used by config generator to resolve { infra: 'postgres', database: 'main' }.
      */
     getConnectionInfo(service: string): Record<string, unknown> | null;
+    /**
+     * Per-name in-flight reconcile lock (T#58). A health sweep that
+     * overlaps a previous slow sweep — or a reconcile triggered by
+     * `ensureXxx` calls happening concurrently — used to enter
+     * `reconcileService` twice for the same container. The first did
+     * `removeContainer` then `createAndStart`; the second saw the
+     * container missing mid-recreate and tried to recreate it too,
+     * producing name conflicts, partial state, and (with Docker's
+     * own restart-policy in the mix) occasional duplicate containers.
+     */
+    private readonly reconciling;
+    /**
+     * Where this node keeps each service's config files, once a master sent them.
+     *
+     * The gateway is not an ordinary preset container — it needs four bind
+     * mounts, an entrypoint and fifteen upstream variables, and `resolveGateway`
+     * is the only thing that produces them. The constructor resolved everything
+     * through the generic preset path, because at construction time this node
+     * had no files and no local path to mount from.
+     *
+     * So this RE-RESOLVES rather than patching the volumes of a container built
+     * without them. A spec assembled by the wrong resolver is missing more than
+     * paths, and rewriting one field of it would produce a container that looks
+     * configured and is not — which is the failure that was already shipping,
+     * one level down.
+     *
+     * Called before `provision()`, so the recreation the changed spec hash
+     * implies happens on the first pass rather than the second.
+     */
+    setConfigRoots(roots: Map<string, string>, staticRoots: Map<string, string>, redis: {
+        host: string;
+        port: number;
+        db: number;
+        password?: string;
+    }, 
+    /**
+     * The static ROOT per service, when the delivery made a
+     * `current-<project>-<stack>` link inside it. Given, the gateway mounts
+     * the root and serves through the link — a path that does not change from
+     * one release to the next, so the container is not recreated and the
+     * second of 000 a deployment cost goes with it. Empty, everything below
+     * resolves exactly as before.
+     */
+    staticLinkRoots?: Map<string, string>): void;
     private reconcileService;
+    private reconcileServiceLocked;
     private computeAction;
     private createAndStart;
     private waitHealthy;
@@ -73,12 +294,32 @@ export declare class InfrastructureService {
      */
     private provisionOmnitronDatabase;
     /**
-     * Connect to Omnitron's PG and run migrations via Kysely's built-in Migrator.
-     * Uses dynamic import to keep pg/kysely optional until actually needed.
+     * Apply the internal-database schema through the shared runner.
+     *
+     * This used to be a second, independently-maintained copy of the daemon's
+     * migration code that only knew migrations 001–002 — and, like the
+     * daemon's copy, it constructed `Migrator` from the `kysely` package root
+     * where that export does not exist, so it threw on every run. Both are now
+     * one call into `database/migration-runner.ts`.
+     *
+     * Errors propagate to `provisionOmnitronDatabase`'s caller: provisioning
+     * that ends with an unmigrated database has not succeeded.
      */
     private runOmnitronMigrations;
     private startHealthMonitor;
     private stopHealthMonitor;
+    /**
+     * The control-plane database container this service keeps, if any: none on
+     * a node — it keeps its own state in SQLite (`decideControlPlaneDatabase`) —
+     * and none where the global `omnitron-pg` serves.
+     *
+     * `provision()` asked `needsControlPlaneDatabase`; the health sweep and the
+     * teardown asked only `usingGlobalOmnitronPg`. So on a node the sweep found
+     * `<prefix>-pg` «not running» every 30 s and created it: daos/test got one on
+     * 2026-09-20, and again on 2026-09-25 forty seconds after it was removed —
+     * on the default credentials, for a database nothing reads.
+     */
+    private controlPlaneContainers;
     private healthSweep;
 }
 //# sourceMappingURL=infrastructure.service.d.ts.map

package/dist/infrastructure/infrastructure.service.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"infrastructure.service.d.ts","sourceRoot":"","sources":["../../src/infrastructure/infrastructure.service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,mCAAmC,CAAC;AACjE,OAAO,KAAK,EACV,oBAAoB,EACpB,mBAAmB,EAEnB,iBAAiB,EAElB,MAAM,YAAY,CAAC;AAsBpB,qBAAa,qBAAqB;IAS9B,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,MAAM;IAEvB,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;IAXlC,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAA2B;IAC7D,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAoB;IACxD,OAAO,CAAC,qBAAqB,CAAS;IACtC,OAAO,CAAC,WAAW,CAA+B;IAClD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAuD;IAC7E,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAA2D;gBAG3E,MAAM,EAAE,OAAO,EACf,MAAM,EAAE,oBAAoB,EAC7C,kBAAkB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,YAAY,EAAE,mBAAmB,CAAC,EAC5D,cAAc,CAAC,EAAE,OAAO,uBAAuB,EAAE,cAAc,YAAA;IAOlF;;;OAGG;IACH,gBAAgB,CAAC,UAAU,EAAE,iBAAiB,EAAE,GAAG,IAAI;IAIvD;;;;OAIG;IACG,SAAS,IAAI,OAAO,CAAC,mBAAmB,CAAC;IAyF/C;;OAEG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAuB/B;;OAEG;IACH,QAAQ,IAAI,mBAAmB;IAI/B;;;OAGG;IACH,qBAAqB,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,YAAY,EAAE,mBAAmB,CAAC;IAIjF;;;OAGG;IACH,iBAAiB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;YAiDpD,gBAAgB;IA4C9B,OAAO,CAAC,aAAa;YAkCP,cAAc;YAwBd,WAAW;YAgDX,aAAa;IAI3B;;;OAGG;YACW,2BAA2B;IAsDzC;;;OAGG;YACW,yBAAyB;IAevC;;;OAGG;YACW,qBAAqB;IAuDnC,OAAO,CAAC,kBAAkB;IAS1B,OAAO,CAAC,iBAAiB;YAOX,WAAW;CA+C1B"}
+{"version":3,"file":"infrastructure.service.d.ts","sourceRoot":"","sources":["../../src/infrastructure/infrastructure.service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,mCAAmC,CAAC;AAEjE,OAAO,KAAK,EACV,oBAAoB,EACpB,mBAAmB,EAEnB,iBAAiB,EAElB,MAAM,YAAY,CAAC;AAgCpB;;;;;;;;;;GAUG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAGzD;AAMD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,wBAAgB,0BAA0B,CAAC,KAAK,EAAE;IAChD,yBAAyB,EAAE,OAAO,CAAC;IACnC,OAAO,EAAE;QAAE,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;KAAE,GAAG,IAAI,CAAC;IAClF,WAAW,EAAE,MAAM,CAAC;CACrB,GAAG;IAAE,MAAM,EAAE,MAAM,GAAG,WAAW,CAAA;CAAE,GAAG;IAAE,MAAM,EAAE,WAAW,CAAC;IAAC,aAAa,EAAE,MAAM,CAAA;CAAE,CAIpF;AAED,qBAAa,qBAAqB;IAuC9B,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,MAAM;IAEd,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;IAChC;;;;;OAKG;IACH,OAAO,CAAC,gBAAgB,CAAC;IACzB;;;;;;OAMG;IACH,OAAO,CAAC,QAAQ,CAAC,yBAAyB;IAxD5C,OAAO,CAAC,iBAAiB,CAA2B;IACpD,0DAA0D;IAC1D,OAAO,CAAC,QAAQ,CAAC,WAAW,CAA6B;IACzD,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAoB;IACxD,OAAO,CAAC,qBAAqB,CAAS;IACtC,OAAO,CAAC,WAAW,CAA+B;IAClD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAuD;IAC7E,OAAO,CAAC,kBAAkB,CAA2D;IAErF;;;;;;;OAOG;IACH,OAAO,CAAC,cAAc,CAA6B;IAEnD;;;;;;;;;OASG;IACH,OAAO,CAAC,cAAc,CAAkE;IAExF;;;OAGG;IACH,OAAO,CAAC,cAAc,CAAuC;IAE7D,YACmB,MAAM,EAAE,OAAO,EACxB,MAAM,EAAE,oBAAoB,EACpC,kBAAkB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,YAAY,EAAE,mBAAmB,CAAC,EAC5D,cAAc,CAAC,EAAE,OAAO,uBAAuB,EAAE,cAAc,YAAA;IAChF;;;;;OAKG;IACK,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,YAAY,EAAE,gBAAgB,CAAC,YAAA;IAChF;;;;;;OAMG;IACc,yBAAyB,UAAO,EAKlD;IAED;;;;;;;;;;;;;;;;;;OAkBG;IACH,gBAAgB,CAAC,UAAU,EAAE,iBAAiB,EAAE,GAAG,IAAI,CAmBtD;IAED;;;;OAIG;IACG,SAAS,IAAI,OAAO,CAAC,mBAAmB,CAAC,CAsK9C;IAED;;;;;;OAMG;IACH,kBAAkB,IAAI,aAAa,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;KAAE,CAAC,CAEhF;IAED;;;;;OAKG;IACH,oBAAoB,IAAI,MAAM,EAAE,CAI/B;IAED;;OAEG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC,CA0B9B;IAED;;OAEG;IACH,QAAQ,IAAI,mBAAmB,CAE9B;IAED;;;;;;;;;;;;OAYG;IACH,QAAQ,CACN,MAAM,EAAE,oBAAoB,EAC5B,kBAAkB,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,YAAY,EAAE,mBAAmB,CAAC,EAC5E,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,YAAY,EAAE,gBAAgB,CAAC,GACvE,IAAI,CAKN;IAED;;;OAGG;IACH,qBAAqB,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,YAAY,EAAE,mBAAmB,CAAC,CAEhF;IAED;;;;;OAKG;IACH,oBAAoB,IAAI,MAAM,EAAE,CAI/B;IAED;;;;;OAKG;IACH,wBAAwB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAGvD;IAED;;;OAGG;IACH,iBAAiB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CA2CjE;IAMD;;;;;;;;;OASG;IACH,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAqB;IAEjD;;;;;;;;;;;;;;;;;OAiBG;IACH,cAAc,CACZ,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAC1B,WAAW,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAChC,KAAK,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE;IACpE;;;;;;;OAOG;IACH,eAAe,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GACpC,IAAI,CASN;YAEa,gBAAgB;YAchB,sBAAsB;IAmDpC,OAAO,CAAC,aAAa;YA2FP,cAAc;YAwBd,WAAW;YAgDX,aAAa;IAI3B;;;OAGG;YACW,2BAA2B;IA8EzC;;;OAGG;YACW,yBAAyB;IAevC;;;;;;;;;;;OAWG;YACW,qBAAqB;IASnC,OAAO,CAAC,kBAAkB;IAS1B,OAAO,CAAC,iBAAiB;IAOzB;;;;;;;;;;OAUG;IACH,OAAO,CAAC,sBAAsB;YAIhB,WAAW;CA+F1B"}