@underpostnet/cyberia 3.3.0 → 3.3.76

package/src/client-builder/client-bundle.js

@@ -0,0 +1,347 @@
+/**
+ * Client bundle transport: enumerates the zip artifacts a deployment's routes build, uploads
+ * them to file storage, and restores them into the served public directory.
+ *
+ * Push and pull are two directions of one contract, so both resolve their routes, bundle ids,
+ * storage keys and part files through the helpers here rather than re-deriving them. Route
+ * selection mirrors {@link buildClient}: a route that the build skips produces no artifact for
+ * either direction to carry.
+ * @module src/client-builder/client-bundle.js
+ * @namespace clientBundle
+ */
+
+import fs from 'fs-extra';
+import UnderpostFileStorage from '../cli/fs.js';
+import * as dir from 'path';
+import { cli } from '../server/build/execution.js';
+import { loadConfServerJson, loadReplicas } from '../server/runtime/conf.js';
+import { loggerFactory } from '../server/ops/logger.js';
+import { shellArgumentFactory, shellExec } from '../server/runtime/process.js';
+import { clientBundleIdFactory, getZipPartPaths, mergeClientBuildZip, unzipClientBuild } from './client-build.js';
+
+const logger = loggerFactory(import.meta);
+
+/** @type {string} Directory `buildClient --build-zip` writes bundles to, and the only place push and pull look. */
+const CLIENT_BUNDLE_DIRECTORY = './build';
+
+/** @type {number} Part size in MB applied when the caller names none. */
+const DEFAULT_SPLIT_MB = 8;
+
+/** @type {string} `--split` value that turns splitting off. */
+const SPLIT_DISABLED = 'none';
+
+const CLIENT_BUNDLE_STORAGE_ID = 'bundle';
+
+/**
+ * Resolves the conf.server path a deployment's routes are read from.
+ * @function clientBundleConfServerPath
+ * @param {string} deployId - The deployment identifier.
+ * @returns {string} Path to the deployment's `conf.server.json`.
+ * @memberof clientBundle
+ */
+const clientBundleConfServerPath = (deployId) => `./engine-private/conf/${deployId}/conf.server.json`;
+
+/**
+ * Resolves the storage manifest that tracks a deployment's uploaded bundle parts. Push writes
+ * it and pull reads it, so both must name the same file.
+ * @function clientBundleStorageFilePath
+ * @param {string} deployId - The deployment identifier.
+ * @returns {string} Path to the deployment's bundle storage manifest.
+ * @memberof clientBundle
+ */
+const clientBundleStorageFilePath = (deployId) =>
+  UnderpostFileStorage.API.resolveManifest({ deployId, storageId: CLIENT_BUNDLE_STORAGE_ID });
+
+/**
+ * Parses the comma-separated host filter both runners accept as their positional argument.
+ * @function clientBundleHostFilter
+ * @param {string} [hosts] - Comma-separated host names, or empty for every host.
+ * @returns {string[]} The requested hosts, empty when no filter was given.
+ * @memberof clientBundle
+ */
+const clientBundleHostFilter = (hosts = '') =>
+  `${hosts ?? ''}`
+    .split(',')
+    .map((host) => host.trim())
+    .filter(Boolean);
+
+/**
+ * Resolves the `--split` option into the flag `underpost client` expects.
+ * @function clientBundleSplitFlag
+ * @param {string|number} [split] - Part size in MB, or `none` to disable splitting.
+ * @returns {string} The flag to append, empty when splitting is disabled.
+ * @memberof clientBundle
+ */
+const clientBundleSplitFlag = (split) => {
+  const requested = `${split ?? ''}`.trim();
+  if (!requested) return `--split ${DEFAULT_SPLIT_MB}`;
+  if (requested === SPLIT_DISABLED) return '';
+
+  const splitMb = Number(requested);
+  if (Number.isFinite(splitMb) && splitMb > 0) return `--split ${splitMb}`;
+
+  logger.warn('invalid split option, falling back to the default', { split: requested, default: DEFAULT_SPLIT_MB });
+  return `--split ${DEFAULT_SPLIT_MB}`;
+};
+
+/**
+ * Enumerates the bundles a deployment's routes build to.
+ *
+ * A route is skipped exactly where {@link buildClient} skips it — redirects and
+ * rebuild-disabled routes emit nothing, and a `singleReplica` route's replicas are built under
+ * their own deploy ids, so they carry their own manifests rather than the parent's.
+ * @function clientBundleEntriesFactory
+ * @param {object} options
+ * @param {string} options.deployId - The deployment identifier.
+ * @param {string[]} [options.hosts] - Optional host filter; empty selects every configured host.
+ * @returns {Array<{host: string, routePath: string, buildId: string, zipPath: string, publicPath: string}>}
+ *   One entry per bundle, in configuration order.
+ * @memberof clientBundle
+ */
+const clientBundleEntriesFactory = ({ deployId, hosts = [] }) => {
+  const confServerPath = clientBundleConfServerPath(deployId);
+  if (!fs.existsSync(confServerPath)) {
+    logger.warn('conf.server not found', { deployId, confServerPath });
+    return [];
+  }
+
+  const confServer = loadReplicas(deployId, loadConfServerJson(confServerPath));
+  const selectedHosts = hosts.length > 0 ? hosts : Object.keys(confServer);
+
+  return selectedHosts.flatMap((host) => {
+    if (!confServer[host]) {
+      logger.warn('host not present in conf.server', { deployId, host });
+      return [];
+    }
+    return Object.keys(confServer[host])
+      .filter((routePath) => {
+        const { redirect, disabledRebuild, singleReplica } = confServer[host][routePath] ?? {};
+        return !redirect && !disabledRebuild && !singleReplica;
+      })
+      .map((routePath) => {
+        const buildId = clientBundleIdFactory(host, routePath);
+        return {
+          host,
+          routePath,
+          buildId,
+          zipPath: `${CLIENT_BUNDLE_DIRECTORY}/${buildId}.zip`,
+          publicPath: routePath === '/' ? `public/${host}` : `public/${host}${routePath}`,
+        };
+      });
+  });
+};
+
+/**
+ * Reads the storage manifest as a plain lookup, for the recorded size of each uploaded artifact.
+ * Read-only and tolerant: a manifest that is absent or unreadable disables verification rather
+ * than failing the pull, since the artifacts themselves are still what the bundle is made of.
+ * @function clientBundleManifestFactory
+ * @param {string} deployId - The deployment identifier.
+ * @returns {Object<string, {bytes?: number}>} The tracked entries, empty when unavailable.
+ * @memberof clientBundle
+ */
+const clientBundleManifestFactory = (deployId) => {
+  const manifestPath = clientBundleStorageFilePath(deployId);
+  try {
+    return UnderpostFileStorage.API.readManifest({ deployId, storageId: CLIENT_BUNDLE_STORAGE_ID }).storage;
+  } catch (error) {
+    logger.warn('bundle manifest unreadable, size verification skipped', { manifestPath, error: error?.message });
+    return {};
+  }
+};
+
+/**
+ * The tracked keys that belong to one bundle.
+ *
+ * A bundle owns its zip and every split part of it, and nothing else: the id already carries the
+ * route, so `underpost.net-` and `underpost.net-peer` cannot claim each other's parts.
+ * @function clientBundleTrackedKeys
+ * @param {object} manifest - The bundle storage manifest.
+ * @param {string} buildId - The bundle's build id.
+ * @returns {string[]} The manifest keys for that bundle.
+ * @memberof clientBundle
+ */
+const clientBundleTrackedKeys = (manifest, buildId) =>
+  Object.keys(manifest).filter((key) =>
+    key.replace(/^\.\//, '').startsWith(`${CLIENT_BUNDLE_DIRECTORY.replace(/^\.\//, '')}/${buildId}.zip`),
+  );
+
+/**
+ * Lists the local artifacts that make up one bundle: its split parts when the build split it,
+ * otherwise the single zip.
+ * @function clientBundleArtifactPaths
+ * @param {string} zipPath - The bundle's zip path.
+ * @returns {{zipPath: string, partPaths: string[], artifactPaths: string[]}} The resolved artifacts.
+ * @memberof clientBundle
+ */
+const clientBundleArtifactPaths = (zipPath) => {
+  const partPaths = fs.existsSync(dir.dirname(zipPath)) ? getZipPartPaths(zipPath) : [];
+  const artifactPaths = partPaths.length > 0 ? partPaths : fs.existsSync(zipPath) ? [zipPath] : [];
+  return { zipPath, partPaths, artifactPaths };
+};
+
+/**
+ * Builds a deployment's client bundles and uploads them to file storage.
+ *
+ * The build and the uploads run as child processes because `underpost app load` writes the
+ * `.env` this deployment's credentials live in, and only a freshly started process reads it.
+ * @function pushClientBundle
+ * @param {object} options
+ * @param {string} options.deployId - The deployment identifier.
+ * @param {string[]} [options.hosts] - Optional host filter; empty selects every configured host.
+ * @param {string|number} [options.split] - Part size in MB, or `none` to upload one zip.
+ * @param {boolean} [options.dev] - Build for development; defaults to production.
+ * @returns {{pushed: number, skipped: number}} How many bundles were uploaded and skipped.
+ * @memberof clientBundle
+ */
+const pushClientBundle = ({ deployId, hosts = [], split, dev = false }) => {
+  const baseCommand = cli('underpost', { local: true });
+  const env = dev ? 'development' : 'production';
+  const splitFlag = clientBundleSplitFlag(split);
+
+  shellExec(`${baseCommand} app load --env ${env} --args deploy-id=${deployId}`);
+  shellExec(`${baseCommand} client ${deployId} --env ${env} --build-zip${splitFlag ? ` ${splitFlag}` : ''}`);
+
+  const entries = clientBundleEntriesFactory({ deployId, hosts });
+  if (entries.length === 0) logger.warn('no bundle routes resolved', { deployId, hosts });
+
+  let pushed = 0;
+  let skipped = 0;
+  for (const { host, routePath, buildId, zipPath } of entries) {
+    const { artifactPaths } = clientBundleArtifactPaths(zipPath);
+    if (artifactPaths.length === 0) {
+      logger.warn('no bundle artifacts found', { host, routePath, buildId });
+      skipped++;
+      continue;
+    }
+    for (const artifactPath of artifactPaths) {
+      shellExec(
+        `${baseCommand} fs ${artifactPath} --deploy-id ${deployId} --storage-id ${CLIENT_BUNDLE_STORAGE_ID} --force`,
+      );
+    }
+
+    // Remove obsolete manifest entries only after all current artifacts upload.
+    const current = new Set(artifactPaths.map((artifactPath) => artifactPath.replace(/^\.\//, '')));
+    const stale = clientBundleTrackedKeys(clientBundleManifestFactory(deployId), buildId).filter(
+      (key) => !current.has(key.replace(/^\.\//, '')),
+    );
+    for (const key of stale) {
+      shellExec(
+        `${baseCommand} fs --tracked --key ${shellArgumentFactory(key)} ` +
+          `--deploy-id ${deployId} --storage-id ${CLIENT_BUNDLE_STORAGE_ID} --rm`,
+      );
+    }
+    if (stale.length > 0) logger.warn('removed bundle parts a smaller build no longer produces', { buildId, stale });
+
+    logger.info('pushed bundle', { host, routePath, artifacts: artifactPaths.length });
+    pushed++;
+  }
+  return { pushed, skipped };
+};
+
+/**
+ * Downloads a deployment's client bundles from file storage and installs them into the served
+ * public directory.
+ *
+ * Merging and extraction run in-process through the build module; only the download needs a
+ * child process, for the same `.env` reason {@link pushClientBundle} documents.
+ * @function pullClientBundle
+ * @param {object} options
+ * @param {string} options.deployId - The deployment identifier.
+ * @param {string[]} [options.hosts] - Optional host filter; empty selects every configured host.
+ * @param {boolean} [options.dev] - Pull the development environment; defaults to production.
+ * @returns {Promise<{pulled: number, skipped: number}>} How many bundles were installed and skipped.
+ * @memberof clientBundle
+ */
+const expectedBundleBytes = ({ manifest, partPaths }) => {
+  const sizes = partPaths.map((partPath) => manifest[clientBundleManifestKey(partPath)]?.bytes);
+  return sizes.every((bytes) => Number.isFinite(bytes)) ? sizes.reduce((total, bytes) => total + bytes, 0) : undefined;
+};
+
+/**
+ * The manifest key a locally pulled part corresponds to. A pull writes the downloaded archive as
+ * `<key>.zip`, so the wrapper suffix is what has to come off to address the entry that produced it.
+ * @function clientBundleManifestKey
+ * @param {string} partPath - Local path of a pulled part.
+ * @returns {string} The tracked storage key.
+ * @memberof clientBundle
+ */
+const clientBundleManifestKey = (partPath) => partPath.replace(/^\.\//, '').replace(/\.zip$/i, '');
+
+const pullClientBundle = async ({ deployId, hosts = [], dev = false }) => {
+  const baseCommand = cli('underpost', { local: true });
+  const env = dev ? 'development' : 'production';
+
+  const entries = clientBundleEntriesFactory({ deployId, hosts });
+  if (entries.length === 0) {
+    logger.error('no bundle routes resolved', { deployId, hosts });
+    return { pulled: 0, skipped: 0 };
+  }
+
+  shellExec(`${baseCommand} app load --env ${env} --args deploy-id=${deployId}`);
+  fs.mkdirSync(CLIENT_BUNDLE_DIRECTORY, { recursive: true });
+  shellExec(
+    `${baseCommand} fs ${CLIENT_BUNDLE_DIRECTORY.replace(/^\.\//, '')} --tracked --deploy-id ${deployId} ` +
+      `--storage-id ${CLIENT_BUNDLE_STORAGE_ID} --pull --omit-unzip`,
+  );
+
+  const manifest = clientBundleManifestFactory(deployId);
+
+  let pulled = 0;
+  let skipped = 0;
+  for (const { host, routePath, buildId, zipPath, publicPath } of entries) {
+    const { partPaths, artifactPaths } = clientBundleArtifactPaths(zipPath);
+    if (artifactPaths.length === 0) {
+      logger.warn('bundle not found, skipping', { host, routePath, buildId, zipPath });
+      skipped++;
+      continue;
+    }
+
+    if (partPaths.length > 0) {
+      const { mergedBytes } = await mergeClientBuildZip({ buildPrefix: zipPath, logger });
+      // The parts are reassembled blind, so a single stale or short one is only visible as a
+      // corrupt archive far downstream. The pushed sizes are what identify it here instead.
+      const expectedBytes = expectedBundleBytes({ manifest, partPaths });
+      if (expectedBytes !== undefined && expectedBytes !== mergedBytes)
+        throw new Error(
+          `Bundle ${buildId} reassembled to ${mergedBytes} bytes, expected ${expectedBytes} from ${partPaths.length} pushed parts`,
+        );
+    }
+    const { outputPath } = await unzipClientBuild({ buildPrefix: zipPath, logger });
+
+    fs.removeSync(zipPath);
+    for (const partPath of partPaths) fs.removeSync(partPath);
+
+    if (!fs.existsSync(outputPath)) {
+      logger.warn('extracted bundle directory not found, skipping', { host, routePath, outputPath });
+      skipped++;
+      continue;
+    }
+
+    // The previous deployment's document root can be owned by the web server user, so the
+    // replace goes through sudo; the extracted tree this process just wrote does not.
+    shellExec(`sudo rm -rf ${publicPath}`);
+    if (routePath !== '/') shellExec(`sudo mkdir -p public/${host}`);
+    fs.copySync(outputPath, publicPath);
+
+    logger.info('pulled bundle', { host, routePath, publicPath });
+    pulled++;
+  }
+  return { pulled, skipped };
+};
+
+export {
+  CLIENT_BUNDLE_DIRECTORY,
+  clientBundleTrackedKeys,
+  clientBundleManifestFactory,
+  clientBundleManifestKey,
+  DEFAULT_SPLIT_MB,
+  clientBundleArtifactPaths,
+  clientBundleConfServerPath,
+  clientBundleEntriesFactory,
+  clientBundleHostFilter,
+  clientBundleSplitFlag,
+  clientBundleStorageFilePath,
+  pullClientBundle,
+  pushClientBundle,
+};

package/src/client-builder/client-dev-server.js

@@ -6,8 +6,8 @@
 import fs from 'fs-extra';
 import nodemon from 'nodemon';
 import dotenv from 'dotenv';
-import { shellExec } from '../server/process.js';
-import { loggerFactory } from '../server/logger.js';
+import { shellExec } from '../server/runtime/process.js';
+import { loggerFactory } from '../server/ops/logger.js';
 import Underpost from '../index.js';
 
 const logger = loggerFactory(import.meta);
@@ -32,7 +32,7 @@ const createClientDevServer = async (
   const devClientEnvPath = `./engine-private/conf/${deployId}/.env.${process.env.NODE_ENV}.${subConf}-dev-client`;
   if (fs.existsSync(devClientEnvPath)) dotenv.config({ path: devClientEnvPath, override: true });
 
-  await Underpost.repo.client(deployId, `${subConf}-dev-client`.trim(), host, path);
+  await Underpost.client.callback(deployId, `${subConf}-dev-client`.trim(), host, path);
 
   shellExec(`node src/server ${deployId} ${subConf}-dev-client`.trim(), {
     async: true,

package/src/client-builder/client-icons.js

@@ -4,7 +4,7 @@
  * @namespace clientIcons
  */
 import { favicons } from 'favicons';
-import { loggerFactory } from '../server/logger.js';
+import { loggerFactory } from '../server/ops/logger.js';
 import fs from 'fs-extra';
 import { getCapVariableName } from '../client/components/core/CommonJs.js';
 
@@ -99,7 +99,7 @@ const buildIcons = async ({
 
     const ssrPath = `./src/client/ssr/head/Pwa${getCapVariableName(publicClientId)}.js`;
     if (!fs.existsSync(ssrPath))
-      fs.writeFileSync(ssrPath, 'SrrComponent = () => html`' + response.html.join(`\n`) + '`;', 'utf8');
+      fs.writeFileSync(ssrPath, 'SSRComponent = () => html`' + response.html.join(`\n`) + '`;', 'utf8');
   } catch (error) {
     logger.error(error.message); // Error description e.g. "An unknown error has occurred"
   }

package/src/client-builder/ssr.js

@@ -10,7 +10,7 @@ import vm from 'node:vm';
 import Underpost from '../index.js';
 
 import { srcFormatted } from './client-formatted.js';
-import { loggerFactory } from '../server/logger.js';
+import { loggerFactory } from '../server/ops/logger.js';
 
 const logger = loggerFactory(import.meta);
 
@@ -22,10 +22,10 @@ const logger = loggerFactory(import.meta);
  * @memberof ServerSideRendering
  */
 const ssrFactory = async (componentPath = `./src/client/ssr/RootDocument.js`) => {
-  const context = { SrrComponent: () => {}, npm_package_version: Underpost.version };
+  const context = { SSRComponent: () => {}, npm_package_version: Underpost.version };
   vm.createContext(context);
   vm.runInContext(await srcFormatted(fs.readFileSync(componentPath, 'utf8')), context);
-  return context.SrrComponent;
+  return context.SSRComponent;
 };
 
 /**

package/src/client.build.js

@@ -1,11 +1,8 @@
 'use strict';
 
-// https://nodejs.org/api
-// https://expressjs.com/en/4x/api.html
-
-import { loggerFactory } from './server/logger.js';
-import { Config } from './server/conf.js';
-import { ProcessController } from './server/process.js';
+import { loggerFactory } from './server/ops/logger.js';
+import { Config } from './server/runtime/conf.js';
+import { ProcessController } from './server/runtime/process.js';
 import { clientLiveBuild } from './client-builder/client-build-live.js';
 
 await Config.build();

package/src/client.dev.js

@@ -1,11 +1,8 @@
 'use strict';
 
-// https://nodejs.org/api
-// https://expressjs.com/en/4x/api.html
-
-import { loggerFactory } from './server/logger.js';
-import { ProcessController } from './server/process.js';
-import { Config, buildClientStaticConf } from './server/conf.js';
+import { loggerFactory } from './server/ops/logger.js';
+import { ProcessController } from './server/runtime/process.js';
+import { Config, buildClientStaticConf } from './server/runtime/conf.js';
 import { createClientDevServer } from './client-builder/client-dev-server.js';
 
 const logger = loggerFactory(import.meta);

package/src/db/DataBaseProvider.js

@@ -1,8 +1,9 @@
 import { MongooseDB } from './mongo/MongooseDB.js';
-import { loggerFactory } from '../server/logger.js';
+import { loggerFactory } from '../server/ops/logger.js';
 import { getCapVariableName } from '../client/components/core/CommonJs.js';
-import { resolveHostKeyContext } from '../server/conf.js';
+import { resolveHostKeyContext } from '../server/runtime/conf.js';
 import Underpost from '../index.js';
+import { latchRuntimeError } from '../server/runtime/runtime-status.js';
 
 /**
  * Module for managing and loading various database connections (e.g., Mongoose, MariaDB).
@@ -12,6 +13,60 @@ import Underpost from '../index.js';
 
 const logger = loggerFactory(import.meta);
 
+// Health watch defaults. `DB_HEALTH_INTERVAL_MS=0` disables the watch entirely.
+const HEALTH_INTERVAL_MS = Number(process.env.DB_HEALTH_INTERVAL_MS ?? 15000);
+const HEALTH_PING_TIMEOUT_MS = 5000;
+
+/**
+ * Bounds a liveness probe that would otherwise wait on the driver's own connection timeout.
+ * @param {Promise<boolean>} probe - Provider probe resolving to liveness.
+ * @returns {Promise<boolean>} `false` when the probe throws or outlives the bound.
+ */
+const withPingTimeout = async (probe) => {
+  let timer;
+  try {
+    return await Promise.race([
+      probe,
+      new Promise((resolve) => {
+        timer = setTimeout(() => resolve(false), HEALTH_PING_TIMEOUT_MS);
+      }),
+    ]);
+  } catch {
+    return false;
+  } finally {
+    clearTimeout(timer);
+  }
+};
+
+/**
+ * Per-provider health strategies, keyed by `db.provider`.
+ *
+ * A provider with no entry here gets no health watch and no reconnect: liveness probes,
+ * rebuild semantics, and teardown differ per client, so there is nothing generic to fall
+ * back on. Adding one (MariaDB, PostgreSQL, …) means adding its case here — the rest of the
+ * service dispatches through this table and needs no change.
+ *
+ * @type {object.<string, {isAlive: Function, rebuild: Function, close: Function}>}
+ */
+const PROVIDER_HEALTH = {
+  mongoose: {
+    isAlive: async (connection) => {
+      if (!connection || connection.readyState !== 1) return false;
+      return await withPingTimeout(
+        connection.db
+          .admin()
+          .command({ ping: 1 })
+          .then(() => true),
+      );
+    },
+    rebuild: async ({ apis, db }) => {
+      const connection = await MongooseDB.connect(db);
+      return { connection, models: await MongooseDB.loadModels({ conn: connection, apis }) };
+    },
+    close: async (connection) => await connection?.close(),
+  },
+};
+
 /**
  * @class
  * @alias DataBaseProviderService
@@ -126,6 +181,128 @@ class DataBaseProviderService {
     });
   }
 
+  /**
+   * Timers for the per-provider health watch, keyed by `<contextKey>:<provider>`.
+   * @type {object.<string, {timer: object, attempts: number, rebuilding: boolean}>}
+   */
+  static #watchers = {};
+
+  /**
+   * Rebuilds a loaded provider in place: new connection, freshly bound models, old
+   * connection closed once the replacement is live.
+   *
+   * A reconnect is not the same as letting the driver heal itself. For mongoose: after a
+   * replica set is reconfigured the driver can hold a topology it will never accept a primary
+   * for again — it records the highest `setVersion`/`electionId` it has seen and rejects
+   * anything lower for the rest of the process — so only a new client recovers. Every consumer
+   * resolves models through this bucket on each call, so swapping the fields in place is
+   * enough; nothing has to be re-registered.
+   *
+   * Only providers present in {@link PROVIDER_HEALTH} can be rebuilt.
+   *
+   * @param {{host?: string, path?: string}|string} context - Context object or key.
+   * @param {string} [provider='mongoose'] - Provider name.
+   * @returns {Promise<boolean>} `true` when the provider was rebuilt.
+   */
+  static async reconnect(context, provider = 'mongoose') {
+    const key = resolveHostKeyContext(context);
+    const strategy = PROVIDER_HEALTH[provider];
+    if (!strategy) {
+      logger.warn('No reconnect strategy for provider', { key, provider });
+      return false;
+    }
+
+    const bucket = this.#instance[key]?.[provider];
+    if (!bucket?.rebuild) {
+      logger.warn('Cannot reconnect a provider that was never loaded', { key, provider });
+      return false;
+    }
+
+    const previousConnection = bucket.connection;
+
+    try {
+      const { connection, models } = await strategy.rebuild(bucket.rebuild);
+
+      bucket.connection = connection;
+      bucket.models = models;
+      logger.info('Database connection rebuilt', { key, provider });
+    } catch (error) {
+      logger.error('Database reconnect failed, keeping the existing connection', {
+        key,
+        provider,
+        error: error.message,
+      });
+      return false;
+    }
+
+    // Closed last and best-effort: the replacement is already serving, and a close that
+    // hangs on a dead server must not take the recovery with it.
+    try {
+      await strategy.close(previousConnection);
+    } catch (error) {
+      logger.warn('Could not close the replaced connection', { key, provider, error: error.message });
+    }
+    return true;
+  }
+
+  /**
+   * Starts the health watch for a provider: pings on an interval and rebuilds the connection
+   * on every tick that finds it down, until it answers again. A provider with no
+   * {@link PROVIDER_HEALTH} entry is left unwatched.
+   * @param {string} key - Resolved context key.
+   * @param {string} provider - Provider name.
+   */
+  static #startHealthWatch(key, provider) {
+    const strategy = PROVIDER_HEALTH[provider];
+    if (!strategy || !(HEALTH_INTERVAL_MS > 0)) return;
+    const watchKey = `${key}:${provider}`;
+    if (this.#watchers[watchKey]) return;
+
+    const state = { attempts: 0, rebuilding: false, timer: null };
+    state.timer = setInterval(async () => {
+      // A rebuild can outlast the interval (the driver's own connect timeout is longer), so
+      // ticks that land mid-attempt are skipped rather than stacking attempts.
+      if (state.rebuilding) return;
+      const bucket = this.#instance[key]?.[provider];
+      if (!bucket) return;
+
+      if (await strategy.isAlive(bucket.connection)) {
+        if (state.attempts > 0)
+          logger.info('Database connection restored', { key, provider, attempts: state.attempts });
+        state.attempts = 0;
+        return;
+      }
+
+      // Every tick retries until the connection answers again: there is no attempt budget to
+      // exhaust, because giving up leaves the runtime permanently unable to reach its database.
+      state.rebuilding = true;
+      state.attempts++;
+      logger.warn('Database connection is down, rebuilding', { key, provider, attempt: state.attempts });
+      try {
+        await this.reconnect(key, provider);
+      } finally {
+        state.rebuilding = false;
+      }
+    }, HEALTH_INTERVAL_MS);
+
+    // Never hold the event loop open on account of a health check.
+    state.timer.unref?.();
+    this.#watchers[watchKey] = state;
+  }
+
+  /**
+   * Stops the health watch for a provider.
+   * @param {string} key - Resolved context key.
+   * @param {string} provider - Provider name.
+   */
+  static #stopHealthWatch(key, provider) {
+    const watchKey = `${key}:${provider}`;
+    const state = this.#watchers[watchKey];
+    if (!state) return;
+    clearInterval(state.timer);
+    delete this.#watchers[watchKey];
+  }
+
   /**
    * Loads and initializes a database provider based on the configuration.
    * If the connection is already loaded for the given host/path, it returns the existing instance.
@@ -166,9 +343,12 @@ class DataBaseProviderService {
             const conn = await MongooseDB.connect(db);
             this.#instance[key][db.provider] = {
               dbSignature,
+              // Kept so the connection can be rebuilt later without the caller being present.
+              rebuild: { apis, host, path, db },
               models: await MongooseDB.loadModels({ conn, apis }),
               connection: conn,
               close: async () => {
+                this.#stopHealthWatch(key, db.provider);
                 return await new Promise((resolve) => {
                   this.#instance[key][db.provider].connection.close().then(() => {
                     // logger.info('Mongoose connection is disconnected', db);
@@ -182,6 +362,9 @@ class DataBaseProviderService {
         default:
           break;
       }
+      // Self-guarding: only providers with a PROVIDER_HEALTH entry are actually watched, so a
+      // provider added above does not have to remember to opt in here.
+      this.#startHealthWatch(key, db.provider);
       return this.#instance[key][db.provider];
     } catch (error) {
       // Sanitize options to prevent credential exposure in logs
@@ -200,7 +383,7 @@ class DataBaseProviderService {
           : {},
       };
       logger.error(error.message, { safeOptions });
-      if (Underpost.env.isInsideContainer()) Underpost.env.set('container-status', 'error');
+      latchRuntimeError();
       return undefined;
     }
   }