@underpostnet/cyberia 3.3.0 → 3.3.76

package/src/cli/cluster.js

@@ -4,28 +4,34 @@
  * @namespace UnderpostCluster
  */
 
-import { clusterTypeFactory, gatewayApiEnabledFactory, resolveReplicaCount } from '../server/conf.js';
-import { getNpmRootPath, HOST_VOLUME_ROOT } from '../server/environment.js';
-import { loggerFactory } from '../server/logger.js';
-import { shellExec } from '../server/process.js';
-import { crictlCommandFactory, resolveCriSocket } from '../server/cri.js';
+import { clusterTypeFactory, gatewayApiEnabledFactory, resolveReplicaCount } from '../server/runtime/conf.js';
+import { getNpmRootPath, HOST_VOLUME_ROOT } from '../server/runtime/environment.js';
+import { loggerFactory } from '../server/ops/logger.js';
+import { shellExec } from '../server/runtime/process.js';
+import { crictlCommandFactory, resolveCriSocket } from '../server/ops/cri.js';
 import {
   runSELinuxCommands,
-  selinuxContainerSharedContextCommandsFactory,
   selinuxEnforcingCommandsFactory,
   selinuxPackagesCommandFactory,
   selinuxRestoreconCommandFactory,
-} from '../server/selinux.js';
-import { UNDERPOST_GATEWAY, seedDefaultStatusPage } from '../server/underpost-gateway.js';
+} from '../server/security/selinux.js';
+import ContainerStorageService, {
+  containerStorageRootsFactory,
+  ensureContainerStorage,
+  localPathConfigCommandFactory,
+  localPathRootsFactory,
+} from '../server/security/container-storage.js';
+import { UNDERPOST_GATEWAY, seedDefaultStatusPage } from '../server/network/underpost-gateway.js';
 import {
   UNDERPOST_INGRESS,
   gatewayBackendFactory,
   underpostIngressConfFactory,
   underpostIngressHostMapFactory,
   underpostIngressManifestsFactory,
-} from '../server/underpost-ingress.js';
+} from '../server/network/underpost-ingress.js';
 import { MONGODB_DEFAULT_REPLICA_COUNT } from '../db/mongo/MongooseDB.js';
 import { MongoBootstrap } from '../db/mongo/MongoBootstrap.js';
+import { MongoExpress } from '../db/mongo/MongoExpress.js';
 import os from 'os';
 import fs from 'fs-extra';
 import Underpost from '../index.js';
@@ -35,8 +41,8 @@ const logger = loggerFactory(import.meta);
 const GATEWAY_API_RELEASE = 'v1.5.1';
 const ENVOY_GATEWAY_VERSION = 'v1.8.3';
 const CONTOUR_NAMESPACE = 'projectcontour';
-const KUBEADM_CONTAINER_MOUNT_PATHS = ['/etc/kubernetes', '/var/lib/etcd', '/var/lib/calico'];
-const HOST_VOLUME_PATHS = ['/data', '/opt/local-path-provisioner', HOST_VOLUME_ROOT];
+const LOCAL_PATH_PROVISIONER_MANIFEST =
+  'https://cdn.jsdelivr.net/gh/rancher/local-path-provisioner@master/deploy/local-path-storage.yaml';
 const K3S_SELINUX_PATHS = [
   '/usr/local/bin/k3s',
   '/etc/rancher',
@@ -45,15 +51,16 @@ const K3S_SELINUX_PATHS = [
   '/var/lib/cni',
 ];
 
-const shareWithContainers = (paths = []) => {
-  for (const path of paths) shellExec(`sudo mkdir -p ${path}`);
-  runSELinuxCommands(selinuxContainerSharedContextCommandsFactory(paths), { execute: shellExec });
-};
-
-const enforceSELinux = (paths = []) => {
+// `semanage` and `restorecon` come from separate packages and neither is guaranteed on a minimal
+// Rocky install. Installed before anything tries to register a mapping, because a missing
+// `semanage` would otherwise turn every persistent labeling command into a hard failure.
+const ensureSELinuxTooling = () =>
   shellExec(
     `if [ -f /etc/redhat-release ] && command -v dnf >/dev/null 2>&1 && { ! command -v restorecon >/dev/null 2>&1 || ! command -v semanage >/dev/null 2>&1; }; then ${selinuxPackagesCommandFactory()}; fi`,
   );
+
+const enforceSELinux = (paths = []) => {
+  ensureSELinuxTooling();
   const commands = selinuxEnforcingCommandsFactory({ restorePaths: paths });
   runSELinuxCommands(commands, { execute: shellExec });
 };
@@ -76,6 +83,7 @@ class UnderpostCluster {
      * @param {object} [options] - Configuration options for cluster initialization.
      * @param {boolean} [options.mongodb=false] - Deploy MongoDB.
      * @param {boolean} [options.mongodb4=false] - Deploy MongoDB 4.4.
+     * @param {boolean} [options.mongoExpress=false] - Deploy the mongo-express web client for inspecting the MongoDB StatefulSet collections.
      * @param {string} [options.serviceHost=''] - Set a custom host/IP for exposed MongoDB and Valkey clients.
      * @param {boolean} [options.mariadb=false] - Deploy MariaDB.
      * @param {boolean} [options.mysql=false] - Deploy MySQL.
@@ -112,9 +120,10 @@ class UnderpostCluster {
      * @param {boolean} [options.nodePort=false] - Expose enabled ready services (e.g. MongoDB 4.4, Valkey)
      *   to the host/public network via their NodePort Service manifest. The node port value lives directly
      *   in each manifest (mongodb-4.4/mongodb-nodeport.yaml, valkey/valkey-nodeport.yaml).
-     * @param {string} [options.nodeSelector=''] - Pin the just-deployed StatefulSet (MongoDB 4.4 / Valkey)
-     *   to a specific Kubernetes node by name (e.g. 'localhost.localdomain'). Applied via a
-     *   `kubernetes.io/hostname` nodeSelector patch once the workload reports Ready.
+     * @param {string} [options.nodeName=''] - Pin the just-deployed StatefulSet (MongoDB 4.4 / Valkey)
+     *   and the observability workloads to a specific Kubernetes node by name (e.g.
+     *   'localhost.localdomain'). Applied via a `kubernetes.io/hostname` nodeSelector patch once
+     *   the workload reports Ready.
      * @memberof UnderpostCluster
      */
     async init(
@@ -122,6 +131,7 @@ class UnderpostCluster {
       options = {
         mongodb: false,
         mongodb4: false,
+        mongoExpress: false,
         serviceHost: '',
         mariadb: false,
         mysql: false,
@@ -154,7 +164,7 @@ class UnderpostCluster {
         hosts: '',
         replicas: '',
         nodePort: false,
-        nodeSelector: '',
+        nodeName: '',
       },
     ) {
       if (options.initHost) return Underpost.cluster.initHost();
@@ -232,6 +242,12 @@ class UnderpostCluster {
         logger.info(
           `Detected existing ${runtime.type} cluster (${runtime.ready ? 'Ready' : 'NotReady'}); skipping initialization.`,
         );
+        // Initialization is skipped, but the labeling invariant is not: the node directories
+        // backing PersistentVolumes outlive the bring-up that created them, and a policy update,
+        // a full relabel or a directory recreated by kubelet resets them to the policy default.
+        // Re-asserting it costs a `semanage` lookup and a `restorecon` pass, mutates no workload,
+        // and never changes SELinux mode.
+        Underpost.cluster.ensureContainerStorageContexts();
       }
 
       // --- Kubeadm/Kind/K3s Cluster Initialization ---
@@ -265,7 +281,7 @@ class UnderpostCluster {
         } else if (options.kubeadm) {
           Underpost.cluster.config();
           Underpost.cluster.natSetup({ underpostRoot });
-          shareWithContainers([...KUBEADM_CONTAINER_MOUNT_PATHS, ...HOST_VOLUME_PATHS]);
+          Underpost.cluster.ensureContainerStorageContexts({ controlPlane: true });
           logger.info('Initializing Kubeadm control plane...');
           // Set default values if not provided
           const podNetworkCidr = options.podNetworkCidr || '192.168.0.0/16';
@@ -293,18 +309,13 @@ class UnderpostCluster {
           // Untaint control plane node to allow scheduling pods
           const nodeName = os.hostname();
           shellExec(`kubectl taint nodes ${nodeName} node-role.kubernetes.io/control-plane:NoSchedule-`);
-          // Install local-path-provisioner for dynamic PVCs
-          logger.info('Installing local-path-provisioner...');
-          shellExec(
-            `kubectl apply -f https://cdn.jsdelivr.net/gh/rancher/local-path-provisioner@master/deploy/local-path-storage.yaml`,
-          );
-
           // The CNI lands asynchronously, and the node stays NotReady until it
           // does. Block here so the deploy steps that follow (MongoDB and the
           // application workloads) schedule onto a node that can actually run
           // them, instead of racing the Calico rollout.
           logger.info('Waiting for the node to become Ready (Calico rollout)...');
           shellExec(`kubectl wait --for=condition=Ready node --all --timeout=300s`);
+          Underpost.cluster.ensureLocalPathProvisioner();
         } else {
           Underpost.cluster.config();
           Underpost.cluster.natSetup({ underpostRoot });
@@ -355,36 +366,19 @@ class UnderpostCluster {
         shellExec(`node ${underpostRoot}/bin/deploy kubeflow-spark-operator${options.kubeadm ? ' kubeadm' : ''}`);
       }
 
-      if (options.grafana) {
-        shellExec(`kubectl delete deployment grafana -n ${options.namespace} --ignore-not-found`);
-        shellExec(`kubectl apply -k ${underpostRoot}/manifests/grafana -n ${options.namespace}`);
-        const yaml = `${fs
-          .readFileSync(`${underpostRoot}/manifests/grafana/deployment.yaml`, 'utf8')
-          .replace('{{GF_SERVER_ROOT_URL}}', options.hosts.split(',')[0])}`;
-        console.log(yaml);
-        shellExec(`kubectl apply -f - -n ${options.namespace} <<'EOF'
-${yaml}
-EOF
-`);
-      }
-
-      if (options.prom) {
-        shellExec(`kubectl delete deployment prometheus --ignore-not-found`);
-        shellExec(`kubectl delete configmap prometheus-config --ignore-not-found`);
-        shellExec(`kubectl delete service prometheus --ignore-not-found`);
-        // Prometheus server host: http://<prometheus-cluster-ip>:9090
-        const yaml = `${fs.readFileSync(`${underpostRoot}/manifests/prometheus/deployment.yaml`, 'utf8').replace(
-          '- targets: []',
-          `- targets: [${options.prom
-            .split(',')
-            .map((host) => `'${host}'`)
-            .join(',')}]`,
-        )}`;
-        console.log(yaml);
-        shellExec(`kubectl apply -f - -n ${options.namespace} <<'EOF'
-${yaml}
-EOF
-`);
+      // Grafana and Prometheus are two faces of one stack, so either flag
+      // converges the whole of it — including Alertmanager and the Blackbox
+      // Exporter, which the alert rules and probes are useless without. The
+      // scrape configuration is rendered from the deploy configuration and the
+      // event registry by the monitor CLI; `--prom [hosts]` only adds hosts
+      // that configuration does not name.
+      if (options.grafana || options.prom) {
+        await Underpost.monitor.deployObservability({
+          namespace: options.namespace,
+          dev: options.dev,
+          nodeName: options.nodeName,
+          hosts: typeof options.prom === 'string' ? options.prom : options.hosts,
+        });
       }
 
       if (options.valkey) {
@@ -396,11 +390,11 @@ EOF
         // The node port (32079) is set directly in the manifest.
         if (valkeyReady && options.nodePort)
           shellExec(`kubectl apply -f ${underpostRoot}/manifests/valkey/valkey-nodeport.yaml -n ${options.namespace}`);
-        if (valkeyReady && options.nodeSelector)
+        if (valkeyReady && options.nodeName)
           Underpost.cluster.pinToNode({
             name: 'valkey-service',
             namespace: options.namespace,
-            node: options.nodeSelector,
+            node: options.nodeName,
           });
         if (valkeyReady && serviceHost)
           Underpost.cluster.syncServiceConnectionEnv({
@@ -417,32 +411,32 @@ EOF
         // Encrypted store first; when no manifest exists the secret is seeded from its origin
         // seed path below. A manifest that exists but fails validation raises rather than
         // silently seeding stale credentials.
-        if (!Underpost.secret.sops.applyIfPresent('mariadb-secret', options.namespace))
-          shellExec(
-            `sudo kubectl create secret generic mariadb-secret --from-file=username=/home/dd/engine/engine-private/mariadb-username --from-file=password=/home/dd/engine/engine-private/mariadb-password --dry-run=client -o yaml | kubectl apply -f - -n ${options.namespace}`,
-          );
+        if (!Underpost.secret.applyIfPresent('mariadb-secret', options.namespace))
+          Underpost.secret.applyFromOriginSeed('mariadb-secret', options.namespace);
         shellExec(`kubectl delete statefulset mariadb-statefulset -n ${options.namespace} --ignore-not-found`);
 
         if (options.pullImage) Underpost.cluster.pullImage('mariadb:latest', options);
+        // The StatefulSet's volumeClaimTemplate is dynamically provisioned, so the data lands in
+        // a subdirectory of the local-path root and inherits that root's label.
+        Underpost.cluster.ensureContainerStorageContexts({ extraPaths: ['/data/mariadb'] });
         shellExec(`kubectl apply -f ${underpostRoot}/manifests/mariadb/storage-class.yaml -n ${options.namespace}`);
         shellExec(`kubectl apply -k ${underpostRoot}/manifests/mariadb -n ${options.namespace}`);
       }
       if (options.mysql) {
-        if (!Underpost.secret.sops.applyIfPresent('mysql-secret', options.namespace))
-          shellExec(
-            `sudo kubectl create secret generic mysql-secret --from-file=username=/home/dd/engine/engine-private/mysql-username --from-file=password=/home/dd/engine/engine-private/mysql-password --dry-run=client -o yaml | kubectl apply -f - -n ${options.namespace}`,
-          );
-        shellExec(`sudo mkdir -p /mnt/data`);
-        shellExec(`sudo chmod 777 /mnt/data`);
-        shellExec(`sudo chown -R $(whoami):$(whoami) /mnt/data`);
+        if (!Underpost.secret.applyIfPresent('mysql-secret', options.namespace))
+          Underpost.secret.applyFromOriginSeed('mysql-secret', options.namespace);
+        // The data directory is prepared, not opened up: a world-writable host path plus the
+        // operator's ownership was standing in for the SELinux label the container actually
+        // needed. `container_file_t` is what lets mysqld write here; 0755 root-owned is enough,
+        // because the image's entrypoint chowns the tree to its own uid on first start.
+        Underpost.cluster.ensureContainerStorageContexts({ extraPaths: ['/data/mysql'] });
         shellExec(`kubectl apply -k ${underpostRoot}/manifests/mysql -n ${options.namespace}`);
       }
       if (options.postgresql) {
         if (options.pullImage) Underpost.cluster.pullImage('postgres:latest', options);
-        if (!Underpost.secret.sops.applyIfPresent('postgres-secret', options.namespace))
-          shellExec(
-            `sudo kubectl create secret generic postgres-secret --from-file=password=/home/dd/engine/engine-private/postgresql-password --dry-run=client -o yaml | kubectl apply -f - -n ${options.namespace}`,
-          );
+        if (!Underpost.secret.applyIfPresent('postgres-secret', options.namespace))
+          Underpost.secret.applyFromOriginSeed('postgres-secret', options.namespace);
+        Underpost.cluster.ensureContainerStorageContexts({ extraPaths: ['/data/postgresql'] });
         shellExec(`kubectl apply -k ${underpostRoot}/manifests/postgresql -n ${options.namespace}`);
       }
       if (options.mongodb4) {
@@ -486,11 +480,11 @@ EOF
             shellExec(
               `kubectl apply -f ${underpostRoot}/manifests/mongodb-4.4/mongodb-nodeport.yaml -n ${options.namespace}`,
             );
-          if (options.nodeSelector)
+          if (options.nodeName)
             Underpost.cluster.pinToNode({
               name: statefulSetName,
               namespace: options.namespace,
-              node: options.nodeSelector,
+              node: options.nodeName,
             });
           if (serviceHost)
             Underpost.cluster.syncServiceConnectionEnv({
@@ -518,6 +512,10 @@ EOF
           });
       }
 
+      // After both StatefulSet branches, so one invocation can bring up MongoDB and its inspector
+      // and the client reads the auth mode from the set this run just applied.
+      if (options.mongoExpress) await MongoExpress.deploy({ namespace: options.namespace, underpostRoot, options });
+
       // Installing one stack must never break the other. Whichever is already
       // present decides whether this install takes the node's 80/443 for itself
       // or joins the underpost ingress, so it is read before anything is applied.
@@ -724,13 +722,107 @@ EOF
       return { type, ready, nodes };
     },
 
+    /**
+     * @method discoverLocalPathRoots
+     * @description Reads the node directories the local-path provisioner writes into, from its
+     * live ConfigMap. Every dynamically provisioned claim in this platform (Grafana, Prometheus,
+     * Alertmanager, IPFS, MariaDB) becomes a subdirectory of one of them, so the root has to be
+     * discovered rather than assumed — a wrong guess leaves those claims carrying the `/opt`
+     * policy default (`usr_t`), which no `container_t` pod can write.
+     * @returns {string[]} Configured node paths; empty when no provisioner is reachable.
+     * @memberof UnderpostCluster
+     */
+    discoverLocalPathRoots() {
+      for (const namespace of ContainerStorageService.LOCAL_PATH_NAMESPACES) {
+        const config = `${
+          shellExec(localPathConfigCommandFactory({ namespace }), {
+            stdout: true,
+            silent: true,
+            silentOnError: true,
+            disableLog: true,
+          }) || ''
+        }`.trim();
+        const roots = localPathRootsFactory(config);
+        if (roots.length > 0) return roots;
+      }
+      return [];
+    },
+
+    /**
+     * @method ensureContainerStorageContexts
+     * @description Makes SELinux compatibility an invariant of the storage lifecycle rather than
+     * a one-time bring-up step: creates the node directories this platform hands to containers
+     * (only when missing, so live data trees keep their ownership and mode), registers the
+     * persistent `container_file_t` mapping for each, and restores the trees.
+     *
+     * Idempotent and safe to call on every deploy. Never relabels paths outside the discovered
+     * set, and never changes SELinux mode.
+     * @param {object} [options]
+     * @param {boolean} [options.controlPlane=false] - Also cover the kubeadm control-plane trees
+     *   bind-mounted into containers (`/etc/kubernetes`, `/var/lib/etcd`, `/var/lib/calico`).
+     * @param {string[]} [options.extraPaths=[]] - Additional node directories to include.
+     * @returns {string[]} The roots that were prepared.
+     * @memberof UnderpostCluster
+     */
+    ensureContainerStorageContexts(options = {}) {
+      const roots = containerStorageRootsFactory({
+        localPathRoots: Underpost.cluster.discoverLocalPathRoots(),
+        extraPaths: options.extraPaths || [],
+        controlPlane: options.controlPlane === true,
+      });
+      logger.info('Preparing container-mounted host storage', { roots });
+      // HOST_VOLUME_ROOT is the one root the deploy flow writes into unprivileged (it lives in the
+      // operator's home tree and receives each hostPath volume's contents), so it is created owned
+      // by the invoking user. The rest stay root-owned: their subdirectories are chowned by each
+      // workload's own init container, which is the single owner of that concern.
+      const { uid, gid } = os.userInfo();
+      const operatorRoots = roots.filter((root) => root === HOST_VOLUME_ROOT);
+      const systemRoots = roots.filter((root) => root !== HOST_VOLUME_ROOT);
+      if (systemRoots.length > 0) ensureContainerStorage(systemRoots, { execute: shellExec });
+      if (operatorRoots.length > 0)
+        ensureContainerStorage(operatorRoots, { execute: shellExec, owner: `${uid}:${gid}` });
+      return roots;
+    },
+
+    /**
+     * @method ensureLocalPathProvisioner
+     * @description Ensures dynamic local-path PVC provisioning is available.
+     * @returns {string} Namespace containing the ready provisioner.
+     * @memberof UnderpostCluster
+     */
+    ensureLocalPathProvisioner() {
+      const locations = ['local-path-storage', 'kube-system'];
+      let namespace = locations.find((candidate) =>
+        `${
+          shellExec(`kubectl get deployment local-path-provisioner -n ${candidate} -o name`, {
+            stdout: true,
+            silent: true,
+            silentOnError: true,
+            disableLog: true,
+          }) || ''
+        }`.trim(),
+      );
+
+      if (!namespace) {
+        logger.info('Installing local-path-provisioner');
+        shellExec(`kubectl apply -f ${LOCAL_PATH_PROVISIONER_MANIFEST}`);
+        namespace = 'local-path-storage';
+      }
+      shellExec(`kubectl rollout status deployment/local-path-provisioner -n ${namespace} --timeout=5m`);
+      // The provisioner's root exists (or is about to) only now, and its helper pod creates each
+      // claim directory by inheriting that root's label. Prepare it here so the first dynamically
+      // provisioned claim already lands on `container_file_t` instead of `/opt`'s `usr_t`.
+      Underpost.cluster.ensureContainerStorageContexts();
+      return namespace;
+    },
+
     /**
      * @method pinToNode
      * @description Pins a workload to a specific Kubernetes node by patching its
      * pod template with a `kubernetes.io/hostname` nodeSelector. General-purpose;
-     * currently used to place the MongoDB 4.4 / Valkey StatefulSets on a chosen
-     * node (`--node-selector`). The patch triggers a rolling reschedule onto the
-     * target node.
+     * currently used to place the MongoDB 4.4 / Valkey StatefulSets and the
+     * observability Deployments on a chosen node (`--node-name`). The patch
+     * triggers a rolling reschedule onto the target node.
      * @param {object} params
      * @param {string} [params.kind='statefulset'] - Workload kind to patch.
      * @param {string} params.name - Workload name.
@@ -1383,6 +1475,16 @@ EOF
     config(options = { underpostRoot: '.' }) {
       const { underpostRoot } = options;
       console.log('Applying host configuration: SELinux, Docker, Containerd, and Sysctl settings.');
+      // Label before enforcing, never after: flipping the mode first would put the host into
+      // Enforcing with the storage trees still carrying the policy default, and every pod that
+      // mounts one would start failing for real instead of only logging a denial.
+      //
+      // Registered on every run, not only at first cluster creation: the node directories backing
+      // PersistentVolumes are recreated by resets, by `DirectoryOrCreate`, and by the local-path
+      // helper pod long after `kubeadm init` ran, and each of those inherits the policy default
+      // unless the mapping is already in the store.
+      ensureSELinuxTooling();
+      Underpost.cluster.ensureContainerStorageContexts();
       enforceSELinux(['/var/lib/kubelet', '/var/lib/containerd']);
 
       // Enable and start Docker and Kubelet services
@@ -1439,8 +1541,11 @@ EOF
       // forever (the upstream unit ships TimeoutStartSec=0).
       shellExec(`if systemctl is-active --quiet firewalld; then sudo systemctl disable --now firewalld; fi`);
 
+      // Storage labeling precedes the Enforcing flip, so the mode is never activated over trees
+      // that still carry the policy default.
+      ensureSELinuxTooling();
+      Underpost.cluster.ensureContainerStorageContexts();
       enforceSELinux(K3S_SELINUX_PATHS);
-      shareWithContainers(HOST_VOLUME_PATHS);
 
       // Disable swap. `swapoff -a` is a no-op without swap; the sed only edits
       // fstab when a swap line is present.
@@ -1714,6 +1819,10 @@ fi`);
       else logger.info('  -> Skipping (pass --remove-volume-host-paths to enable).');
 
       logger.info('Phase 2/7: Enforcing SELinux and restoring runtime contexts...');
+      // The hostPath cleanup above emptied the volume trees; re-asserting the mapping here means
+      // a rebuild starts from correctly labeled storage instead of inheriting the policy default.
+      ensureSELinuxTooling();
+      Underpost.cluster.ensureContainerStorageContexts();
       enforceSELinux(['/var/lib/kubelet', '/var/lib/containerd']);
 
       logger.info('Phase 3/7: Stopping host kubelet and container runtimes (kubeadm-scope only)...');
@@ -1916,7 +2025,7 @@ EOF`);
 
       // SOPS and Age for Git-native encrypted secret manifests. Owned by UnderpostSecret so the
       // same idempotent install backs `underpost secret --install-tools`.
-      Underpost.secret.sops.installTooling();
+      Underpost.secret.installTooling();
 
       // Install snap
       shellExec(`sudo yum install -y snapd`);