cyberia 3.2.22 → 3.2.70

package/src/cli/cluster.js

@@ -691,7 +691,9 @@ EOF
       // shellExec(
       //   `sudo sed -i '/SystemdCgroup = true/a       selinux_disabled = true' /etc/containerd/config.toml`,
       // );
-      shellExec(`sudo service docker restart`); // Restart docker after containerd config changes
+      // Restart docker after containerd config changes. Rocky 9 uses systemctl,
+      // not the legacy service command.
+      shellExec(`sudo systemctl restart docker || sudo service docker restart || true`);
       shellExec(`sudo systemctl enable --now containerd.service`);
       shellExec(`sudo systemctl restart containerd`); // Restart containerd to apply changes
 
@@ -874,6 +876,74 @@ net.ipv4.ip_forward = 1' | sudo tee /etc/sysctl.d/99-k3s.conf > /dev/null`,
       }
     },
 
+    /**
+     * @method _pruneContainerCaches
+     * @description Reclaims container-runtime disk left behind after a cluster
+     * teardown: stopped containers, unused images, build cache and anonymous
+     * volumes. Best-effort across every runtime present on the host (docker,
+     * podman, and optionally the CRI runtime via crictl). Each block is guarded
+     * by a command-existence check, so it is safe to call when a runtime is
+     * absent. This is what turns a "cluster deleted" into "disk actually freed":
+     * `kind delete` / `kubeadm reset` remove the cluster but leave gigabytes of
+     * images and overlay layers under /var/lib/{docker,containers}.
+     * @param {object} [options]
+     * @param {boolean} [options.all=true] - Remove all unused images, not just dangling ones.
+     * @param {boolean} [options.crictl=false] - Also prune the CRI runtime via crictl.
+     * @param {string} [options.criSocket] - Optional crictl --runtime-endpoint override.
+     * @private
+     */
+    _pruneContainerCaches(options = {}) {
+      const all = options.all !== false;
+      const a = all ? '-a ' : '';
+      logger.info(`  -> Pruning container-runtime caches (all=${all})...`);
+      // Docker (also matches the podman-docker shim when docker is symlinked to podman).
+      shellExec(
+        `if command -v docker >/dev/null 2>&1; then sudo docker system prune ${a}--volumes -f; sudo docker builder prune ${a}-f; fi`,
+        { silentOnError: true },
+      );
+      // Podman native — on this host images/overlays live under /var/lib/containers/storage.
+      shellExec(
+        `if command -v podman >/dev/null 2>&1; then sudo podman system prune ${a}--volumes -f; fi`,
+        { silentOnError: true },
+      );
+      if (options.crictl) {
+        const ep = options.criSocket ? `--runtime-endpoint ${options.criSocket} ` : '';
+        shellExec(
+          `if command -v crictl >/dev/null 2>&1; then sudo env PATH="$PATH:/usr/local/bin:/usr/bin" crictl ${ep}rmi --prune; fi`,
+          { silentOnError: true },
+        );
+      }
+      Underpost.cluster._unmountOrphanContainerOverlays();
+    },
+
+    /**
+     * @method _unmountOrphanContainerOverlays
+     * @description Unmounts leaked container overlay 'merged' mounts under
+     * /var/lib/{containers/storage,docker}/overlay. Pruning images/containers
+     * frees the backing data but leaves these mountpoints attached, so they
+     * keep showing up as identical `overlay ... /merged` rows in `df -h` long
+     * after the containers are gone. Only overlays NOT backing a still-existing
+     * container are unmounted, so running workloads are never disturbed.
+     * Best-effort; safe to call when no runtime is present.
+     * @private
+     */
+    _unmountOrphanContainerOverlays() {
+      logger.info('  -> Unmounting orphaned container overlay mounts...');
+      shellExec(`if command -v podman >/dev/null 2>&1; then sudo podman umount --all >/dev/null 2>&1 || true; fi`, {
+        silentOnError: true,
+      });
+      shellExec(
+        `active="$(sudo podman ps -aq 2>/dev/null | xargs -r -I{} sudo podman inspect --format '{{.GraphDriver.Data.MergedDir}}' {} 2>/dev/null || true)"
+findmnt -rn -o TARGET 2>/dev/null | grep -E '/var/lib/(containers/storage|docker)/overlay.*/merged' | sort -r | while IFS= read -r m; do
+  if ! printf '%s\\n' "$active" | grep -qxF "$m"; then
+    echo "Unmounting orphaned overlay: $m"
+    sudo umount -l "$m" 2>/dev/null || true
+  fi
+done`,
+        { silentOnError: true },
+      );
+    },
+
     /**
      * @method _lazyUmountKubeletMounts
      * @description Lazy-unmounts every mount under /var/lib/kubelet so a
@@ -908,7 +978,7 @@ net.ipv4.ip_forward = 1' | sudo tee /etc/sysctl.d/99-k3s.conf > /dev/null`,
       if (options.removeVolumeHostPaths) Underpost.cluster._cleanHostPathPvs();
       else logger.info('  -> Skipping (pass --remove-volume-host-paths to enable).');
 
-      logger.info('Phase 3/5: Deleting all Kind clusters...');
+      logger.info('Phase 3/6: Deleting all Kind clusters...');
       shellExec(`clusters=$(kind get clusters)
 if [ -n "$clusters" ]; then
   for c in $clusters; do
@@ -917,11 +987,14 @@ if [ -n "$clusters" ]; then
   done
 fi`);
 
-      logger.info('Phase 4/5: Cleaning kubeconfig and Kind Docker networks...');
+      logger.info('Phase 4/6: Cleaning kubeconfig and Kind Docker networks...');
       shellExec(`rm -rf "$HOME/.kube"`);
       Underpost.cluster.recoverKindDockerNetworks();
 
-      logger.info('Phase 5/5: Re-applying host configuration (Docker, containerd, sysctl).');
+      logger.info('Phase 5/6: Pruning container-runtime caches (kindest/node images, build cache, volumes)...');
+      Underpost.cluster._pruneContainerCaches({ all: true });
+
+      logger.info('Phase 6/6: Re-applying host configuration (Docker, containerd, sysctl).');
       Underpost.cluster.config();
 
       logger.info('=== KIND SAFE RESET COMPLETE ===');
@@ -993,7 +1066,7 @@ fi`);
       shellExec(`if ip link show tunl0 >/dev/null 2>&1; then sudo ip link del tunl0; fi`);
       shellExec(`sudo iptables -F`);
       shellExec(`sudo iptables -t nat -F`);
-      shellExec(`if command -v crictl >/dev/null 2>&1; then sudo crictl rmi --prune; fi`);
+      Underpost.cluster._pruneContainerCaches({ all: true, crictl: true });
 
       logger.info('Phase 7/7: Re-applying host configuration (Docker, containerd, sysctl).');
       Underpost.cluster.config();
@@ -1040,6 +1113,8 @@ fi`);
       shellExec(`if [ -d /etc/rancher/k3s ]; then sudo rm -rf /etc/rancher/k3s; fi`);
       shellExec(`if ip link show flannel.1 >/dev/null 2>&1; then sudo ip link del flannel.1; fi`);
       shellExec(`if ip link show cni0 >/dev/null 2>&1; then sudo ip link del cni0; fi`);
+      // k3s-uninstall.sh removes /var/lib/rancher/k3s; still prune any host docker/podman leftovers.
+      Underpost.cluster._pruneContainerCaches({ all: true });
 
       logger.info('Phase 5/5: Re-applying minimal K3s host config.');
       Underpost.cluster.configMinimalK3s();

package/src/cli/deploy.js

@@ -383,6 +383,10 @@ spec:
      * @param {string} [options.imagePullPolicy] - Container imagePullPolicy override (`Always`, `IfNotPresent`, `Never`); forwarded to deploymentYamlPartsFactory. Defaults to `Never` for `localhost/` images and `IfNotPresent` otherwise.
      * @param {boolean} [options.disableRuntimeProbes] - Omit internal-status HTTP probes from generated manifests. When true no readiness/liveness/startup probes are emitted.
      * @param {boolean} [options.tcpProbes] - Emit legacy TCP socket probes instead of HTTP internal-status probes (migration path).
+     * @param {string} [options.node] - Explicit target node for hostPath PV nodeAffinity pinning; resolved through {@link UnderpostDeploy.resolveDeployNode} together with the cluster flags.
+     * @param {boolean} [options.kind] - Kind cluster context; affects the cluster-type node default when no explicit node is set.
+     * @param {boolean} [options.kubeadm] - Kubeadm cluster context; affects the cluster-type node default when no explicit node is set.
+     * @param {boolean} [options.k3s] - K3s cluster context; affects the cluster-type node default when no explicit node is set.
      * @returns {Promise<void>} - Promise that resolves when the manifest is built.
      * @memberof UnderpostDeploy
      */
@@ -455,6 +459,15 @@ ${Underpost.deploy
           ? JSON.parse(fs.readFileSync(`./engine-private/conf/${deployId}/conf.volume.json`, 'utf8'))
           : [];
         if (confVolume.length > 0) {
+          // Mirror deployVolume's data-node resolution so the generated manifest
+          // pins the PV to the same node that physically receives the volume data.
+          const pvDataNode = Underpost.deploy.resolveDeployNode({
+            node: options.node,
+            kind: options.kind,
+            kubeadm: options.kubeadm,
+            k3s: options.k3s,
+            env,
+          });
           let volumeYaml = '';
           for (const deploymentVersion of deploymentVersions) {
             for (const volume of confVolume) {
@@ -466,6 +479,7 @@ ${Underpost.deploy
                 pvcId,
                 namespace: options.namespace,
                 hostPath,
+                nodeName: pvDataNode,
               })}\n`;
             }
           }
@@ -548,7 +562,7 @@ ${Underpost.deploy
           const yamlPath = `./engine-private/conf/${deployId}/build/${env}/secret.yaml`;
           fs.writeFileSync(yamlPath, secretYaml, 'utf8');
         } else {
-          const deploymentsFiles = ['Dockerfile', 'proxy.yaml', 'deployment.yaml', 'pv-pvc.yaml'];
+          const deploymentsFiles = ['Dockerfile', 'proxy.yaml', 'deployment.yaml', 'pv-pvc.yaml', 'grpc-service.yaml'];
           for (const file of deploymentsFiles) {
             if (fs.existsSync(`./engine-private/conf/${deployId}/build/${env}/${file}`)) {
               fs.copyFileSync(
@@ -720,7 +734,8 @@ spec:
      * @param {string} options.image - Docker image for the deployment.
      * @param {string} options.traffic - Traffic status for the deployment.
      * @param {string} options.replicas - Number of replicas for the deployment.
-     * @param {string} options.node - Node name for resource allocation.
+     * @param {string} options.node - Explicit target node (highest precedence in the node chain). When empty, {@link UnderpostDeploy.resolveDeployNode} falls back to the cluster-type default (`kind-worker` for kind, host for kubeadm/k3s). Used for both volume placement and hostPath PV nodeAffinity.
+     * @param {string} [options.sshKeyPath] - Private key path for node SSH operations, forwarded to deployVolume when shipping a hostPath volume to a remote target node over SSH. Defaults to engine-private/deploy/id_rsa.
      * @param {boolean} options.disableUpdateDeployment - Whether to disable deployment updates.
      * @param {boolean} options.disableUpdateProxy - Whether to disable proxy updates.
      * @param {boolean} options.disableDeploymentProxy - Whether to disable deployment proxy.
@@ -938,9 +953,16 @@ EOF`);
                   env,
                   version,
                   namespace,
-                  nodeName: options.node ? options.node : env === 'development' ? 'kind-worker' : os.hostname(),
+                  nodeName: Underpost.deploy.resolveDeployNode({
+                    node: options.node,
+                    kind: options.kind,
+                    kubeadm: options.kubeadm,
+                    k3s: options.k3s,
+                    env,
+                  }),
                   clusterContext: options.k3s ? 'k3s' : options.kubeadm ? 'kubeadm' : 'kind',
                   gitClean: options.gitClean || false,
+                  sshKeyPath: options.sshKeyPath || '',
                 });
           }
 
@@ -969,8 +991,12 @@ EOF`);
           if (
             Underpost.deploy.isValidTLSContext({ host: Object.keys(confServer)[0], env, options }) &&
             !options.selfSigned
-          )
-            shellExec(`sudo kubectl apply -f ./${manifestsPath}/secret.yaml -n ${namespace}`);
+          ) {
+            const secretPath = `./${manifestsPath}/secret.yaml`;
+            if (fs.existsSync(secretPath) && fs.readFileSync(secretPath, 'utf8').trim()) {
+              shellExec(`sudo kubectl apply -f ${secretPath} -n ${namespace}`);
+            } else logger.info('Skipping secret.yaml apply (no objects yet; applied by the --cert step)');
+          }
         }
       }
     },
@@ -983,10 +1009,17 @@ EOF`);
      */
     configMap(env, namespace = 'default') {
       const cronDeployId = cronDeployIdResolve() || 'dd-cron';
+      const envFilePath = `/home/dd/engine/engine-private/conf/${cronDeployId}/.env.${env}`;
+      // `--from-env-file` turns every KEY=VALUE into a secret key that the Deployment injects via
+      // `envFrom`. Strip shell/runtime-critical keys (notably PATH) first — an injected PATH
+      // overrides the image's own and breaks coreutils/sudo resolution inside the pod.
+      const sanitizedEnvPath = `${envFilePath}.secret`;
+      fs.writeFileSync(sanitizedEnvPath, Underpost.secret.sanitizeSecretEnvFile(fs.readFileSync(envFilePath, 'utf8')));
       shellExec(`kubectl delete secret underpost-config -n ${namespace} --ignore-not-found`);
       shellExec(
-        `kubectl create secret generic underpost-config --from-env-file=/home/dd/engine/engine-private/conf/${cronDeployId}/.env.${env} --dry-run=client -o yaml | kubectl apply -f - -n ${namespace}`,
+        `kubectl create secret generic underpost-config --from-env-file=${sanitizedEnvPath} --dry-run=client -o yaml | kubectl apply -f - -n ${namespace}`,
       );
+      fs.removeSync(sanitizedEnvPath);
     },
     /**
      * Switches the traffic for a deployment.
@@ -1032,6 +1065,37 @@ EOF`);
       Underpost.env.set(`${deployId}-${env}-traffic`, targetTraffic);
     },
 
+    /**
+     * Resolves the effective target node for a deployment, applying a single
+     * precedence chain shared by every deploy workflow — the default `deploy`
+     * callback, `run sync`, and custom `run instance` — so node customization
+     * behaves identically everywhere:
+     *
+     *   1. **Explicit node** — `node` (the resolved `--node` value). Upstream
+     *      runners derive it from the comma-path field or `--node-name`
+     *      (`run sync`: `path.split(',')[4]` > `--node-name` > default) and from
+     *      `--node-name` directly (`run instance`).
+     *   2. **Cluster-type default** — when no explicit node is given: `kind-worker`
+     *      for a kind cluster (the node that hosts kind hostPath volumes),
+     *      otherwise the control-plane / current host (`os.hostname()`) for
+     *      kubeadm / k3s. With no explicit cluster flag, `development` is treated
+     *      as kind and `production` as the host, preserving legacy behaviour.
+     *
+     * @param {object} params
+     * @param {string} [params.node=''] - Explicit node (`--node`); highest precedence.
+     * @param {boolean} [params.kind=false] - Kind cluster context.
+     * @param {boolean} [params.kubeadm=false] - Kubeadm cluster context.
+     * @param {boolean} [params.k3s=false] - K3s cluster context.
+     * @param {string} [params.env=''] - Deployment environment; tie-breaker when no cluster flag is set.
+     * @returns {string} The effective node name.
+     * @memberof UnderpostDeploy
+     */
+    resolveDeployNode({ node = '', kind = false, kubeadm = false, k3s = false, env = '' } = {}) {
+      if (node) return node;
+      const isKind = kind || (!kubeadm && !k3s && env !== 'production');
+      return isKind ? 'kind-worker' : os.hostname();
+    },
+
     /**
      * Deploys a volume for a deployment.
      * @param {object} volume - Volume configuration.
@@ -1043,9 +1107,10 @@ EOF`);
      * @param {string} options.env - Environment for the deployment.
      * @param {string} options.version - Version of the deployment.
      * @param {string} options.namespace - Kubernetes namespace for the deployment.
-     * @param {string} options.nodeName - Node name for the deployment.
+     * @param {string} options.nodeName - Effective target node (already resolved via {@link UnderpostDeploy.resolveDeployNode}). The volume data is written/shipped here and the PV is pinned to it; an empty value falls back to the cluster-type default inside this method.
      * @param {string} [options.clusterContext='kind'] - Cluster context type ('kind', 'kubeadm', or 'k3s').
      * @param {boolean} [options.gitClean=false] - Whether to run git clean on volumeMountPath before copying.
+     * @param {string} [options.sshKeyPath=''] - Private key path used when the target node is remote and the volume is shipped over SSH. Empty falls back to copyDirToNode's default (engine-private/deploy/id_rsa).
      * @memberof UnderpostDeploy
      */
     deployVolume(
@@ -1058,6 +1123,7 @@ EOF`);
         nodeName: '',
         clusterContext: 'kind',
         gitClean: false,
+        sshKeyPath: '',
       },
     ) {
       if (!volume.claimName) {
@@ -1072,16 +1138,48 @@ EOF`);
       if (options.gitClean && volume.volumeMountPath) {
         Underpost.repo.clean({ paths: [volume.volumeMountPath] });
       }
+      // The node that physically receives the volume data. hostPath volumes are
+      // node-local, so the data must land on the node where the pod will run, and
+      // the PV is pinned there (nodeAffinity) so the scheduler co-locates the pod
+      // with its volume — never mounting an empty DirectoryOrCreate on another node.
+      let dataNode;
       if (clusterContext === 'kind') {
         const kindNode = options.nodeName || 'kind-worker';
+        dataNode = kindNode;
         shellExec(`docker exec -i ${kindNode} bash -c "mkdir -p ${rootVolumeHostPath}"`);
         shellExec(`tar -C ${volume.volumeMountPath} -c . | docker cp - ${kindNode}:${rootVolumeHostPath}`);
         shellExec(
           `docker exec -i ${kindNode} bash -c "chown -R 1000:1000 ${rootVolumeHostPath}; chmod -R 755 ${rootVolumeHostPath}"`,
         );
       } else {
-        if (!fs.existsSync(rootVolumeHostPath)) fs.mkdirSync(rootVolumeHostPath, { recursive: true });
-        fs.copySync(volume.volumeMountPath, rootVolumeHostPath);
+        const localHost = os.hostname();
+        dataNode = options.nodeName || localHost;
+        if (dataNode === localHost) {
+          // Target node is the control plane / current host: write directly.
+          if (!fs.existsSync(rootVolumeHostPath)) fs.mkdirSync(rootVolumeHostPath, { recursive: true });
+          fs.copySync(volume.volumeMountPath, rootVolumeHostPath);
+        } else {
+          // Target node is remote: fs.copySync would only write the control-plane
+          // filesystem, leaving the real node's hostPath empty. Ship the folder to
+          // the node over SSH so the data exists where the pod is pinned.
+          const nodeHost =
+            shellExec(
+              `kubectl get node ${dataNode} -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}'`,
+              { stdout: true, silent: true, silentOnError: true },
+            ).trim() || dataNode;
+          logger.info('Shipping volume to remote node over SSH', {
+            node: dataNode,
+            host: nodeHost,
+            src: volume.volumeMountPath,
+            dest: rootVolumeHostPath,
+          });
+          Underpost.ssh.copyDirToNode({
+            host: nodeHost,
+            localDir: volume.volumeMountPath,
+            remoteDir: rootVolumeHostPath,
+            ...(options.sshKeyPath ? { keyPath: options.sshKeyPath } : {}),
+          });
+        }
       }
       shellExec(`kubectl delete pvc ${pvcId} -n ${namespace} --ignore-not-found`);
       shellExec(`kubectl delete pv ${pvId} --ignore-not-found`);
@@ -1090,6 +1188,7 @@ ${Underpost.deploy.persistentVolumeFactory({
   hostPath: rootVolumeHostPath,
   pvcId,
   namespace,
+  nodeName: dataNode,
 })}
 EOF
 `);
@@ -1178,11 +1277,28 @@ ${secret ? `          readOnly: true\n` : ''}`;
      * @param {string} options.hostPath - Host path for the persistent volume.
      * @param {string} options.pvcId - Persistent volume claim ID.
      * @param {string} [options.namespace='default'] - Kubernetes namespace for the PVC claimRef.
+     * @param {string} [options.nodeName=''] - Node name to which the persistent volume is pinned (optional).
      * @returns {string} - YAML configuration for the persistent volume and claim.
      * @memberof UnderpostDeploy
      */
-    persistentVolumeFactory({ hostPath, pvcId, namespace = 'default' }) {
+    persistentVolumeFactory({ hostPath, pvcId, namespace = 'default', nodeName = '' }) {
       const pvId = pvcId.replace(/^pvc-/, 'pv-');
+      // hostPath volumes are node-local: deployVolume writes the content to the
+      // filesystem of a single node. Without nodeAffinity the scheduler can place
+      // the pod on a different node and mount an empty DirectoryOrCreate hostPath
+      // (missing the materialized assets). Pin the PV to the node that holds the
+      // data so the pod is always co-located with its volume.
+      const nodeAffinity = nodeName
+        ? `
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+        - matchExpressions:
+            - key: kubernetes.io/hostname
+              operator: In
+              values:
+                - ${nodeName}`
+        : '';
       return `apiVersion: v1
 kind: PersistentVolume
 metadata:
@@ -1193,7 +1309,7 @@ spec:
   accessModes:
     - ReadWriteOnce
   persistentVolumeReclaimPolicy: Retain
-  storageClassName: manual
+  storageClassName: manual${nodeAffinity}
   claimRef:
     apiVersion: v1
     kind: PersistentVolumeClaim