cyberia 3.2.12 → 3.2.70

package/src/cli/deploy.js

@@ -18,9 +18,9 @@ import {
 } from '../server/conf.js';
 import { loggerFactory } from '../server/logger.js';
 import { shellExec } from '../server/process.js';
+import { INTERNAL_READY_PATH, INTERNAL_HEALTH_PATH } from '../server/runtime-status.js';
 import fs from 'fs-extra';
 import dotenv from 'dotenv';
-import { timer } from '../client/components/core/CommonJs.js';
 import os from 'node:os';
 import Underpost from '../index.js';
 
@@ -114,6 +114,64 @@ class UnderpostDeploy {
       )
       .join('')}`;
     },
+    /**
+     * Builds Kubernetes probes that gate on the in-pod internal status endpoint.
+     *
+     * HTTP mode (default) aligns Kubernetes pod readiness with actual Underpost
+     * runtime readiness:
+     *   - readinessProbe → GET /_internal/ready  (200 only when running-deployment)
+     *   - livenessProbe  → GET /_internal/health (deadlock / hung-process detection)
+     *   - startupProbe   → GET /_internal/ready  (long window for hot-built/slow boots)
+     *
+     * Migration: pass `useHttp: false` to emit the legacy TCP socket probes
+     * (port-bound only) for deployments not yet serving the internal endpoint.
+     *
+     * @param {object} opts
+     * @param {number} opts.port - In-pod internal status port (deployment base PORT).
+     * @param {boolean} [opts.useHttp=true] - Emit HTTP probes; false → legacy TCP.
+     * @param {boolean} [opts.liveness=true] - Include a livenessProbe.
+     * @param {boolean} [opts.startup=true] - Include a startupProbe.
+     * @returns {{readinessProbe: object, livenessProbe?: object, startupProbe?: object}}
+     * @memberof UnderpostDeploy
+     */
+    runtimeProbesFactory({ port, useHttp = true, liveness = true, startup = true } = {}) {
+      if (!port) return {};
+      if (!useHttp) {
+        const tcp = { tcpSocket: { port }, initialDelaySeconds: 5, periodSeconds: 10, failureThreshold: 6 };
+        const probes = { readinessProbe: tcp };
+        if (liveness) probes.livenessProbe = { ...tcp, initialDelaySeconds: 30 };
+        return probes;
+      }
+      const probes = {
+        readinessProbe: {
+          httpGet: { path: INTERNAL_READY_PATH, port },
+          initialDelaySeconds: 5,
+          periodSeconds: 5,
+          timeoutSeconds: 3,
+          failureThreshold: 3,
+        },
+      };
+      if (liveness)
+        probes.livenessProbe = {
+          httpGet: { path: INTERNAL_HEALTH_PATH, port },
+          initialDelaySeconds: 30,
+          periodSeconds: 15,
+          timeoutSeconds: 3,
+          failureThreshold: 3,
+        };
+      if (startup)
+        // A startupProbe suspends readiness/liveness until it first succeeds, so
+        // its window bounds in-container hot builds and slow boots. 180 × 10s =
+        // 30 min before the pod is considered failed to start.
+        probes.startupProbe = {
+          httpGet: { path: INTERNAL_READY_PATH, port },
+          initialDelaySeconds: 10,
+          periodSeconds: 10,
+          timeoutSeconds: 3,
+          failureThreshold: 180,
+        };
+      return probes;
+    },
     /**
      * Creates a YAML deployment configuration for a deployment.
      * @param {string} deployId - Deployment ID for which the deployment is being created.
@@ -128,6 +186,11 @@ class UnderpostDeploy {
      * @param {boolean} skipFullBuild - Whether to skip the full client bundle build during deployment.
      * @param {boolean} pullBundle - Whether to pull the pre-built client bundle from Cloudinary before starting. Use together with skipFullBuild to skip the local build entirely.
      * @param {string} [imagePullPolicy] - Container imagePullPolicy override (`Always`, `IfNotPresent`, `Never`). When omitted, defaults to `Never` for `localhost/` images and `IfNotPresent` otherwise.
+     * @param {object} lifecycle - Kubernetes lifecycle hooks configuration for the deployment container.
+     * @param {object} readinessProbe - Kubernetes readiness probe configuration for the deployment container.
+     * @param {object} livenessProbe - Kubernetes liveness probe configuration for the deployment container.
+     * @param {object} startupProbe - Kubernetes startup probe configuration for the deployment container.
+     * @param {number} containerPort - Container port to expose for the deployment.
      * @returns {string} - YAML deployment configuration for the specified deployment.
      * @memberof UnderpostDeploy
      */
@@ -153,7 +216,12 @@ class UnderpostDeploy {
       lifecycle,
       readinessProbe,
       livenessProbe,
+      startupProbe,
       containerPort,
+      // Explicit, secret-free internal status port injected as an env var so the
+      // in-pod endpoint binds exactly what the probes and the monitor target,
+      // independent of the ambient `PORT` baked into the image/secret.
+      internalStatusPort,
     }) {
       if (!cmd)
         cmd =
@@ -205,12 +273,19 @@ spec:
             - secretRef:
                 name: underpost-config
 ${
-  containerPort
-    ? `          ports:
-            - containerPort: ${containerPort}
+  internalStatusPort
+    ? `          env:
+            - name: UNDERPOST_INTERNAL_PORT
+              value: "${internalStatusPort}"
 `
     : ''
 }${
+        containerPort
+          ? `          ports:
+            - containerPort: ${containerPort}
+`
+          : ''
+      }${
         resources
           ? `          resources:
             requests:
@@ -242,6 +317,15 @@ ${JSON.stringify(livenessProbe, null, 2)
   .split('\n')
   .map((l) => '            ' + l)
   .join('\n')}
+`
+          : ''
+      }${
+        startupProbe
+          ? `          startupProbe:
+${JSON.stringify(startupProbe, null, 2)
+  .split('\n')
+  .map((l) => '            ' + l)
+  .join('\n')}
 `
           : ''
       }${
@@ -283,18 +367,26 @@ spec:
      * @param {object} options - Options for the manifest build process.
      * @param {string} options.replicas - Number of replicas for each deployment.
      * @param {string} options.image - Docker image for the deployment.
-     * @param {string} options.namespace - Kubernetes namespace for the deployment.
+     * @param {string} options.namespace - Kubernetes namespace for the deployment (defaults to "default").
      * @param {string} [options.versions] - Comma-separated list of versions to deploy.
      * @param {string} [options.cmd] - Custom initialization command for deploymentYamlPartsFactory (comma-separated commands).
-     * @param {string} [options.timeoutResponse] - Timeout response setting for the deployment.
-     * @param {string} [options.timeoutIdle] - Timeout idle setting for the deployment.
-     * @param {string} [options.retryCount] - Retry count setting for the deployment.
-     * @param {string} [options.retryPerTryTimeout] - Retry per-try timeout setting for the deployment.
-     * @param {boolean} [options.disableDeploymentProxy] - Whether to disable deployment proxy.
-     * @param {string} [options.traffic] - Traffic status for the deployment.
-     * @param {boolean} [options.skipFullBuild] - Whether to skip the full client bundle build; forwarded to deploymentYamlPartsFactory to generate a pull-bundle startup command.
+     * @param {string} [options.timeoutResponse] - HTTPProxy per-route response timeout (e.g. "300000ms", "infinity").
+     * @param {string} [options.timeoutIdle] - HTTPProxy per-route idle timeout (e.g. "10s", "infinity").
+     * @param {string} [options.retryCount] - HTTPProxy per-route retry count (e.g. 3).
+     * @param {string} [options.retryPerTryTimeout] - HTTPProxy per-route per-try timeout (e.g. "150ms").
+     * @param {boolean} [options.disableDeploymentProxy] - Whether to disable deployment proxy route generation.
+     * @param {string} [options.traffic] - Comma-separated active traffic colour(s) used to select which versions receive traffic (e.g. "blue", "green").
+     * @param {boolean} [options.cert] - Whether to include cert-manager Certificate resources in secret.yaml (production only).
+     * @param {boolean} [options.selfSigned] - Whether to include TLS block in HTTPProxy using a pre-created self-signed secret. Enables HTTPS for development without cert-manager.
+     * @param {boolean} [options.skipFullBuild] - Whether to skip the full client bundle build; forwarded to deploymentYamlPartsFactory.
      * @param {boolean} [options.pullBundle] - Whether to pull the pre-built client bundle from Cloudinary; forwarded to deploymentYamlPartsFactory. Use together with skipFullBuild.
-     * @param {string} [options.imagePullPolicy] - Container imagePullPolicy override (`Always`, `IfNotPresent`, `Never`); forwarded to deploymentYamlPartsFactory. When omitted, the builder defaults to `Never` for `localhost/` images and `IfNotPresent` otherwise.
+     * @param {string} [options.imagePullPolicy] - Container imagePullPolicy override (`Always`, `IfNotPresent`, `Never`); forwarded to deploymentYamlPartsFactory. Defaults to `Never` for `localhost/` images and `IfNotPresent` otherwise.
+     * @param {boolean} [options.disableRuntimeProbes] - Omit internal-status HTTP probes from generated manifests. When true no readiness/liveness/startup probes are emitted.
+     * @param {boolean} [options.tcpProbes] - Emit legacy TCP socket probes instead of HTTP internal-status probes (migration path).
+     * @param {string} [options.node] - Explicit target node for hostPath PV nodeAffinity pinning; resolved through {@link UnderpostDeploy.resolveDeployNode} together with the cluster flags.
+     * @param {boolean} [options.kind] - Kind cluster context; affects the cluster-type node default when no explicit node is set.
+     * @param {boolean} [options.kubeadm] - Kubeadm cluster context; affects the cluster-type node default when no explicit node is set.
+     * @param {boolean} [options.k3s] - K3s cluster context; affects the cluster-type node default when no explicit node is set.
      * @returns {Promise<void>} - Promise that resolves when the manifest is built.
      * @memberof UnderpostDeploy
      */
@@ -319,6 +411,17 @@ spec:
 
         logger.info('port range', { deployId, fromPort, toPort });
 
+        // The internal status endpoint binds `fromPort - 1`: app instances bind
+        // the router range starting at `fromPort`, so this slot is always free
+        // inside the pod. It is injected into the pod env (UNDERPOST_INTERNAL_PORT)
+        // and used for both the probes and the monitor's port-forward target so
+        // all three agree regardless of the image's ambient PORT.
+        // Opt out with `--disable-runtime-probes` to keep legacy probe-less pods.
+        const internalPort = fromPort - 1;
+        const probes = options.disableRuntimeProbes
+          ? {}
+          : Underpost.deploy.runtimeProbesFactory({ port: internalPort, useHttp: !options.tcpProbes });
+
         let deploymentYamlParts = '';
         for (const deploymentVersion of deploymentVersions) {
           deploymentYamlParts += `---
@@ -334,6 +437,10 @@ ${Underpost.deploy
     skipFullBuild: options.skipFullBuild,
     pullBundle: options.pullBundle,
     imagePullPolicy: options.imagePullPolicy,
+    internalStatusPort: options.disableRuntimeProbes ? undefined : internalPort,
+    readinessProbe: probes.readinessProbe,
+    livenessProbe: probes.livenessProbe,
+    startupProbe: probes.startupProbe,
   })
   .replace('{{ports}}', buildKindPorts(fromPort, toPort))}
 `;
@@ -352,6 +459,15 @@ ${Underpost.deploy
           ? JSON.parse(fs.readFileSync(`./engine-private/conf/${deployId}/conf.volume.json`, 'utf8'))
           : [];
         if (confVolume.length > 0) {
+          // Mirror deployVolume's data-node resolution so the generated manifest
+          // pins the PV to the same node that physically receives the volume data.
+          const pvDataNode = Underpost.deploy.resolveDeployNode({
+            node: options.node,
+            kind: options.kind,
+            kubeadm: options.kubeadm,
+            k3s: options.k3s,
+            env,
+          });
           let volumeYaml = '';
           for (const deploymentVersion of deploymentVersions) {
             for (const volume of confVolume) {
@@ -363,6 +479,7 @@ ${Underpost.deploy
                 pvcId,
                 namespace: options.namespace,
                 hostPath,
+                nodeName: pvDataNode,
               })}\n`;
             }
           }
@@ -376,7 +493,7 @@ ${Underpost.deploy
           : [];
 
         for (const host of Object.keys(confServer)) {
-          if (env === 'production')
+          if (env === 'production' && options.cert === true)
             secretYaml += Underpost.deploy.buildCertManagerCertificate({ host, namespace: options.namespace });
 
           const pathPortAssignment = pathPortAssignmentData[host];
@@ -445,7 +562,7 @@ ${Underpost.deploy
           const yamlPath = `./engine-private/conf/${deployId}/build/${env}/secret.yaml`;
           fs.writeFileSync(yamlPath, secretYaml, 'utf8');
         } else {
-          const deploymentsFiles = ['Dockerfile', 'proxy.yaml', 'deployment.yaml', 'pv-pvc.yaml'];
+          const deploymentsFiles = ['Dockerfile', 'proxy.yaml', 'deployment.yaml', 'pv-pvc.yaml', 'grpc-service.yaml'];
           for (const file of deploymentsFiles) {
             if (fs.existsSync(`./engine-private/conf/${deployId}/build/${env}/${file}`)) {
               fs.copyFileSync(
@@ -579,6 +696,7 @@ spec:
      * @memberof UnderpostDeploy
      */
     baseProxyYamlFactory({ host, env, options }) {
+      const includeTls = env !== 'development' || options.selfSigned === true;
       return `
 ---
 apiVersion: projectcontour.io/v1
@@ -589,11 +707,11 @@ metadata:
 spec:
   virtualhost:
     fqdn: ${host}${
-      env === 'development'
-        ? ''
-        : `
+      includeTls
+        ? `
     tls:
       secretName: ${host}`
+        : ''
     }
   routes:`;
     },
@@ -609,35 +727,42 @@ spec:
      * @param {boolean} options.buildManifest - Whether to build the deployment manifest.
      * @param {boolean} options.infoUtil - Whether to display utility information.
      * @param {boolean} options.expose - Whether to expose the deployment.
-     * @param {boolean} options.cert - Whether to create certificates for the deployment.
-     * @param {string} options.certHosts - Comma-separated list of hosts for which to create certificates.
+     * @param {boolean} options.cert - Whether to create cert-manager Certificate resources for the deployment.
+     * @param {string} options.certHosts - Comma-separated list of hosts for which to create cert-manager certificates.
+     * @param {boolean} options.selfSigned - Use a pre-created self-signed TLS secret instead of cert-manager. The secret must already exist in the namespace with the same name as the host. Enables TLS in the Contour HTTPProxy virtualhost without requiring a production ClusterIssuer.
      * @param {string} options.versions - Comma-separated list of versions to deploy.
      * @param {string} options.image - Docker image for the deployment.
      * @param {string} options.traffic - Traffic status for the deployment.
      * @param {string} options.replicas - Number of replicas for the deployment.
-     * @param {string} options.node - Node name for resource allocation.
+     * @param {string} options.node - Explicit target node (highest precedence in the node chain). When empty, {@link UnderpostDeploy.resolveDeployNode} falls back to the cluster-type default (`kind-worker` for kind, host for kubeadm/k3s). Used for both volume placement and hostPath PV nodeAffinity.
+     * @param {string} [options.sshKeyPath] - Private key path for node SSH operations, forwarded to deployVolume when shipping a hostPath volume to a remote target node over SSH. Defaults to engine-private/deploy/id_rsa.
      * @param {boolean} options.disableUpdateDeployment - Whether to disable deployment updates.
      * @param {boolean} options.disableUpdateProxy - Whether to disable proxy updates.
      * @param {boolean} options.disableDeploymentProxy - Whether to disable deployment proxy.
      * @param {boolean} options.disableUpdateVolume - Whether to disable volume updates.
      * @param {boolean} options.status - Whether to display deployment status.
      * @param {boolean} options.disableUpdateUnderpostConfig - Whether to disable Underpost config updates.
-     * @param {string} [options.namespace] - Kubernetes namespace for the deployment.
-     * @param {string} [options.timeoutResponse] - Timeout response setting for the deployment.
-     * @param {string} [options.timeoutIdle] - Timeout idle setting for the deployment.
-     * @param {string} [options.retryCount] - Retry count setting for the deployment.
-     * @param {string} [options.retryPerTryTimeout] - Retry per-try timeout setting for the deployment.
-     * @param {string} [options.kindType] - Type of Kubernetes resource to retrieve information for.
-     * @param {number} [options.port] - Port number for exposing the deployment.
-     * @param {string} [options.cmd] - Custom initialization command for deploymentYamlPartsFactory (comma-separated commands).
-     * @param {number} [options.exposePort] - Local:remote port override when --expose is active (overrides auto-detected service port).
+     * @param {string} [options.namespace] - Kubernetes namespace for the deployment (defaults to "default").
+     * @param {string} [options.timeoutResponse] - HTTPProxy per-route response timeout (e.g. "300000ms", "infinity").
+     * @param {string} [options.timeoutIdle] - HTTPProxy per-route idle timeout (e.g. "10s", "infinity").
+     * @param {string} [options.retryCount] - HTTPProxy per-route retry count (e.g. 3).
+     * @param {string} [options.retryPerTryTimeout] - HTTPProxy per-route per-try timeout (e.g. "150ms").
+     * @param {string} [options.kindType] - Kubernetes resource kind to target when using --expose (defaults to "svc").
+     * @param {number} [options.port] - Port number override for exposing the deployment.
+     * @param {string} [options.cmd] - Custom initialization command (comma-separated) for deploymentYamlPartsFactory.
+     * @param {number} [options.exposePort] - Remote port override when --expose is active (overrides auto-detected service port). Used as both local and remote port unless exposeLocalPort is also set.
+     * @param {number} [options.exposeLocalPort] - Local port override for --expose (e.g. 80); remote port is still auto-detected. Enables /etc/hosts access without a port in the browser URL.
+     * @param {boolean} [options.localProxy] - When true (with --expose), forward all service TCP ports locally and start the Node.js path-routing proxy for full path-based routing (e.g. /wp alongside /).
+     * @param {boolean} [options.tls] - When true (with --expose --local-proxy), start the proxy on port 443 with TLS using self-signed certificates resolved from the local SSL store.
      * @param {boolean} [options.k3s] - Whether to use k3s cluster context.
      * @param {boolean} [options.kubeadm] - Whether to use kubeadm cluster context.
      * @param {boolean} [options.kind] - Whether to use kind cluster context.
      * @param {boolean} [options.gitClean] - Whether to run git clean on volume mount paths before copying.
      * @param {boolean} [options.skipFullBuild] - Whether to skip the full client bundle build; passed through to buildManifest/deploymentYamlPartsFactory.
      * @param {boolean} [options.pullBundle] - Whether to pull the pre-built client bundle from Cloudinary; passed through to buildManifest/deploymentYamlPartsFactory. Use together with skipFullBuild.
-     * @param {string} [options.imagePullPolicy] - Container imagePullPolicy override (`Always`, `IfNotPresent`, `Never`); passed through to buildManifest/deploymentYamlPartsFactory. When omitted, the builder defaults to `Never` for `localhost/` images and `IfNotPresent` otherwise.
+     * @param {string} [options.imagePullPolicy] - Container imagePullPolicy override (`Always`, `IfNotPresent`, `Never`); passed through to buildManifest/deploymentYamlPartsFactory. Defaults to `Never` for `localhost/` images and `IfNotPresent` otherwise.
+     * @param {boolean} [options.disableRuntimeProbes] - Omit internal-status HTTP probes from generated manifests. When true no readiness/liveness/startup probes are emitted.
+     * @param {boolean} [options.tcpProbes] - Emit legacy TCP socket probes instead of HTTP internal-status probes.
      * @returns {Promise<void>} - Promise that resolves when the deployment process is complete.
      * @memberof UnderpostDeploy
      */
@@ -672,6 +797,10 @@ spec:
         kindType: '',
         port: 0,
         exposePort: 0,
+        exposeLocalPort: 0,
+        localProxy: false,
+        tls: false,
+        selfSigned: false,
         cmd: '',
         k3s: false,
         kubeadm: false,
@@ -757,20 +886,50 @@ EOF`);
             logger.error(`No ${kindType} found matching '${deployId}', skipping expose`);
             continue;
           }
-          const port = options.exposePort
-            ? parseInt(options.exposePort)
-            : options.port
-              ? parseInt(options.port)
-              : kindType !== 'svc'
-                ? 80
-                : parseInt(svc[`PORT(S)`].split('/TCP')[0]);
-          logger.info(deployId, {
-            svc,
-            port,
-          });
-          shellExec(`sudo kubectl port-forward -n ${namespace} ${kindType}/${svc.NAME} ${port}:${port}`, {
-            async: true,
-          });
+          if (options.localProxy) {
+            const svcPorts = [
+              ...new Set(
+                svc['PORT(S)']
+                  .split(',')
+                  .filter((p) => p.includes('/TCP'))
+                  .map((p) => parseInt(p.split(':')[0])),
+              ),
+            ];
+            for (const svcPort of svcPorts) {
+              shellExec(`sudo kubectl port-forward -n ${namespace} ${kindType}/${svc.NAME} ${svcPort}:${svcPort}`, {
+                async: true,
+              });
+            }
+            const envFile = `./engine-private/conf/${deployId}/.env.${env}`;
+            let basePort = svcPorts[0] - 1;
+            if (fs.existsSync(envFile)) {
+              const portMatch = fs.readFileSync(envFile, 'utf8').match(/^PORT=(\d+)/m);
+              if (portMatch) basePort = parseInt(portMatch[1]);
+            }
+            logger.info(deployId, { svc, svcPorts, basePort });
+            const tlsFlag = options.tls ? ' tls' : '';
+            shellExec(
+              `NODE_ENV=${env} PORT=${basePort} DEV_PROXY_PORT_OFFSET=0 node src/proxy proxy ${deployId} ${env}${tlsFlag}`,
+              { async: true },
+            );
+          } else {
+            const remotePort = options.exposePort
+              ? parseInt(options.exposePort)
+              : options.port
+                ? parseInt(options.port)
+                : kindType !== 'svc'
+                  ? 80
+                  : parseInt(svc[`PORT(S)`].split('/TCP')[0]);
+            const localPort = options.exposeLocalPort ? parseInt(options.exposeLocalPort) : remotePort;
+            logger.info(deployId, {
+              svc,
+              localPort,
+              remotePort,
+            });
+            shellExec(`sudo kubectl port-forward -n ${namespace} ${kindType}/${svc.NAME} ${localPort}:${remotePort}`, {
+              async: true,
+            });
+          }
           continue;
         }
 
@@ -794,9 +953,16 @@ EOF`);
                   env,
                   version,
                   namespace,
-                  nodeName: options.node ? options.node : env === 'development' ? 'kind-worker' : os.hostname(),
+                  nodeName: Underpost.deploy.resolveDeployNode({
+                    node: options.node,
+                    kind: options.kind,
+                    kubeadm: options.kubeadm,
+                    k3s: options.k3s,
+                    env,
+                  }),
                   clusterContext: options.k3s ? 'k3s' : options.kubeadm ? 'kubeadm' : 'kind',
                   gitClean: options.gitClean || false,
+                  sshKeyPath: options.sshKeyPath || '',
                 });
           }
 
@@ -822,63 +988,17 @@ EOF`);
           if (!options.disableUpdateProxy)
             shellExec(`sudo kubectl apply -f ./${manifestsPath}/proxy.yaml -n ${namespace}`);
 
-          if (Underpost.deploy.isValidTLSContext({ host: Object.keys(confServer)[0], env, options }))
-            shellExec(`sudo kubectl apply -f ./${manifestsPath}/secret.yaml -n ${namespace}`);
-        }
-      }
-    },
-    /**
-     * Checks the status of a deployment.
-     * @param {string} deployId - Deployment ID for which the status is being checked.
-     * @param {string} env - Environment for which the status is being checked.
-     * @param {string} traffic - Current traffic status for the deployment.
-     * @param {Array<string>} ignoresNames - List of pod names to ignore.
-     * @param {string} [namespace='default'] - Kubernetes namespace for the deployment.
-     * @returns {object} - Object containing the status of the deployment.
-     * @memberof UnderpostDeploy
-     */
-    async checkDeploymentReadyStatus(deployId, env, traffic, ignoresNames = [], namespace = 'default') {
-      const pods = Underpost.kubectl.get(`${deployId}-${env}-${traffic}`, 'pods', namespace);
-      const readyPods = [];
-      const notReadyPods = [];
-
-      // Readiness signal: the pod's Kubernetes `Ready` condition driven by the
-      // container's readinessProbe (TCP socket, HTTP get, or exec). Set by kubelet
-      // when the probe passes. A failed or crashing runtime never becomes Ready —
-      // kubelet surfaces CrashLoopBackOff and this gate stays closed.
-      for (const pod of pods) {
-        const { NAME } = pod;
-        if (ignoresNames && ignoresNames.find((t) => NAME.trim().toLowerCase().match(t.trim().toLowerCase()))) continue;
-
-        let podJson = null;
-        try {
-          // Pod may not exist yet (between deployment apply and pod
-          // scheduling). silentOnError lets the monitor loop continue
-          // instead of aborting on the transient NotFound exit.
-          const raw = shellExec(`sudo kubectl get pod ${NAME} -n ${namespace} -o json`, {
-            silent: true,
-            disableLog: true,
-            stdout: true,
-            silentOnError: true,
-          });
-          podJson = raw ? JSON.parse(raw) : null;
-        } catch (_) {
-          podJson = null;
+          if (
+            Underpost.deploy.isValidTLSContext({ host: Object.keys(confServer)[0], env, options }) &&
+            !options.selfSigned
+          ) {
+            const secretPath = `./${manifestsPath}/secret.yaml`;
+            if (fs.existsSync(secretPath) && fs.readFileSync(secretPath, 'utf8').trim()) {
+              shellExec(`sudo kubectl apply -f ${secretPath} -n ${namespace}`);
+            } else logger.info('Skipping secret.yaml apply (no objects yet; applied by the --cert step)');
+          }
         }
-        const conditions = podJson?.status?.conditions || [];
-        const readyCondition = conditions.find((c) => c.type === 'Ready');
-        const k8sReady = readyCondition?.status === 'True';
-
-        pod.out = JSON.stringify({ k8sReady, condition: readyCondition ?? null });
-
-        if (k8sReady) readyPods.push(pod);
-        else notReadyPods.push(pod);
       }
-      return {
-        ready: pods.length > 0 && notReadyPods.length === 0,
-        notReadyPods,
-        readyPods,
-      };
     },
     /**
      * Creates a Kubernetes Secret for a deployment (replaces configMap for secret data).
@@ -889,10 +1009,17 @@ EOF`);
      */
     configMap(env, namespace = 'default') {
       const cronDeployId = cronDeployIdResolve() || 'dd-cron';
+      const envFilePath = `/home/dd/engine/engine-private/conf/${cronDeployId}/.env.${env}`;
+      // `--from-env-file` turns every KEY=VALUE into a secret key that the Deployment injects via
+      // `envFrom`. Strip shell/runtime-critical keys (notably PATH) first — an injected PATH
+      // overrides the image's own and breaks coreutils/sudo resolution inside the pod.
+      const sanitizedEnvPath = `${envFilePath}.secret`;
+      fs.writeFileSync(sanitizedEnvPath, Underpost.secret.sanitizeSecretEnvFile(fs.readFileSync(envFilePath, 'utf8')));
       shellExec(`kubectl delete secret underpost-config -n ${namespace} --ignore-not-found`);
       shellExec(
-        `kubectl create secret generic underpost-config --from-env-file=/home/dd/engine/engine-private/conf/${cronDeployId}/.env.${env} --dry-run=client -o yaml | kubectl apply -f - -n ${namespace}`,
+        `kubectl create secret generic underpost-config --from-env-file=${sanitizedEnvPath} --dry-run=client -o yaml | kubectl apply -f - -n ${namespace}`,
       );
+      fs.removeSync(sanitizedEnvPath);
     },
     /**
      * Switches the traffic for a deployment.
@@ -938,6 +1065,37 @@ EOF`);
       Underpost.env.set(`${deployId}-${env}-traffic`, targetTraffic);
     },
 
+    /**
+     * Resolves the effective target node for a deployment, applying a single
+     * precedence chain shared by every deploy workflow — the default `deploy`
+     * callback, `run sync`, and custom `run instance` — so node customization
+     * behaves identically everywhere:
+     *
+     *   1. **Explicit node** — `node` (the resolved `--node` value). Upstream
+     *      runners derive it from the comma-path field or `--node-name`
+     *      (`run sync`: `path.split(',')[4]` > `--node-name` > default) and from
+     *      `--node-name` directly (`run instance`).
+     *   2. **Cluster-type default** — when no explicit node is given: `kind-worker`
+     *      for a kind cluster (the node that hosts kind hostPath volumes),
+     *      otherwise the control-plane / current host (`os.hostname()`) for
+     *      kubeadm / k3s. With no explicit cluster flag, `development` is treated
+     *      as kind and `production` as the host, preserving legacy behaviour.
+     *
+     * @param {object} params
+     * @param {string} [params.node=''] - Explicit node (`--node`); highest precedence.
+     * @param {boolean} [params.kind=false] - Kind cluster context.
+     * @param {boolean} [params.kubeadm=false] - Kubeadm cluster context.
+     * @param {boolean} [params.k3s=false] - K3s cluster context.
+     * @param {string} [params.env=''] - Deployment environment; tie-breaker when no cluster flag is set.
+     * @returns {string} The effective node name.
+     * @memberof UnderpostDeploy
+     */
+    resolveDeployNode({ node = '', kind = false, kubeadm = false, k3s = false, env = '' } = {}) {
+      if (node) return node;
+      const isKind = kind || (!kubeadm && !k3s && env !== 'production');
+      return isKind ? 'kind-worker' : os.hostname();
+    },
+
     /**
      * Deploys a volume for a deployment.
      * @param {object} volume - Volume configuration.
@@ -949,9 +1107,10 @@ EOF`);
      * @param {string} options.env - Environment for the deployment.
      * @param {string} options.version - Version of the deployment.
      * @param {string} options.namespace - Kubernetes namespace for the deployment.
-     * @param {string} options.nodeName - Node name for the deployment.
+     * @param {string} options.nodeName - Effective target node (already resolved via {@link UnderpostDeploy.resolveDeployNode}). The volume data is written/shipped here and the PV is pinned to it; an empty value falls back to the cluster-type default inside this method.
      * @param {string} [options.clusterContext='kind'] - Cluster context type ('kind', 'kubeadm', or 'k3s').
      * @param {boolean} [options.gitClean=false] - Whether to run git clean on volumeMountPath before copying.
+     * @param {string} [options.sshKeyPath=''] - Private key path used when the target node is remote and the volume is shipped over SSH. Empty falls back to copyDirToNode's default (engine-private/deploy/id_rsa).
      * @memberof UnderpostDeploy
      */
     deployVolume(
@@ -964,6 +1123,7 @@ EOF`);
         nodeName: '',
         clusterContext: 'kind',
         gitClean: false,
+        sshKeyPath: '',
       },
     ) {
       if (!volume.claimName) {
@@ -978,19 +1138,48 @@ EOF`);
       if (options.gitClean && volume.volumeMountPath) {
         Underpost.repo.clean({ paths: [volume.volumeMountPath] });
       }
-      if (options.nodeName) {
-        if (!fs.existsSync(rootVolumeHostPath)) fs.mkdirSync(rootVolumeHostPath, { recursive: true });
-        fs.copySync(volume.volumeMountPath, rootVolumeHostPath);
-      } else if (clusterContext === 'kind') {
-        shellExec(`docker exec -i kind-worker bash -c "mkdir -p ${rootVolumeHostPath}"`);
-        // shellExec(`docker cp ${volume.volumeMountPath} kind-worker:${rootVolumeHostPath}`);
-        shellExec(`tar -C ${volume.volumeMountPath} -c . | docker cp - kind-worker:${rootVolumeHostPath}`);
+      // The node that physically receives the volume data. hostPath volumes are
+      // node-local, so the data must land on the node where the pod will run, and
+      // the PV is pinned there (nodeAffinity) so the scheduler co-locates the pod
+      // with its volume — never mounting an empty DirectoryOrCreate on another node.
+      let dataNode;
+      if (clusterContext === 'kind') {
+        const kindNode = options.nodeName || 'kind-worker';
+        dataNode = kindNode;
+        shellExec(`docker exec -i ${kindNode} bash -c "mkdir -p ${rootVolumeHostPath}"`);
+        shellExec(`tar -C ${volume.volumeMountPath} -c . | docker cp - ${kindNode}:${rootVolumeHostPath}`);
         shellExec(
-          `docker exec -i kind-worker bash -c "chown -R 1000:1000 ${rootVolumeHostPath}; chmod -R 755 ${rootVolumeHostPath}"`,
+          `docker exec -i ${kindNode} bash -c "chown -R 1000:1000 ${rootVolumeHostPath}; chmod -R 755 ${rootVolumeHostPath}"`,
         );
       } else {
-        if (!fs.existsSync(rootVolumeHostPath)) fs.mkdirSync(rootVolumeHostPath, { recursive: true });
-        fs.copySync(volume.volumeMountPath, rootVolumeHostPath);
+        const localHost = os.hostname();
+        dataNode = options.nodeName || localHost;
+        if (dataNode === localHost) {
+          // Target node is the control plane / current host: write directly.
+          if (!fs.existsSync(rootVolumeHostPath)) fs.mkdirSync(rootVolumeHostPath, { recursive: true });
+          fs.copySync(volume.volumeMountPath, rootVolumeHostPath);
+        } else {
+          // Target node is remote: fs.copySync would only write the control-plane
+          // filesystem, leaving the real node's hostPath empty. Ship the folder to
+          // the node over SSH so the data exists where the pod is pinned.
+          const nodeHost =
+            shellExec(
+              `kubectl get node ${dataNode} -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}'`,
+              { stdout: true, silent: true, silentOnError: true },
+            ).trim() || dataNode;
+          logger.info('Shipping volume to remote node over SSH', {
+            node: dataNode,
+            host: nodeHost,
+            src: volume.volumeMountPath,
+            dest: rootVolumeHostPath,
+          });
+          Underpost.ssh.copyDirToNode({
+            host: nodeHost,
+            localDir: volume.volumeMountPath,
+            remoteDir: rootVolumeHostPath,
+            ...(options.sshKeyPath ? { keyPath: options.sshKeyPath } : {}),
+          });
+        }
       }
       shellExec(`kubectl delete pvc ${pvcId} -n ${namespace} --ignore-not-found`);
       shellExec(`kubectl delete pv ${pvId} --ignore-not-found`);
@@ -999,6 +1188,7 @@ ${Underpost.deploy.persistentVolumeFactory({
   hostPath: rootVolumeHostPath,
   pvcId,
   namespace,
+  nodeName: dataNode,
 })}
 EOF
 `);
@@ -1087,11 +1277,28 @@ ${secret ? `          readOnly: true\n` : ''}`;
      * @param {string} options.hostPath - Host path for the persistent volume.
      * @param {string} options.pvcId - Persistent volume claim ID.
      * @param {string} [options.namespace='default'] - Kubernetes namespace for the PVC claimRef.
+     * @param {string} [options.nodeName=''] - Node name to which the persistent volume is pinned (optional).
      * @returns {string} - YAML configuration for the persistent volume and claim.
      * @memberof UnderpostDeploy
      */
-    persistentVolumeFactory({ hostPath, pvcId, namespace = 'default' }) {
+    persistentVolumeFactory({ hostPath, pvcId, namespace = 'default', nodeName = '' }) {
       const pvId = pvcId.replace(/^pvc-/, 'pv-');
+      // hostPath volumes are node-local: deployVolume writes the content to the
+      // filesystem of a single node. Without nodeAffinity the scheduler can place
+      // the pod on a different node and mount an empty DirectoryOrCreate hostPath
+      // (missing the materialized assets). Pin the PV to the node that holds the
+      // data so the pod is always co-located with its volume.
+      const nodeAffinity = nodeName
+        ? `
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+        - matchExpressions:
+            - key: kubernetes.io/hostname
+              operator: In
+              values:
+                - ${nodeName}`
+        : '';
       return `apiVersion: v1
 kind: PersistentVolume
 metadata:
@@ -1102,7 +1309,7 @@ spec:
   accessModes:
     - ReadWriteOnce
   persistentVolumeReclaimPolicy: Retain
-  storageClassName: manual
+  storageClassName: manual${nodeAffinity}
   claimRef:
     apiVersion: v1
     kind: PersistentVolumeClaim
@@ -1136,183 +1343,10 @@ spec:
      * @memberof UnderpostDeploy
      */
     isValidTLSContext: ({ host, env, options }) =>
-      env === 'production' &&
-      options.cert === true &&
-      (!options.certHosts || options.certHosts.split(',').includes(host)),
-    /**
-     * Monitors the ready status of a deployment.
-     *
-     * Ready signal:
-     *   The orchestrator gate is the Kubernetes pod Ready condition. When the
-     *   container's `readinessProbe` succeeds, kubelet flips
-     *   `status.conditions[Ready]` to True and `checkDeploymentReadyStatus`
-     *   returns the pod in `readyPods`. This is the only required signal — see
-     *   `src/client/public/nexodev/docs/references/Deploy custom instance to K8S.md`.
-     *
-     * Container-status:
-     *   `underpost config get container-status` is read from each pod for both
-     *   the display column and as a second ready gate alongside the K8S Ready
-     *   condition. Both must be satisfied before the monitor exits:
-     *     1. K8S readinessProbe (TCP socket) — ensures the port is bound.
-     *     2. container-status == `<deploy>-<env>-running-deployment` — ensures
-     *        the application has completed its own startup sequence.
-     *   Early-abort on `error` container-status remains in effect.
-     *
-     * @param {string} deployId - Deployment ID for which the ready status is being monitored.
-     * @param {string} env - Environment for which the ready status is being monitored.
-     * @param {string} targetTraffic - Target traffic status for the deployment.
-     * @param {Array<string>} ignorePods - List of pod names to ignore.
-     * @param {string} [namespace='default'] - Kubernetes namespace for the deployment.
-     * @returns {object} - Object containing the ready status of the deployment.
-     * @memberof UnderpostDeploy
-     */
-    async monitorReadyRunner(deployId, env, targetTraffic, ignorePods = [], namespace = 'default') {
-      const delayMs = 1000;
-      const maxIterations = 3000;
-      const deploymentId = `${deployId}-${env}-${targetTraffic}`;
-      const expectedContainerStatus = `${deployId}-${env}-running-deployment`;
-      const tag = `[${deploymentId}]`;
-      const containerStatusDefault = 'waiting for status';
-
-      logger.info('Deployment init', { deployId, env, targetTraffic, namespace });
-
-      // Per-pod cache of last-known container-status (persists across retries)
-      const podStatusCache = new Map();
-
-      const readContainerStatus = (podName) => {
-        try {
-          const raw = shellExec(
-            `sudo kubectl exec ${podName} -n ${namespace} -- sh -c 'underpost config get container-status --plain'`,
-            { silent: true, disableLog: true, stdout: true, silentOnError: true },
-          );
-          const val = raw ? raw.toString().trim() : '';
-          return val && val !== 'undefined' ? val : containerStatusDefault;
-        } catch (_) {
-          // exec failed (e.g. pod not yet running) — preserve last known value
-          return podStatusCache.get(podName) || containerStatusDefault;
-        }
-      };
-
-      for (let i = 0; i < maxIterations; i++) {
-        const result = await Underpost.deploy.checkDeploymentReadyStatus(
-          deployId,
-          env,
-          targetTraffic,
-          ignorePods,
-          namespace,
-        );
-
-        const allPods = [...result.readyPods, ...result.notReadyPods];
-
-        // Update cache with latest status for each pod (informational + error gate)
-        for (const pod of allPods) {
-          if (!pod?.NAME) continue;
-          const status = readContainerStatus(pod.NAME);
-          if (status === 'error') throw new Error(`Pod ${pod.NAME} has error status`);
-          podStatusCache.set(pod.NAME, status);
-        }
-
-        const allPodsK8sReady = allPods.length > 0 && result.notReadyPods.length === 0;
-
-        const allPodsStatusReady =
-          allPods.length > 0 && allPods.every((pod) => podStatusCache.get(pod.NAME) === expectedContainerStatus);
-
-        // Print snapshot for every pod — annotate when container-status hasn't caught
-        // up to the K8S Ready condition yet.
-        for (const pod of allPods) {
-          const status = podStatusCache.get(pod.NAME) || containerStatusDefault;
-          const podStatus = pod.STATUS || 'Unknown';
-          const statusMatchesExpected = status === expectedContainerStatus;
-          const statusDisplay = statusMatchesExpected ? status : `${status} (pending)`;
-
-          console.log(
-            'Target pod:',
-            pod.NAME[pod.NAME.includes('green') ? 'bgGreen' : 'bgBlue'].bold.black,
-            '| Pod status:',
-            podStatus.bold.yellow,
-            '| Runtime status:',
-            statusDisplay.bold.cyan,
-          );
-        }
-
-        // Both K8S readinessProbe AND container-status must be satisfied before
-        // declaring the deployment ready. The TCP probe ensures the port is bound;
-        // container-status == running-deployment ensures the application has
-        // completed its own startup sequence so traffic is not switched prematurely.
-        if (allPodsK8sReady && allPodsStatusReady) {
-          logger.info(`${tag} | All pods Ready (K8S readinessProbe satisfied)`);
-          return result;
-        }
-
-        await timer(delayMs);
-
-        if ((i + 1) % 10 === 0) {
-          logger.info(`${tag} | In progress... iteration ${i + 1}`);
-        }
-      }
-
-      logger.error(`${tag} | Deployment timeout after ${maxIterations} iterations`);
-      throw new Error(
-        `monitorReadyRunner timeout: ${deploymentId} did not become Ready within ${maxIterations}*${delayMs}ms`,
-      );
-    },
-
-    /**
-     * Retrieves the currently loaded images in the Kubernetes cluster.
-     * @param {string} [node='kind-worker'] - Node name to check for loaded images.
-     * @param {object} options - Options for the image retrieval.
-     * @param {boolean} options.spec - Whether to retrieve images from the pod specifications.
-     * @param {string} options.namespace - Kubernetes namespace to filter pods.
-     * @returns {Array<object>} - Array of objects containing pod names and their corresponding images.
-     * @memberof UnderpostDeploy
-     */
-    getCurrentLoadedImages(node = 'kind-worker', options = { spec: false, namespace: '' }) {
-      if (options.spec) {
-        const raw = shellExec(
-          `kubectl get pods ${options.namespace ? `--namespace ${options.namespace}` : `--all-namespaces`} -o=jsonpath='{range .items[*]}{"\\n"}{.metadata.namespace}{"/"}{.metadata.name}{":\\t"}{range .spec.containers[*]}{.image}{", "}{end}{end}'`,
-          {
-            stdout: true,
-            silent: true,
-          },
-        );
-        return raw
-          .split(`\n`)
-          .map((lines) => ({
-            pod: lines.split('\t')[0].replaceAll(':', '').trim(),
-            image: lines.split('\t')[1] ? lines.split('\t')[1].replaceAll(',', '').trim() : null,
-          }))
-          .filter((o) => o.image);
-      }
-      const raw = shellExec(node === 'kind-worker' ? `docker exec -i ${node} crictl images` : `crictl images`, {
-        stdout: true,
-        silent: true,
-      });
-
-      const heads = raw
-        .split(`\n`)[0]
-        .split(' ')
-        .filter((_r) => _r.trim());
-
-      const pods = raw
-        .split(`\n`)
-        .filter((r) => !r.match('IMAGE'))
-        .map((r) => r.split(' ').filter((_r) => _r.trim()));
-
-      const result = [];
-
-      for (const row of pods) {
-        if (row.length === 0) continue;
-        const pod = {};
-        let index = -1;
-        for (const head of heads) {
-          if (head in pod) continue;
-          index++;
-          pod[head] = row[index];
-        }
-        result.push(pod);
-      }
-      return result;
-    },
+      (env === 'production' &&
+        options.cert === true &&
+        (!options.certHosts || options.certHosts.split(',').includes(host))) ||
+      options.selfSigned === true,
 
     /**
      * Predefined resource templates for Kubernetes deployments.