cyberia 3.2.9 → 3.2.22

package/src/cli/run.js

@@ -10,9 +10,12 @@ import {
   awaitDeployMonitor,
   buildKindPorts,
   Config,
+  cronDeployIdResolve,
+  etcHostFactory,
   getNpmRootPath,
   isDeployRunnerContext,
   loadConfServerJson,
+  loadReplicas,
   writeEnv,
 } from '../server/conf.js';
 import { actionInitLog, loggerFactory } from '../server/logger.js';
@@ -24,6 +27,7 @@ import { range, setPad, timer } from '../client/components/core/CommonJs.js';
 import os from 'os';
 import Underpost from '../index.js';
 import dotenv from 'dotenv';
+import { MongoBootstrap } from '../db/mongo/MongoBootstrap.js';
 
 const waitForPort = (port, host = '127.0.0.1', { maxAttempts = 30, interval = 2000 } = {}) =>
   new Promise((resolve, reject) => {
@@ -97,6 +101,7 @@ const logger = loggerFactory(import.meta);
  * @property {string} deployId - The deployment ID.
  * @property {string} instanceId - The instance ID.
  * @property {string} user - The user to run as.
+ * @property {string} group - The group to use.
  * @property {string} pid - The process ID.
  * @property {boolean} disablePrivateConfUpdate - Whether to disable private configuration updates.
  * @property {string} monitorStatus - The monitor status option.
@@ -114,6 +119,8 @@ const logger = loggerFactory(import.meta);
  * @property {boolean} copy - Whether to copy the command to the clipboard instead of executing it.
  * @property {boolean} skipFullBuild - Whether to skip the full client bundle build during deployment (supported by: sync, template-deploy).
  * @property {boolean} pullBundle - Whether to pull the bundle before running. Use together with --skip-full-build to skip the local build entirely (supported by: sync, template-deploy).
+ * @property {boolean} remove - Whether to remove/teardown resources instead of creating them (e.g. delete-expose for k3s proxy devices in dev-cluster).
+ * @property {boolean} test - Whether to enable test/generic-purpose mode (e.g. use self-signed TLS instead of cert-manager).
  * @memberof UnderpostRun
  */
 const DEFAULT_OPTION = {
@@ -165,6 +172,7 @@ const DEFAULT_OPTION = {
   deployId: '',
   instanceId: '',
   user: '',
+  group: '',
   pid: '',
   disablePrivateConfUpdate: false,
   monitorStatus: '',
@@ -180,6 +188,8 @@ const DEFAULT_OPTION = {
   copy: false,
   skipFullBuild: false,
   pullBundle: false,
+  remove: false,
+  test: false,
 };
 
 /**
@@ -209,28 +219,39 @@ class UnderpostRun {
     'dev-cluster': (path, options = DEFAULT_OPTION) => {
       const baseCommand = options.dev ? 'node bin' : 'underpost';
       const mongoHosts = ['mongodb-0.mongodb-service'];
+      let primaryMongoHost = 'mongodb-0.mongodb-service';
       if (!options.expose) {
         shellExec(`${baseCommand} cluster${options.dev ? ' --dev' : ''} --reset`);
         shellExec(`${baseCommand} cluster${options.dev ? ' --dev' : ''}`);
 
         shellExec(
-          `${baseCommand} cluster${options.dev ? ' --dev' : ''} --mongodb --mongo-db-host ${mongoHosts.join(
+          `${baseCommand} cluster${options.dev ? ' --dev' : ''} --mongodb --service-host ${mongoHosts.join(
             ',',
           )} --pull-image`,
         );
         shellExec(`${baseCommand} cluster${options.dev ? ' --dev' : ''} --valkey --pull-image`);
       }
-
-      {
-        // Detect MongoDB primary pod using method
-        let primaryMongoHost = 'mongodb-0.mongodb-service';
+      if (options.k3s) {
+        if (options.remove) {
+          shellExec(`${baseCommand} lxd --delete-expose k3s-control:27017`);
+          shellExec(`${baseCommand} lxd --delete-expose k3s-control:6379`);
+        } else {
+          shellExec(`${baseCommand} lxd --expose k3s-control:27017 --node-port 32017`);
+          shellExec(`${baseCommand} lxd --expose k3s-control:6379 --node-port 32079`);
+        }
+        shellExec(`lxc config device show k3s-control`);
+      } else {
         try {
-          const primaryPodName = Underpost.db.getMongoPrimaryPodName({
-            namespace: options.namespace,
-            podName: 'mongodb-0',
-          });
-          // shellExec(`${baseCommand} deploy --expose --disable-update-underpost-config mongo`, { async: true });
-          shellExec(`kubectl port-forward -n ${options.namespace} pod/${primaryPodName} 27017:27017`, { async: true });
+          const primaryPodName =
+            MongoBootstrap.getPrimaryPodName({
+              namespace: options.namespace,
+              podName: 'mongodb-0',
+              disableAuth: options.dev,
+            }) || 'mongodb-0';
+          shellExec(
+            `${baseCommand} deploy --expose --namespace ${options.namespace} --disable-update-underpost-config mongo`,
+            { async: true },
+          );
           shellExec(
             `${baseCommand} deploy --expose --namespace ${options.namespace} --disable-update-underpost-config valkey`,
             { async: true },
@@ -241,10 +262,19 @@ class UnderpostRun {
             default: primaryMongoHost,
           });
         }
-
-        const hostListenResult = Underpost.deploy.etcHostFactory([primaryMongoHost]);
-        logger.info(hostListenResult.renderHosts);
       }
+      const hostListenResult = etcHostFactory([primaryMongoHost]);
+      logger.info(hostListenResult.renderHosts);
+    },
+
+    /**
+     * @method etc-hosts
+     * @description Modifies the `/etc/hosts` file to add entries for local access to services,
+     * based on the provided path input.
+     * @param {string} path - The input value, identifier, or path for the operation (used to specify the entries to add to /etc/hosts).
+     */
+    'etc-hosts': (path = '', options = DEFAULT_OPTION) => {
+      etcHostFactory(path.split(','));
     },
 
     /**
@@ -405,6 +435,7 @@ class UnderpostRun {
         return;
       }
       shellExec(`${baseCommand} run pull`);
+      shellExec(`${baseCommand} run shared-dir`);
 
       // Capture last N commit messages for propagation.
       // When --from-n-commit is not set, auto-detect unpushed commit count (same as --unpush flag).
@@ -485,6 +516,7 @@ class UnderpostRun {
         return;
       }
       shellExec(`${baseCommand} run pull`);
+      shellExec(`${baseCommand} run shared-dir`);
 
       // Capture last N commit messages from the engine repo.
       // When --from-n-commit is not set, auto-detect unpushed commit count (same as --unpush flag).
@@ -505,14 +537,16 @@ class UnderpostRun {
     },
     /**
      * @method docker-image
-     * @description Dispatches the Docker image CI workflow (`docker-image.ci.yml`) for the `engine` repository via `workflow_dispatch`.
-     * @param {string} path - The input value, identifier, or path for the operation.
+     * @description Dispatches the Docker image CI workflow (`docker-image[.<runtime>].ci.yml`) via `workflow_dispatch`.
+     * Repository resolution is delegated to `Underpost.repo.resolveInstanceRepo(path)`.
+     * @param {string} path - Optional runtime / workflow suffix (e.g. `cyberia-server`, `cyberia-client`).
      * @param {Object} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'docker-image': (path, options = DEFAULT_OPTION) => {
+      const repo = Underpost.repo.resolveInstanceRepo(path);
       Underpost.repo.dispatchWorkflow({
-        repo: `${process.env.GITHUB_USERNAME}/engine`,
+        repo,
         workflowFile: `docker-image${path ? `.${path}` : ''}.ci.yml`,
         ref: 'master',
         inputs: {},
@@ -527,6 +561,7 @@ class UnderpostRun {
      */
     clean: (path = '', options = DEFAULT_OPTION) => {
       Underpost.repo.clean({ paths: path ? path.split(',') : ['/home/dd/engine', '/home/dd/engine/engine-private'] });
+      if (options.dev) shellExec(`node bin run shared-dir ${path ? path : '/home/dd/engine'}`);
     },
     /**
      * @method pull
@@ -536,16 +571,14 @@ class UnderpostRun {
      * @memberof UnderpostRun
      */
     pull: (path, options = DEFAULT_OPTION) => {
+      // shellExec is fail-fast by default — any non-zero exit throws and
+      // propagates up to the workflow step. No per-call flag required.
       if (!fs.existsSync(`/home/dd`) || !fs.existsSync(`/home/dd/engine`)) {
         fs.mkdirSync(`/home/dd`, { recursive: true });
-        shellExec(`cd /home/dd && underpost clone ${process.env.GITHUB_USERNAME}/engine`, {
-          silent: true,
-        });
+        shellExec(`cd /home/dd && underpost clone ${process.env.GITHUB_USERNAME}/engine`, { silent: true });
       } else {
         shellExec(`underpost run clean`);
-        shellExec(`cd /home/dd/engine && underpost pull . ${process.env.GITHUB_USERNAME}/engine`, {
-          silent: true,
-        });
+        shellExec(`cd /home/dd/engine && underpost pull . ${process.env.GITHUB_USERNAME}/engine`, { silent: true });
       }
       if (!fs.existsSync(`/home/dd/engine/engine-private`))
         shellExec(`cd /home/dd/engine && underpost clone ${process.env.GITHUB_USERNAME}/engine-private`, {
@@ -554,9 +587,7 @@ class UnderpostRun {
       else
         shellExec(
           `cd /home/dd/engine/engine-private && underpost pull . ${process.env.GITHUB_USERNAME}/engine-private`,
-          {
-            silent: true,
-          },
+          { silent: true },
         );
     },
     /**
@@ -643,6 +674,11 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
     /**
      * @method sync
      * @description Cleans up, and then runs a deployment synchronization command (`underpost deploy --kubeadm --build-manifest --sync...`) using parameters parsed from `path` (deployId, replicas, versions, image, node).
+     *
+     * Forwards `--image-pull-policy <policy>` to the underlying `deploy --build-manifest` invocation when `options.imagePullPolicy` is set,
+     * which then plumbs through `buildManifest` and `deploymentYamlPartsFactory` to override the container `imagePullPolicy` in the generated
+     * `deployment.yaml`. Useful when you want to force `Always` so the kubelet re-pulls a mutable tag on every rollout. Example:
+     *   `node bin run sync dd-core --kubeadm --image-pull-policy Always`
      * @param {string} path - The input value, identifier, or path for the operation (used as a comma-separated string containing deploy parameters).
      * @param {Object} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
@@ -692,6 +728,11 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
       let targetTraffic = currentTraffic ? (currentTraffic === 'blue' ? 'green' : 'blue') : 'green';
       if (targetTraffic) versions = versions ? versions : targetTraffic;
 
+      const ignorePods =
+        isDeployRunnerContext(path, options) && targetTraffic
+          ? Underpost.kubectl.get(`${deployId}-${env}-${targetTraffic}`, 'pods', options.namespace).map((p) => p.NAME)
+          : [];
+
       const timeoutFlags = Underpost.deploy.timeoutFlagsFactory(options);
       const cmdString = options.cmd
         ? ' --cmd ' + (options.cmd.find((c) => c.match('"')) ? '"' + options.cmd + '"' : "'" + options.cmd + "'")
@@ -700,13 +741,14 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
 
       const skipFullBuildFlag = options.skipFullBuild ? ' --skip-full-build' : '';
       const pullBundleFlag = options.pullBundle ? ' --pull-bundle' : '';
+      const imagePullPolicyFlag = options.imagePullPolicy ? ` --image-pull-policy ${options.imagePullPolicy}` : '';
 
       shellExec(
         `${baseCommand} deploy${clusterFlag} --build-manifest --sync --info-router --replicas ${replicas} --node ${node}${
           image ? ` --image ${image}` : ''
         }${versions ? ` --versions ${versions}` : ''}${
           options.namespace ? ` --namespace ${options.namespace}` : ''
-        }${timeoutFlags}${cmdString}${gitCleanFlag}${skipFullBuildFlag}${pullBundleFlag} ${deployId} ${env}`,
+        }${timeoutFlags}${cmdString}${gitCleanFlag}${skipFullBuildFlag}${pullBundleFlag}${imagePullPolicyFlag} ${deployId} ${env}`,
       );
 
       if (isDeployRunnerContext(path, options)) {
@@ -717,11 +759,11 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
         shellExec(
           `${baseCommand} deploy${clusterFlag}${cmdString} --replicas ${replicas} --disable-update-proxy ${deployId} ${env} --versions ${versions}${
             options.namespace ? ` --namespace ${options.namespace}` : ''
-          }${timeoutFlags}${gitCleanFlag}`,
+          }${timeoutFlags}${gitCleanFlag}${imagePullPolicyFlag}`,
         );
         if (!targetTraffic)
           targetTraffic = Underpost.deploy.getCurrentTraffic(deployId, { namespace: options.namespace });
-        await Underpost.deploy.monitorReadyRunner(deployId, env, targetTraffic, [], options.namespace, 'underpost');
+        await Underpost.monitor.monitorReadyRunner(deployId, env, targetTraffic, ignorePods, options.namespace);
         Underpost.deploy.switchTraffic(deployId, env, targetTraffic, replicas, options.namespace, options);
       } else
         logger.info('current traffic', Underpost.deploy.getCurrentTraffic(deployId, { namespace: options.namespace }));
@@ -967,8 +1009,19 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
             // pathRewritePolicy,
           });
         if (options.tls) {
-          shellExec(`sudo kubectl delete Certificate ${_host} -n ${options.namespace} --ignore-not-found`);
-          proxyYaml += Underpost.deploy.buildCertManagerCertificate({ ...options, host: _host });
+          if (options.test) {
+            const sslDir = `./engine-private/ssl/${_host}`;
+            const nameSafe = _host.replace(/[^a-zA-Z0-9_.-]/g, '_');
+            fs.mkdirpSync(sslDir);
+            shellExec(`bash ./scripts/ssl.sh "${sslDir}" "${_host}"`);
+            shellExec(`kubectl delete secret ${_host} -n ${options.namespace} --ignore-not-found`);
+            shellExec(
+              `kubectl create secret tls ${_host} --cert="${sslDir}/${nameSafe}.pem" --key="${sslDir}/${nameSafe}-key.pem" -n ${options.namespace}`,
+            );
+          } else {
+            shellExec(`sudo kubectl delete Certificate ${_host} -n ${options.namespace} --ignore-not-found`);
+            proxyYaml += Underpost.deploy.buildCertManagerCertificate({ ...options, host: _host });
+          }
         }
         // console.log(proxyYaml);
         shellExec(`kubectl delete HTTPProxy ${_host} --namespace ${options.namespace} --ignore-not-found`);
@@ -1023,6 +1076,9 @@ EOF
           cmd: _cmd,
           volumes: _volumes,
           metadata: _metadata,
+          lifecycle: _lifecycle,
+          readinessProbe: _readinessProbe,
+          livenessProbe: _livenessProbe,
         } = instance;
         if (id !== _id) continue;
         const _deployId = `${deployId}-${_id}`;
@@ -1034,6 +1090,7 @@ EOF
         // Examples images:
         // `underpost/underpost-engine:${Underpost.version}`
         // `localhost/rockylinux9-underpost:${Underpost.version}`
+        if (options.imageName) _image = options.imageName;
         if (!_image) _image = `underpost/underpost-engine:${Underpost.version}`;
 
         if (_image && !_image.startsWith('localhost'))
@@ -1087,6 +1144,20 @@ EOF
           ),
         );
 
+        // Resolve env-scoped lifecycle/probe blocks: each can be either
+        //   { ...envObj }                        // shared shape
+        //   { development: {...}, production: {...} }   // env-specific
+        const pickEnv = (v) => (v && (v.development || v.production) ? v[env] : v);
+
+        // Convention: an instance config may place `imagePullPolicy` inside
+        // the env-scoped lifecycle block (alongside postStart/preStop).
+        // Extract it onto the container spec (where K8S expects it) and
+        // strip it from the lifecycle hash so the rendered YAML stays valid.
+        // CLI override (`--image-pull-policy`) wins over the conf value.
+        const { lifecycle: lifecycleForManifest, imagePullPolicy: lifecycleImagePullPolicy } =
+          Underpost.deploy.extractInstanceImagePullPolicy(pickEnv(_lifecycle));
+        const instanceImagePullPolicy = options.imagePullPolicy || lifecycleImagePullPolicy;
+
         let deploymentYaml = `---
 ${Underpost.deploy
   .deploymentYamlPartsFactory({
@@ -1099,6 +1170,11 @@ ${Underpost.deploy
     namespace: options.namespace,
     volumes: _volumes,
     cmd: resolvedCmd,
+    lifecycle: lifecycleForManifest,
+    readinessProbe: pickEnv(_readinessProbe),
+    livenessProbe: pickEnv(_livenessProbe),
+    containerPort: _toPort,
+    imagePullPolicy: instanceImagePullPolicy,
   })
   .replace('{{ports}}', buildKindPorts(_fromPort, _toPort))}
 `;
@@ -1110,12 +1186,16 @@ EOF
 `,
           { disableLog: true },
         );
-        const { ready, readyPods } = await Underpost.deploy.monitorReadyRunner(
+        // Custom instances run a bare binary (no `underpost start` / internal
+        // HTTP endpoint): Kubernetes readiness is the running signal and
+        // container-status is read via exec. See `Deploy custom instance to K8S.md`.
+        const { ready, readyPods } = await Underpost.monitor.monitorReadyRunner(
           _deployId,
           env,
           targetTraffic,
           ignorePods,
           options.namespace,
+          { readyGate: 'kubernetes', statusTransport: 'exec' },
         );
 
         if (!ready) {
@@ -1125,12 +1205,12 @@ EOF
         shellExec(
           `${baseCommand} run${baseClusterCommand} --namespace ${options.namespace}` +
             `${options.nodeName ? ` --node-name ${options.nodeName}` : ''}` +
-            `${options.tls ? ` --tls` : ''}` +
+            `${options.tls ? ` --tls ${options.test ? '--test' : ''}` : ''}` +
             ` instance-promote '${path}'`,
         );
       }
       if (options.etcHosts) {
-        const hostListenResult = Underpost.deploy.etcHostFactory(etcHosts);
+        const hostListenResult = etcHostFactory(etcHosts);
         logger.info(hostListenResult.renderHosts);
       }
     },
@@ -1187,14 +1267,34 @@ EOF
         volumes: _volumes,
         metadata: _metadata,
         runtime: _runtime,
+        lifecycle: _lifecycle,
+        readinessProbe: _readinessProbe,
+        livenessProbe: _livenessProbe,
       } = instance;
 
-      // Resolve Dockerfile source: use runtime-specific path when instance defines a runtime.
-      const dockerfileSourcePath = _runtime ? `src/runtime/${_runtime}/Dockerfile` : `${rootPath}/Dockerfile`;
-      if (fs.existsSync(dockerfileSourcePath)) {
+      // Resolve Dockerfile source. Dev/prod variant rules:
+      //   - When the instance defines a `runtime`, look under
+      //     `src/runtime/<runtime>/`. In `--dev` mode prefer `Dockerfile.dev`
+      //     when it exists, falling back to `Dockerfile`.
+      //   - When `runtime` is not set, look in the project root with the
+      //     same `.dev` → no-suffix precedence.
+      // Dockerfile.dev is a full Dockerfile (not an overlay) — each runtime
+      // owns the contract between its dev image and its prod image (debug
+      // build flags, extra tooling, default ports, etc.).
+      const dockerfileBase = _runtime ? `src/runtime/${_runtime}` : rootPath;
+      const dockerfileCandidates = options.dev
+        ? [`${dockerfileBase}/Dockerfile.dev`, `${dockerfileBase}/Dockerfile`]
+        : [`${dockerfileBase}/Dockerfile`];
+      const dockerfileSourcePath = dockerfileCandidates.find((p) => fs.existsSync(p));
+      if (dockerfileSourcePath) {
+        if (options.dev && !dockerfileSourcePath.endsWith('.dev')) {
+          logger.warn(
+            `[instance-build-manifest] --dev requested but no Dockerfile.dev present; falling back to ${dockerfileSourcePath}`,
+          );
+        }
         fs.copyFileSync(dockerfileSourcePath, dockerfileManifestPath);
       } else {
-        logger.warn(`[instance-build-manifest] Dockerfile not found at ${dockerfileSourcePath}`);
+        logger.warn(`[instance-build-manifest] Dockerfile not found; tried: ${dockerfileCandidates.join(', ')}`);
       }
 
       const _deployId = `${deployId}-${_id}`;
@@ -1239,6 +1339,17 @@ EOF
         ),
       );
 
+      // Env-aware lifecycle / probe selection. Each block may either be
+      // a single object (shared across envs) or `{ development, production }`.
+      const pickEnv = (v) => (v && (v.development || v.production) ? v[env] : v);
+
+      // Convention: an instance config may place `imagePullPolicy` inside
+      // the env-scoped lifecycle block (alongside postStart/preStop).
+      // Extract it onto the container spec and strip it from the lifecycle hash.
+      const { lifecycle: lifecycleForManifest, imagePullPolicy: lifecycleImagePullPolicy } =
+        Underpost.deploy.extractInstanceImagePullPolicy(pickEnv(_lifecycle));
+      const instanceImagePullPolicy = options.imagePullPolicy || lifecycleImagePullPolicy;
+
       const deploymentYaml =
         `---\n` +
         Underpost.deploy
@@ -1252,6 +1363,11 @@ EOF
             namespace: options.namespace,
             volumes: _volumes,
             cmd: resolvedCmd,
+            lifecycle: lifecycleForManifest,
+            readinessProbe: pickEnv(_readinessProbe),
+            livenessProbe: pickEnv(_livenessProbe),
+            containerPort: _toPort,
+            imagePullPolicy: instanceImagePullPolicy,
           })
           .replace('{{ports}}', buildKindPorts(_fromPort, _toPort));
 
@@ -1330,9 +1446,9 @@ EOF`);
       // crictl is in the kubernetes repo but excluded by default — install it explicitly
       shellExec(`sudo yum install -y cri-tools --disableexcludes=kubernetes`);
       // Ensure CRI-O uses systemd cgroup driver (matches kubelet default)
-      shellExec(
-        `sudo sed -i 's/^#\?cgroup_manager =.*/cgroup_manager = "systemd"/' /etc/crio/crio.conf 2>/dev/null || true`,
-      );
+      shellExec(`sudo sed -i 's/^#\?cgroup_manager =.*/cgroup_manager = "systemd"/' /etc/crio/crio.conf`, {
+        silentOnError: true,
+      });
       shellExec(`sudo systemctl enable --now crio`);
       logger.info('CRI-O installed and started.');
       // Write crictl config so all crictl calls default to the CRI-O socket
@@ -1356,8 +1472,8 @@ EOF`);
       const baseClusterCommand = options.dev ? ' --dev' : '';
       const currentImage = options.imageName
         ? options.imageName
-        : Underpost.deploy
-            .getCurrentLoadedImages(options.nodeName ? options.nodeName : 'kind-worker', false)
+        : Underpost.image
+            .getCurrentLoaded(options.nodeName ? options.nodeName : 'kind-worker', false)
             .find((o) => o.IMAGE.match('underpost'));
       const podName = options.podName || `underpost-dev-container`;
       const volumeHostPath = options.claimName || '/home/dd';
@@ -1466,7 +1582,8 @@ EOF`);
           `git config user.name '${username}' && ` +
           `git config user.email '${email}' && ` +
           `git config credential.interactive always &&` +
-          `git config pull.rebase false`,
+          `git config pull.rebase false && ` +
+          `git config core.filemode false`,
         {
           disableLog: true,
           silent: true,
@@ -1633,20 +1750,13 @@ EOF`);
       const currentTraffic = Underpost.deploy.getCurrentTraffic(deployId, { namespace: options.namespace });
       const targetTraffic = currentTraffic === 'blue' ? 'green' : 'blue';
       const env = options.dev ? 'development' : 'production';
-      const ignorePods = Underpost.deploy
+      const ignorePods = Underpost.kubectl
         .get(`${deployId}-${env}-${targetTraffic}`, 'pods', options.namespace)
         .map((p) => p.NAME);
 
       shellExec(`sudo kubectl rollout restart deployment/${deployId}-${env}-${targetTraffic} -n ${options.namespace}`);
 
-      await Underpost.deploy.monitorReadyRunner(
-        deployId,
-        env,
-        targetTraffic,
-        ignorePods,
-        options.namespace,
-        'underpost',
-      );
+      await Underpost.monitor.monitorReadyRunner(deployId, env, targetTraffic, ignorePods, options.namespace);
 
       Underpost.deploy.switchTraffic(deployId, env, targetTraffic, options.replicas, options.namespace, options);
     },
@@ -1738,7 +1848,7 @@ EOF`);
         }`;
         shellExec(cmd, { async: true });
       }
-      await awaitDeployMonitor(true);
+      if ((await awaitDeployMonitor()) !== true) return;
       {
         const cmd = `npm run dev:client ${deployId} ${subConf} ${host} ${_path} proxy${options.tls ? ' tls' : ''}`;
 
@@ -1746,7 +1856,7 @@ EOF`);
           async: true,
         });
       }
-      await awaitDeployMonitor(true);
+      if ((await awaitDeployMonitor()) !== true) return;
       shellExec(
         `NODE_ENV=development node src/proxy proxy ${deployId} ${subConf} ${host} ${_path}${options.tls ? ' tls' : ''}`,
       );
@@ -1843,7 +1953,7 @@ EOF`);
         );
       } else logger.error(`Service pod ${podToMonitor} failed to start in time.`);
       if (options.etcHosts === true) {
-        const hostListenResult = Underpost.deploy.etcHostFactory([host]);
+        const hostListenResult = etcHostFactory([host]);
         logger.info(hostListenResult.renderHosts);
       }
     },
@@ -1861,7 +1971,7 @@ EOF`);
         const confServer = loadConfServerJson(`./engine-private/conf/${options.deployId}/conf.server.json`);
         hosts.push(...Object.keys(confServer));
       }
-      const hostListenResult = Underpost.deploy.etcHostFactory(hosts);
+      const hostListenResult = etcHostFactory(hosts);
       logger.info(hostListenResult.renderHosts);
     },
 
@@ -2180,15 +2290,26 @@ EOF`);
      * @memberof UnderpostRun
      */
     kill: (path = '', options = DEFAULT_OPTION) => {
-      if (options.pid) return shellExec(`sudo kill -9 ${options.pid}`);
+      if (options.pid)
+        return shellExec(`sudo kill -9 ${options.pid}`, {
+          silentOnError: true,
+        });
       for (const _path of path.split(',')) {
         if (_path.split('+')[1]) {
           let [port, sumPortOffSet] = _path.split('+');
           port = parseInt(port);
           sumPortOffSet = parseInt(sumPortOffSet);
           for (const sumPort of range(0, sumPortOffSet))
-            shellExec(`sudo kill -9 $(lsof -t -i:${parseInt(port) + parseInt(sumPort)})`);
-        } else shellExec(`sudo kill -9 $(lsof -t -i:${_path})`);
+            shellExec(
+              `PIDS=$(lsof -t -i:${parseInt(port) + parseInt(sumPort)}); [ -n "$PIDS" ] && sudo kill -9 $PIDS || true`,
+              {
+                silentOnError: true,
+              },
+            );
+        } else
+          shellExec(`PIDS=$(lsof -t -i:${_path}); [ -n "$PIDS" ] && sudo kill -9 $PIDS || true`, {
+            silentOnError: true,
+          });
       }
     },
     /**
@@ -2235,9 +2356,10 @@ EOF`);
      * @memberof UnderpostRun
      */
     secret: (path, options = DEFAULT_OPTION) => {
-      const secretPath = path ? path : `/home/dd/engine/engine-private/conf/dd-cron/.env.production`;
-      const command = `${options.dev ? 'node bin' : 'underpost'} secret underpost --create-from-file ${secretPath}`;
-      shellExec(command);
+      const cronDeployId = cronDeployIdResolve() || 'dd-cron';
+      Underpost.secret.underpost.createFromEnvFile(
+        `/home/dd/engine/engine-private/conf/${cronDeployId}/.env.${options.dev ? 'development' : 'production'}`,
+      );
     },
     /**
      * @method underpost-config
@@ -2411,7 +2533,8 @@ EOF`;
     /**
      * @method push-bundle
      * @description Builds the client zip for the specified deployment, splits it into parts, and uploads to file storage.
-     *   Steps: set env, build+split zip, switch to cron env, upload parts to storage.
+     *   Steps: set env, build+split zip, upload only the zip parts belonging to the deploy-id's hosts (from conf.server.json).
+     *   Only files matching `<host>-<route>.zip.part*` or `<host>-<route>.zip` for each non-skipped route are uploaded.
      * @param {string} path - Optional `fsPath.splitOption` string.
      *   Examples: `build` (default split 8), `build.16` (split 16 MB), `build.none-split` (no split flag).
      * @param {Object} options - The default underpost runner options for customizing workflow.
@@ -2420,7 +2543,7 @@ EOF`;
      * @memberof UnderpostRun
      */
     'push-bundle': (path = '', options = DEFAULT_OPTION) => {
-      const baseCommand = options.dev ? 'node bin' : 'underpost';
+      const baseCommand = 'node bin'; // options.dev ? 'node bin' : 'underpost';
       const env = options.dev ? 'development' : 'production';
       const deployId = options.deployId || 'dd-default';
       const pathParts = (path || '').split('.');
@@ -2444,11 +2567,54 @@ EOF`;
         }
       }
 
+      const confServerPath = `./engine-private/conf/${deployId}/conf.server.json`;
+      const confServer = fs.existsSync(confServerPath)
+        ? loadReplicas(deployId, loadConfServerJson(confServerPath))
+        : {};
+      const storageFilePath = `engine-private/conf/${deployId}/storage.bundle.json`;
+
       shellExec(`${baseCommand} env ${deployId} ${env}`);
       shellExec(`${baseCommand} client ${deployId} --build-zip${splitFlag ? ` ${splitFlag}` : ''}`);
-      shellExec(
-        `${baseCommand} fs ${fsPath} --recursive --deploy-id ${deployId} --storage-file-path engine-private/conf/${deployId}/storage.bundle.json --force`,
-      );
+
+      const pushBundleFiles = (host, routePath) => {
+        const buildId = `${host}-${routePath.replaceAll('/', '')}`;
+        const buildDir = `./${fsPath}`;
+        if (!fs.existsSync(buildDir)) return;
+        const partFiles = fs
+          .readdirSync(buildDir)
+          .filter(
+            (name) =>
+              name.startsWith(`${buildId}.zip.part`) ||
+              name.startsWith(`${buildId}.zip-part`) ||
+              name === `${buildId}.zip`,
+          )
+          .map((name) => `${fsPath}/${name}`);
+        if (partFiles.length === 0) {
+          logger.warn(`push-bundle: no bundle files found for '${host}${routePath}'`, { buildId });
+          return;
+        }
+        for (const partFile of partFiles) {
+          shellExec(
+            `${baseCommand} fs ${partFile} --deploy-id ${deployId} --storage-file-path ${storageFilePath} --force`,
+          );
+        }
+      };
+
+      for (const host of Object.keys(confServer)) {
+        for (const routePath of Object.keys(confServer[host])) {
+          const routeConf = confServer[host][routePath] || {};
+          if (routeConf.redirect || routeConf.disabledRebuild) continue;
+          if (routeConf.singleReplica) {
+            if (routeConf.replicas) {
+              for (const replica of routeConf.replicas) {
+                pushBundleFiles(host, replica);
+              }
+            }
+            continue;
+          }
+          pushBundleFiles(host, routePath);
+        }
+      }
     },
 
     /**
@@ -2465,11 +2631,13 @@ EOF`;
      * @memberof UnderpostRun
      */
     'pull-bundle': (path = '', options = DEFAULT_OPTION) => {
-      const baseCommand = options.dev ? 'node bin' : 'underpost';
+      const baseCommand = 'node bin'; // options.dev ? 'node bin' : 'underpost';
       const env = options.dev ? 'development' : 'production';
       const deployId = options.deployId || 'dd-default';
       const confServerPath = `./engine-private/conf/${deployId}/conf.server.json`;
-      const confServer = fs.existsSync(confServerPath) ? loadConfServerJson(confServerPath) : {};
+      const confServer = fs.existsSync(confServerPath)
+        ? loadReplicas(deployId, loadConfServerJson(confServerPath))
+        : {};
       const hostsArg = path
         ? path
             .split(',')
@@ -2488,62 +2656,124 @@ EOF`;
         `${baseCommand} fs build --recursive --deploy-id ${deployId} --storage-file-path engine-private/conf/${deployId}/storage.bundle.json --pull --omit-unzip`,
       );
 
+      const pullBundleRoute = (host, routePath) => {
+        const buildId = `${host}-${routePath.replaceAll('/', '')}`;
+        const zipPath = `build/${buildId}.zip`;
+        const buildDir = './build';
+        const hasZip = fs.existsSync(zipPath);
+        const hasParts =
+          fs.existsSync(buildDir) &&
+          fs
+            .readdirSync(buildDir)
+            .some((name) => name.startsWith(`${buildId}.zip.part`) || name.startsWith(`${buildId}.zip-part`));
+
+        if (!hasZip && !hasParts) {
+          logger.warn(`Bundle not found for '${host}${routePath}'. Skipping.`, { zipPath, deployId });
+          return;
+        }
+
+        if (hasParts) shellExec(`${baseCommand} client --merge-zip ${zipPath}`);
+        shellExec(`${baseCommand} client --unzip ${zipPath}`);
+        shellExec(`sudo rm -rf ${zipPath}`);
+
+        if (fs.existsSync(buildDir)) {
+          fs.readdirSync(buildDir)
+            .filter((name) => name.startsWith(`${buildId}.zip.part`) || name.startsWith(`${buildId}.zip-part`))
+            .forEach((partFile) => shellExec(`sudo rm -rf ${buildDir}/${partFile}`));
+        }
+
+        const extractedDir = `build/${buildId.replace(/-$/, '')}`;
+        if (!fs.existsSync(extractedDir)) {
+          logger.warn(`Extracted build dir not found: ${extractedDir}. Skipping move for '${host}${routePath}'.`);
+          return;
+        }
+
+        const publicDestPath = routePath === '/' ? `public/${host}` : `public/${host}${routePath}`;
+        if (fs.existsSync(publicDestPath)) shellExec(`sudo rm -rf ${publicDestPath}`);
+        if (routePath !== '/') shellExec(`sudo mkdir -p public/${host}`);
+        fs.copySync(`${extractedDir}`, `${publicDestPath}`);
+      };
+
       for (const host of hostsArg) {
-        // Gather all routes for this host; fall back to root '/' when host is not in confServer
-        // (e.g. when hosts were provided explicitly via the path argument).
         const routePaths = confServer[host] ? Object.keys(confServer[host]) : ['/'];
 
         for (const routePath of routePaths) {
           const routeConf = confServer[host] ? confServer[host][routePath] || {} : {};
-          // Skip routes that are not built by buildClient (mirrors buildClient skip conditions)
-          if (routeConf.singleReplica || routeConf.redirect || routeConf.disabledRebuild) continue;
-
-          // buildClient names the zip as "<host>-<path-no-slashes>.zip"
-          // e.g. host="underpost.net", path="/" → buildId="underpost.net-", zip="build/underpost.net-.zip"
-          // e.g. host="app.net", path="/admin" → buildId="app.net-admin", zip="build/app.net-admin.zip"
-          const buildId = `${host}-${routePath.replaceAll('/', '')}`;
-          const zipPath = `build/${buildId}.zip`;
-          const buildDir = './build';
-          const hasZip = fs.existsSync(zipPath);
-          const hasParts =
-            fs.existsSync(buildDir) &&
-            fs
-              .readdirSync(buildDir)
-              .some((name) => name.startsWith(`${buildId}.zip.part`) || name.startsWith(`${buildId}.zip-part`));
-
-          if (!hasZip && !hasParts) {
-            logger.warn(`Bundle not found for '${host}${routePath}'. Skipping.`, { zipPath, deployId });
+          if (routeConf.redirect || routeConf.disabledRebuild) continue;
+          if (routeConf.singleReplica) {
+            if (routeConf.replicas) {
+              for (const replica of routeConf.replicas) {
+                pullBundleRoute(host, replica);
+              }
+            }
             continue;
           }
+          pullBundleRoute(host, routePath);
+        }
+      }
+    },
 
-          if (hasParts) shellExec(`${baseCommand} client --merge-zip ${zipPath}`);
-          shellExec(`${baseCommand} client --unzip ${zipPath}`);
-          shellExec(`sudo rm -rf ${zipPath}`);
+    /**
+     * @method build-cluster-deployment-manifests
+     * @description Builds deployment manifests for both production and development environments using `node bin deploy --build-manifest`, syncing them, and setting replicas to 1 for the `dd` deployment.
+     * @param {string} path - Unused.
+     * @param {Object} options - The default underpost runner options for customizing workflow.
+     * @memberof UnderpostRun
+     */
+    'build-cluster-deployment-manifests': (path = '', options = DEFAULT_OPTION) => {
+      shellExec(`node bin deploy --build-manifest --sync --info-router --replicas 1 dd development`);
+      shellExec(`node bin deploy --build-manifest --sync --info-router --replicas 1 dd production --cert`);
+    },
 
-          // Clean up downloaded part wrapper zips left by --omit-unzip pull
-          if (fs.existsSync(buildDir)) {
-            fs.readdirSync(buildDir)
-              .filter((name) => name.startsWith(`${buildId}.zip.part`) || name.startsWith(`${buildId}.zip-part`))
-              .forEach((partFile) => shellExec(`sudo rm -rf ${buildDir}/${partFile}`));
-          }
+    /**
+     * @method monitor-ui
+     * @description Installs and enables the Cockpit KVM Dashboard (cockpit, cockpit-machines, libvirt)
+     * and opens the cockpit firewall service. With `--remove`, closes the firewall service instead.
+     * @param {string} path - Unused.
+     * @param {Object} options - The default underpost runner options for customizing workflow.
+     *   `options.remove` — when true, removes the cockpit firewall rule instead of adding it.
+     * @memberof UnderpostRun
+     */
+    'monitor-ui': (path, options = DEFAULT_OPTION) => {
+      if (options.remove) {
+        shellExec(`sudo firewall-cmd --zone=public --remove-service=cockpit --permanent`);
+        shellExec(`sudo firewall-cmd --reload`);
+        return;
+      }
+      shellExec(`sudo dnf install -y cockpit cockpit-machines libvirt`);
+      shellExec(`sudo systemctl enable --now cockpit.socket libvirtd`);
+      shellExec(`sudo firewall-cmd --permanent --add-service=cockpit`);
+      shellExec(`sudo firewall-cmd --reload`);
+    },
 
-          // unzipClientBuild extracts to buildId with trailing '-' stripped
-          // e.g. "build/underpost.net-" → "build/underpost.net"
-          // e.g. "build/app.net-admin" → "build/app.net-admin" (no trailing dash, no change)
-          const extractedDir = `build/${buildId.replace(/-$/, '')}`;
-          if (!fs.existsSync(extractedDir)) {
-            logger.warn(`Extracted build dir not found: ${extractedDir}. Skipping move for '${host}${routePath}'.`);
-            continue;
-          }
+    /**
+     * @method shared-dir
+     * @description Run once for initial shared-directory setup. Creates the group, adds the user,
+     * creates the directory, sets ownership, applies the SGID bit, and configures default ACLs so
+     * all future files inside the directory automatically inherit group write permissions.
+     * Use `reload-shared-dir` for subsequent permission repairs without recreating the group.
+     * @param {string} path - Target directory to set up (defaults to `/home/dd/engine`).
+     *   Customise via the `path` argument or leave empty to use the default.
+     * @param {Object} options - The default underpost runner options for customizing workflow.
+     *   Key fields: `options.user` (default `'admin'`), `options.group` (default `'engine-dev'`).
+     * @memberof UnderpostRun
+     */
+    'shared-dir': (path = '/home/dd/engine', options = DEFAULT_OPTION) => {
+      const dir = path || '/home/dd/engine';
+      const user = options.user || 'admin';
+      const group = options.group || 'engine-dev';
 
-          // Destination mirrors the public directory layout used by the server
-          const publicDestPath = routePath === '/' ? `public/${host}` : `public/${host}${routePath}`;
-          if (fs.existsSync(publicDestPath)) shellExec(`sudo rm -rf ${publicDestPath}`);
-          // Ensure parent directory exists for sub-paths
-          if (routePath !== '/') shellExec(`sudo mkdir -p public/${host}`);
-          shellExec(`sudo mv ${extractedDir} ${publicDestPath}`);
-        }
-      }
+      logger.info(`[setup-shared-dir] dir=${dir} user=${user} group=${group}`);
+
+      shellExec(`sudo groupadd ${group} 2>/dev/null || true`);
+      shellExec(`sudo usermod -aG ${group} ${user}`);
+      shellExec(`sudo mkdir -p ${dir}`);
+      shellExec(`sudo chown -R ${user}:${group} ${dir}`);
+      shellExec(`sudo chmod -R 2775 ${dir}`);
+      shellExec(`sudo setfacl -d -m g:${group}:rwx ${dir}`);
+      shellExec(`sudo setfacl -m g:${group}:rwx ${dir}`);
+
+      logger.info(`[setup-shared-dir] Shared directory setup complete: ${dir}`);
     },
   };
 
@@ -2601,14 +2831,14 @@ EOF`;
         if (options.replicas === '' || options.replicas === null || options.replicas === undefined)
           options.replicas = 1;
         options.npmRoot = npmRoot;
-        logger.info('callback', { path, options });
+        logger.info(`Executing runner`, { runner, namespace: options.namespace });
         if (!Underpost.run.RUNNERS.includes(runner)) throw new Error(`Runner not found: ${runner}`);
         const result = await Underpost.run.CALL(runner, path, options);
         return result;
       } catch (error) {
         console.log(error);
         logger.error(error);
-        return null;
+        process.exit(1);
       }
     },
   };