cyberia 3.2.12 → 3.2.70

package/src/cli/run.js

@@ -15,13 +15,14 @@ import {
   getNpmRootPath,
   isDeployRunnerContext,
   loadConfServerJson,
+  loadReplicas,
   writeEnv,
 } from '../server/conf.js';
 import { actionInitLog, loggerFactory } from '../server/logger.js';
 
 import fs from 'fs-extra';
 import net from 'net';
-import { range, setPad, timer } from '../client/components/core/CommonJs.js';
+import { range, s4, setPad, timer } from '../client/components/core/CommonJs.js';
 
 import os from 'os';
 import Underpost from '../index.js';
@@ -51,10 +52,12 @@ const logger = loggerFactory(import.meta);
 /**
  * @constant DEFAULT_OPTION
  * @description Default options for the UnderpostRun class.
+ * @typedef {Object} UnderpostRunDefaultOptions
  * @type {Object}
  * @property {boolean} dev - Whether to run in development mode.
  * @property {string} podName - The name of the pod to run.
  * @property {string} nodeName - The name of the node to run.
+ * @property {string} sshKeyPath - Private key path for node SSH operations, forwarded to volume shipping over SSH.
  * @property {number} port - Custom port to use.
  * @property {string} volumeHostPath - The host path for the volume.
  * @property {string} volumeMountPath - The mount path for the volume.
@@ -119,12 +122,14 @@ const logger = loggerFactory(import.meta);
  * @property {boolean} skipFullBuild - Whether to skip the full client bundle build during deployment (supported by: sync, template-deploy).
  * @property {boolean} pullBundle - Whether to pull the bundle before running. Use together with --skip-full-build to skip the local build entirely (supported by: sync, template-deploy).
  * @property {boolean} remove - Whether to remove/teardown resources instead of creating them (e.g. delete-expose for k3s proxy devices in dev-cluster).
+ * @property {boolean} test - Whether to enable test/generic-purpose mode (e.g. use self-signed TLS instead of cert-manager).
  * @memberof UnderpostRun
  */
 const DEFAULT_OPTION = {
   dev: false,
   podName: '',
   nodeName: '',
+  sshKeyPath: '',
   port: 0,
   volumeHostPath: '',
   volumeMountPath: '',
@@ -187,6 +192,7 @@ const DEFAULT_OPTION = {
   skipFullBuild: false,
   pullBundle: false,
   remove: false,
+  test: false,
 };
 
 /**
@@ -210,7 +216,7 @@ class UnderpostRun {
      * @method dev-cluster
      * @description Resets and deploys a full development cluster including MongoDB, Valkey, exposes services, and updates `/etc/hosts` for local access.
      * @param {string} path - The input value, identifier, or path for the operation.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'dev-cluster': (path, options = DEFAULT_OPTION) => {
@@ -222,7 +228,7 @@ class UnderpostRun {
         shellExec(`${baseCommand} cluster${options.dev ? ' --dev' : ''}`);
 
         shellExec(
-          `${baseCommand} cluster${options.dev ? ' --dev' : ''} --mongodb4 --service-host ${mongoHosts.join(
+          `${baseCommand} cluster${options.dev ? ' --dev' : ''} --mongodb --service-host ${mongoHosts.join(
             ',',
           )} --pull-image`,
         );
@@ -264,6 +270,16 @@ class UnderpostRun {
       logger.info(hostListenResult.renderHosts);
     },
 
+    /**
+     * @method etc-hosts
+     * @description Modifies the `/etc/hosts` file to add entries for local access to services,
+     * based on the provided path input.
+     * @param {string} path - The input value, identifier, or path for the operation (used to specify the entries to add to /etc/hosts).
+     */
+    'etc-hosts': (path = '', options = DEFAULT_OPTION) => {
+      etcHostFactory(path.split(','));
+    },
+
     /**
      * @method ipfs-expose
      * @description Exposes IPFS Cluster services on specified ports for local access.
@@ -282,7 +298,7 @@ class UnderpostRun {
      * @method metadata
      * @description Generates metadata for the specified path after exposing the development cluster.
      * @param {string} path - The input value, identifier, or path for the operation.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     metadata: async (path, options = DEFAULT_OPTION) => {
@@ -307,7 +323,7 @@ class UnderpostRun {
      * @method svc-ls
      * @description Lists systemd services and installed packages, optionally filtering by the provided path.
      * @param {string} path - The input value, identifier, or path for the operation (used as the optional filter for services and packages).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'svc-ls': (path, options = DEFAULT_OPTION) => {
@@ -327,7 +343,7 @@ class UnderpostRun {
      * @method svc-rm
      * @description Removes a systemd service by stopping it, disabling it, uninstalling the package, and deleting related files.
      * @param {string} path - The input value, identifier, or path for the operation (used as the service name).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'svc-rm': (path, options = DEFAULT_OPTION) => {
@@ -342,7 +358,7 @@ class UnderpostRun {
      * @method ssh-deploy-info
      * @description Retrieves deployment status and pod information from a remote server via SSH.
      * @param {string} path - The input value, identifier, or path for the operation.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'ssh-deploy-info': async (path = '', options = DEFAULT_OPTION) => {
@@ -360,11 +376,233 @@ class UnderpostRun {
       );
     },
 
+    /**
+     * @method node-move
+     * @description Abstract runner that relocates any schedulable Kubernetes workload
+     * (Deployment, StatefulSet, DaemonSet, ReplicaSet, Job, CronJob, ReplicationController)
+     * onto a target node by patching its pod-template `nodeSelector` and rolling it out.
+     * Resource-kind agnostic: it resolves the kind dynamically and applies the right
+     * patch path, so it works for `sts`, `deployment`, etc. without bespoke logic.
+     *
+     * Selection grammar via `path`:
+     *   - `<kind>/<name>`  -> a single resource (e.g. `deployment/dd-core-production-blue`)
+     *   - `<kind>`         -> every resource of that kind in the namespace (e.g. `statefulset`)
+     *   - ``               -> all movable workloads (deployment, statefulset, daemonset) in the namespace
+     *
+     * Placement:
+     *   - default: built-in `kubernetes.io/hostname=<node>` (no node mutation required)
+     *   - `--labels k=v,...`: label the target node with those pairs and use them as the
+     *     nodeSelector (matches the "label node + nodeSelector" pattern), enabling reusable
+     *     workload pools instead of pinning to a single hostname.
+     *
+     * Flags: `--node-name <node>` (target), `--namespace <ns>`, `--dry-run` (preview only),
+     *        `--remove` (clear the nodeSelector / unpin placement).
+     *
+     * Caveats: Services/ConfigMaps and bare Pods are not schedulable controllers and are
+     * skipped (move the owning controller). StatefulSets bound to node-local PVs may stay
+     * Pending after a move until their volume is available on the target node.
+     * @param {string} path - Resource selector (`kind/name`, `kind`, or empty).
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
+     * @memberof UnderpostRun
+     * @returns {Array<{ref:string,kind:string,status:string,node?:string}>} Per-resource outcome.
+     */
+    'node-move': (path = '', options = DEFAULT_OPTION) => {
+      const node = options.nodeName;
+      const ns = options.namespace || 'default';
+      const dryRun = options.dryRun === true;
+      const remove = options.remove === true;
+
+      if (!remove && !node) {
+        throw new Error('node-move requires --node-name <target-node> (or --remove to clear placement)');
+      }
+
+      const normalizeKind = (k) =>
+        ({
+          deploy: 'deployment',
+          deployments: 'deployment',
+          deployment: 'deployment',
+          sts: 'statefulset',
+          statefulsets: 'statefulset',
+          statefulset: 'statefulset',
+          ds: 'daemonset',
+          daemonsets: 'daemonset',
+          daemonset: 'daemonset',
+          rs: 'replicaset',
+          replicasets: 'replicaset',
+          replicaset: 'replicaset',
+          rc: 'replicationcontroller',
+          replicationcontroller: 'replicationcontroller',
+          job: 'job',
+          jobs: 'job',
+          cj: 'cronjob',
+          cronjob: 'cronjob',
+          cronjobs: 'cronjob',
+          po: 'pod',
+          pod: 'pod',
+          pods: 'pod',
+          svc: 'service',
+          service: 'service',
+          services: 'service',
+        })[k] || k;
+
+      // Kinds that own a pod template we can patch; rolloutKinds additionally
+      // support `kubectl rollout restart` to reschedule existing pods now.
+      const templated = [
+        'deployment',
+        'statefulset',
+        'daemonset',
+        'replicaset',
+        'job',
+        'cronjob',
+        'replicationcontroller',
+      ];
+      const rolloutKinds = ['deployment', 'statefulset', 'daemonset'];
+      const templateSelectorPath = (kind) =>
+        kind === 'cronjob'
+          ? ['spec', 'jobTemplate', 'spec', 'template', 'spec', 'nodeSelector']
+          : ['spec', 'template', 'spec', 'nodeSelector'];
+
+      // Resolve the desired nodeSelector. Custom --labels enables reusable pools;
+      // otherwise pin by the always-present hostname label.
+      let selector = { 'kubernetes.io/hostname': node };
+      if (!remove && options.labels) {
+        selector = {};
+        for (const pair of `${options.labels}`
+          .split(',')
+          .map((s) => s.trim())
+          .filter(Boolean)) {
+          const eq = pair.indexOf('=');
+          if (eq < 0) continue;
+          selector[pair.slice(0, eq).trim()] = pair.slice(eq + 1).trim();
+        }
+      }
+
+      // Verify the target node exists, and apply custom labels to it if provided.
+      if (!remove) {
+        const found = shellExec(`kubectl get node ${node} -o name`, {
+          silent: true,
+          stdout: true,
+          silentOnError: true,
+        }).trim();
+        if (!found) throw new Error(`Target node not found: ${node}`);
+        if (options.labels) {
+          const labelArgs = Object.entries(selector)
+            .map(([k, v]) => `${k}=${v}`)
+            .join(' ');
+          const labelCmd = `kubectl label node ${node} ${labelArgs} --overwrite`;
+          if (dryRun) logger.info(`[dry-run] ${labelCmd}`);
+          else shellExec(labelCmd);
+        }
+      }
+
+      const kubectlNames = (kind) =>
+        (
+          shellExec(`kubectl get ${kind} -n ${ns} -o name`, {
+            silent: true,
+            stdout: true,
+            silentOnError: true,
+          }).trim() || ''
+        )
+          .split('\n')
+          .map((s) => s.trim())
+          .filter(Boolean);
+
+      // Build the list of "kind/name" targets from the selection grammar.
+      let targets = [];
+      if (!path) {
+        for (const kind of ['deployment', 'statefulset', 'daemonset']) targets.push(...kubectlNames(kind));
+      } else if (path.includes('/')) {
+        targets = [path];
+      } else {
+        targets = kubectlNames(path);
+      }
+
+      if (targets.length === 0) {
+        logger.warn('node-move: no matching resources found', { path, namespace: ns });
+        return [];
+      }
+
+      // Merge-patch body that sets (or clears, when --remove) the nodeSelector.
+      const buildPatch = (kind) => {
+        const keys = templateSelectorPath(kind);
+        let obj = remove ? null : selector;
+        for (let i = keys.length - 1; i >= 0; i--) obj = { [keys[i]]: obj };
+        return JSON.stringify(obj);
+      };
+
+      const results = [];
+      for (const ref of targets) {
+        const slash = ref.indexOf('/');
+        const rawKind = slash >= 0 ? ref.slice(0, slash) : path && !path.includes('/') ? path : '';
+        const name = slash >= 0 ? ref.slice(slash + 1) : ref;
+        const kind = normalizeKind(`${rawKind}`.split('.')[0].toLowerCase());
+
+        if (!templated.includes(kind)) {
+          logger.warn(`node-move: ${kind}/${name} is not a schedulable controller; skipping`, {
+            hint: 'move its owning controller (deployment/statefulset/daemonset) instead',
+          });
+          results.push({ ref, kind, status: 'skipped' });
+          continue;
+        }
+
+        // Idempotency: skip the patch + rollout if the resource is already where
+        // we want it. Compares the live pod-template nodeSelector against the
+        // desired placement so a repeated run does not trigger an unnecessary
+        // rollout restart.
+        const basePath = kind === 'cronjob' ? 'spec.jobTemplate.spec.template.spec' : 'spec.template.spec';
+        const jsonpath = (expr) =>
+          shellExec(`kubectl get ${kind} ${name} -n ${ns} -o jsonpath='${expr}'`, {
+            silent: true,
+            stdout: true,
+            silentOnError: true,
+            disableLog: true,
+          }).trim();
+
+        if (remove) {
+          const current = jsonpath(`{.${basePath}.nodeSelector}`);
+          if (!current || current === 'map[]') {
+            logger.info(`node-move: ${kind}/${name} already has no nodeSelector; nothing to clear`);
+            results.push({ ref, kind, status: 'already-cleared' });
+            continue;
+          }
+        } else {
+          const alreadyOnNode = Object.entries(selector).every(([k, v]) => {
+            const esc = k.replace(/\./g, '\\.');
+            return jsonpath(`{.${basePath}.nodeSelector.${esc}}`) === v;
+          });
+          if (alreadyOnNode) {
+            logger.info(`node-move: ${kind}/${name} already pinned to ${node}; skipping`, { namespace: ns });
+            results.push({ ref, kind, status: 'already-on-node', node });
+            continue;
+          }
+        }
+
+        const patchCmd = `kubectl patch ${kind} ${name} -n ${ns} --type=merge -p '${buildPatch(kind)}'`;
+        const restartCmd = `kubectl rollout restart ${kind} ${name} -n ${ns}`;
+        if (dryRun) {
+          logger.info(`[dry-run] ${patchCmd}`);
+          if (rolloutKinds.includes(kind)) logger.info(`[dry-run] ${restartCmd}`);
+          results.push({ ref, kind, status: 'dry-run', node: remove ? undefined : node });
+          continue;
+        }
+
+        shellExec(patchCmd);
+        if (rolloutKinds.includes(kind)) shellExec(restartCmd);
+        logger.info(remove ? `Cleared node placement: ${kind}/${name}` : `Moved ${kind}/${name} -> ${node}`, {
+          namespace: ns,
+        });
+        results.push({ ref, kind, status: remove ? 'cleared' : 'moved', node: remove ? undefined : node });
+      }
+
+      logger.info('node-move complete', { namespace: ns, node: remove ? null : node, count: results.length });
+      return results;
+    },
+
     /**
      * @method dev-hosts-expose
      * @description Deploys a specified service in development mode with `/etc/hosts` modification for local access.
      * @param {string} path - The input value, identifier, or path for the operation (used as the deployment ID to deploy).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'dev-hosts-expose': (path, options = DEFAULT_OPTION) => {
@@ -377,7 +615,7 @@ class UnderpostRun {
      * @method dev-hosts-restore
      * @description Restores the `/etc/hosts` file to its original state after modifications made during development deployments.
      * @param {string} path - The input value, identifier, or path for the operation.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'dev-hosts-restore': (path, options = DEFAULT_OPTION) => {
@@ -388,11 +626,13 @@ class UnderpostRun {
      * @method cluster-build
      * @description Build configuration for cluster deployment.
      * @param {string} path - The input value, identifier, or path for the operation.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'cluster-build': (path, options = DEFAULT_OPTION) => {
-      const nodeOptions = options.nodeName ? ` --node-name ${options.nodeName}` : '';
+      const nodeOptions =
+        (options.nodeName ? ` --node-name ${options.nodeName}` : '') +
+        (options.sshKeyPath ? ` --ssh-key-path ${options.sshKeyPath}` : '');
       shellExec(`node bin run clean`);
       shellExec(`node bin run --dev sync-replica template-deploy${nodeOptions}`);
       shellExec(`node bin run sync-replica template-deploy${nodeOptions}`);
@@ -410,7 +650,7 @@ class UnderpostRun {
      * and optionally triggers engine-<conf-id> CI with sync/init which in turn dispatches the CD workflow
      * after the build chain completes (template → ghpkg → engine-<conf-id> → CD).
      * @param {string} path - The deployment path identifier (e.g., 'sync-engine-core', 'init-engine-core', or empty for build-only).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'template-deploy': (path = '', options = DEFAULT_OPTION) => {
@@ -422,14 +662,12 @@ class UnderpostRun {
         return;
       }
       shellExec(`${baseCommand} run pull`);
+      shellExec(`${baseCommand} run shared-dir`);
 
-      // Capture last N commit messages for propagation.
-      // When --from-n-commit is not set, auto-detect unpushed commit count (same as --unpush flag).
-      const fromN =
-        options.fromNCommit && parseInt(options.fromNCommit) > 0
-          ? parseInt(options.fromNCommit)
-          : Underpost.repo.getUnpushedCount('.').count;
-      const message = shellExec(`node bin cmt --changelog ${fromN} --changelog-no-hash`, {
+      // Capture the sanitized message from the last N commits (--from-n-commit, default 1) for
+      // propagation to pwa-microservices-template and every engine-* repo.
+      const fromN = parseInt(options.fromNCommit) > 0 ? parseInt(options.fromNCommit) : 1;
+      const sanitizedMessage = shellExec(`node bin cmt --changelog-msg --from-n-commit ${fromN} --changelog-no-hash`, {
         silent: true,
         stdout: true,
       }).trim();
@@ -441,8 +679,6 @@ class UnderpostRun {
       );
       shellCd('/home/dd/engine');
 
-      const sanitizedMessage = Underpost.repo.sanitizeChangelogMessage(message);
-
       // Push engine repo so workflow YAML changes reach GitHub
       shellExec(`git reset`);
       shellExec(`${baseCommand} push . ${options.force ? '-f ' : ''}${process.env.GITHUB_USERNAME}/engine`);
@@ -490,7 +726,7 @@ class UnderpostRun {
      * @method template-deploy-local
      * @description Similar to `template-deploy` but runs the workflow locally without dispatching GitHub Actions. It pulls the latest changes, pushes to GitHub, builds the template, and optionally triggers a local release with CI push.
      * @param {string} path - The deployment path identifier (e.g., 'sync-engine-core', 'init-engine-core', or empty for build-only).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'template-deploy-local': async (path, options = DEFAULT_OPTION) => {
@@ -502,18 +738,14 @@ class UnderpostRun {
         return;
       }
       shellExec(`${baseCommand} run pull`);
+      shellExec(`${baseCommand} run shared-dir`);
 
-      // Capture last N commit messages from the engine repo.
-      // When --from-n-commit is not set, auto-detect unpushed commit count (same as --unpush flag).
-      const fromN =
-        options.fromNCommit && parseInt(options.fromNCommit) > 0
-          ? parseInt(options.fromNCommit)
-          : Underpost.repo.getUnpushedCount('.').count;
-      const rawMessage = shellExec(`node bin cmt --changelog ${fromN} --changelog-no-hash`, {
+      // Capture the sanitized message from the last N commits (--from-n-commit, default 1).
+      const fromN = parseInt(options.fromNCommit) > 0 ? parseInt(options.fromNCommit) : 1;
+      const sanitizedMessage = shellExec(`node bin cmt --changelog-msg --from-n-commit ${fromN} --changelog-no-hash`, {
         silent: true,
         stdout: true,
       }).trim();
-      const sanitizedMessage = Underpost.repo.sanitizeChangelogMessage(rawMessage);
 
       const { triggerCmd } = path
         ? await Underpost.release.ci(path, sanitizedMessage, options)
@@ -525,15 +757,14 @@ class UnderpostRun {
      * @description Dispatches the Docker image CI workflow (`docker-image[.<runtime>].ci.yml`) via `workflow_dispatch`.
      * Repository resolution is delegated to `Underpost.repo.resolveInstanceRepo(path)`.
      * @param {string} path - Optional runtime / workflow suffix (e.g. `cyberia-server`, `cyberia-client`).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'docker-image': (path, options = DEFAULT_OPTION) => {
-      const repo = Underpost.repo.resolveInstanceRepo(path);
+      const repo = Underpost.repo.resolveInstanceRepo(path, options.dev);
       Underpost.repo.dispatchWorkflow({
         repo,
-        workflowFile: `docker-image${path ? `.${path}` : ''}.ci.yml`,
-        ref: 'master',
+        workflowFile: `docker-image${path ? `.${path}` : ''}${options.dev ? '.dev' : ''}.ci.yml`,
         inputs: {},
       });
     },
@@ -541,7 +772,7 @@ class UnderpostRun {
      * @method clean
      * @description Changes directory to the provided path (defaulting to `/home/dd/engine`) and runs `node bin/deploy clean-core-repo`.
      * @param {string} path - The input value, identifier, or path for the operation (used as the optional directory path).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     clean: (path = '', options = DEFAULT_OPTION) => {
@@ -552,7 +783,7 @@ class UnderpostRun {
      * @method pull
      * @description Clones or pulls updates for the `engine` and `engine-private` repositories into `/home/dd/engine` and `/home/dd/engine/engine-private`.
      * @param {string} path - The input value, identifier, or path for the operation.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     pull: (path, options = DEFAULT_OPTION) => {
@@ -579,7 +810,7 @@ class UnderpostRun {
      * @method release-deploy
      * @description Executes deployment (`underpost run deploy`) for all deployment IDs listed in `./engine-private/deploy/dd.router`.
      * @param {string} path - The input value, identifier, or path for the operation.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'release-deploy': (path, options = DEFAULT_OPTION) => {
@@ -595,7 +826,7 @@ class UnderpostRun {
      * @method ssh-deploy
      * @description Dispatches the corresponding CD workflow for SSH-based deployment, replacing empty commits with workflow_dispatch.
      * @param {string} path - The deployment identifier (e.g., 'engine-core', 'sync-engine-core', 'init-engine-core').
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'ssh-deploy': (path, options = DEFAULT_OPTION) => {
@@ -610,9 +841,9 @@ class UnderpostRun {
         job = 'init';
         confId = path.replace(/^init-/, '');
       }
-
+      const repo = Underpost.repo.resolveInstanceRepo(confId, options.dev);
       Underpost.repo.dispatchWorkflow({
-        repo: `${process.env.GITHUB_USERNAME}/engine`,
+        repo,
         workflowFile: `${confId}.cd.yml`,
         ref: 'master',
         inputs: { job },
@@ -623,7 +854,7 @@ class UnderpostRun {
      * @description Opens a Visual Studio Code (VS Code) session for the specified path using `node ${underpostRoot}/bin/zed ${path}`,
      * or installs Zed and sublime-text IDE if `path` is 'install'.
      * @param {string} path - The input value, identifier, or path for the operation (used as the path to the directory to open in the IDE).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     ide: (path = '', options = DEFAULT_OPTION) => {
@@ -650,7 +881,7 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
      * @method crypto-policy
      * @description Sets the system's crypto policies to `DEFAULT:SHA1` using `update-crypto-policies` command.
      * @param {string} path - The input value, identifier, or path for the operation.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'crypto-policy': (path, options = DEFAULT_OPTION) => {
@@ -665,7 +896,7 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
      * `deployment.yaml`. Useful when you want to force `Always` so the kubelet re-pulls a mutable tag on every rollout. Example:
      *   `node bin run sync dd-core --kubeadm --image-pull-policy Always`
      * @param {string} path - The input value, identifier, or path for the operation (used as a comma-separated string containing deploy parameters).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     sync: async (path, options = DEFAULT_OPTION) => {
@@ -686,7 +917,7 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
       replicas = replicas ? replicas : defaultPath[1];
       versions = versions ? versions.replaceAll('+', ',') : defaultPath[2];
       image = image ? image : defaultPath[3];
-      node = node ? node : defaultPath[4];
+      node = node ? node : options.nodeName ? options.nodeName : defaultPath[4];
       shellExec(`${baseCommand} cluster --ns-use ${options.namespace}`);
 
       if (image && !image.startsWith('localhost'))
@@ -713,6 +944,11 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
       let targetTraffic = currentTraffic ? (currentTraffic === 'blue' ? 'green' : 'blue') : 'green';
       if (targetTraffic) versions = versions ? versions : targetTraffic;
 
+      const ignorePods =
+        isDeployRunnerContext(path, options) && targetTraffic
+          ? Underpost.kubectl.get(`${deployId}-${env}-${targetTraffic}`, 'pods', options.namespace).map((p) => p.NAME)
+          : [];
+
       const timeoutFlags = Underpost.deploy.timeoutFlagsFactory(options);
       const cmdString = options.cmd
         ? ' --cmd ' + (options.cmd.find((c) => c.match('"')) ? '"' + options.cmd + '"' : "'" + options.cmd + "'")
@@ -722,13 +958,14 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
       const skipFullBuildFlag = options.skipFullBuild ? ' --skip-full-build' : '';
       const pullBundleFlag = options.pullBundle ? ' --pull-bundle' : '';
       const imagePullPolicyFlag = options.imagePullPolicy ? ` --image-pull-policy ${options.imagePullPolicy}` : '';
+      const sshKeyPathFlag = options.sshKeyPath ? ` --ssh-key-path ${options.sshKeyPath}` : '';
 
       shellExec(
         `${baseCommand} deploy${clusterFlag} --build-manifest --sync --info-router --replicas ${replicas} --node ${node}${
           image ? ` --image ${image}` : ''
         }${versions ? ` --versions ${versions}` : ''}${
           options.namespace ? ` --namespace ${options.namespace}` : ''
-        }${timeoutFlags}${cmdString}${gitCleanFlag}${skipFullBuildFlag}${pullBundleFlag}${imagePullPolicyFlag} ${deployId} ${env}`,
+        }${timeoutFlags}${cmdString}${gitCleanFlag}${skipFullBuildFlag}${pullBundleFlag}${imagePullPolicyFlag}${sshKeyPathFlag} ${deployId} ${env}`,
       );
 
       if (isDeployRunnerContext(path, options)) {
@@ -737,13 +974,13 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
           `${baseCommand} db ${deployId} ${clusterFlag}${baseClusterCommand} --repo-backup --primary-pod --git --force-clone --preserveUUID ${options.namespace ? ` --ns ${options.namespace}` : ''}`,
         );
         shellExec(
-          `${baseCommand} deploy${clusterFlag}${cmdString} --replicas ${replicas} --disable-update-proxy ${deployId} ${env} --versions ${versions}${
+          `${baseCommand} deploy${clusterFlag}${cmdString} --replicas ${replicas} --node ${node} --disable-update-proxy ${deployId} ${env} --versions ${versions}${
             options.namespace ? ` --namespace ${options.namespace}` : ''
-          }${timeoutFlags}${gitCleanFlag}${imagePullPolicyFlag}`,
+          }${timeoutFlags}${gitCleanFlag}${imagePullPolicyFlag}${sshKeyPathFlag}`,
         );
         if (!targetTraffic)
           targetTraffic = Underpost.deploy.getCurrentTraffic(deployId, { namespace: options.namespace });
-        await Underpost.deploy.monitorReadyRunner(deployId, env, targetTraffic, [], options.namespace, 'underpost');
+        await Underpost.monitor.monitorReadyRunner(deployId, env, targetTraffic, ignorePods, options.namespace);
         Underpost.deploy.switchTraffic(deployId, env, targetTraffic, replicas, options.namespace, options);
       } else
         logger.info('current traffic', Underpost.deploy.getCurrentTraffic(deployId, { namespace: options.namespace }));
@@ -753,7 +990,7 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
      * @method stop
      * @description Stops a deployment by deleting the corresponding Kubernetes deployment and service resources.
      * @param {string} path - The input value, identifier, or path for the operation (used to determine which traffic to stop).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     stop: async (path = '', options = DEFAULT_OPTION) => {
@@ -777,7 +1014,7 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
      * @method ssh-deploy-stop
      * @description Stops a remote deployment via SSH by executing the appropriate Underpost command on the remote server.
      * @param {string} path - The input value, identifier, or path for the operation (used to determine which traffic to stop).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'ssh-deploy-stop': async (path, options = DEFAULT_OPTION) => {
@@ -804,7 +1041,7 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
      * @method ssh-deploy-db-rollback
      * @description Performs a database rollback on remote deployment via SSH.
      * @param {string} path - Comma-separated deployId and optional number of commits to reset (format: "deployId,nCommits")
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @param {string} options.deployId - The deployment identifier
      * @param {string} options.user - The SSH user for credential lookup
      * @param {boolean} options.dev - Development mode flag
@@ -831,7 +1068,7 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
      * @method ssh-deploy-db
      * @description Imports/restores a database on remote deployment via SSH.
      * @param {string} path - The deployment ID for database import
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @param {string} options.deployId - The deployment identifier
      * @param {string} options.user - The SSH user for credential lookup
      * @param {boolean} options.dev - Development mode flag
@@ -856,7 +1093,7 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
      * @method ssh-deploy-db-status
      * @description Retrieves database status/stats for a deployment (or all deployments from dd.router) via SSH.
      * @param {string} path - Comma-separated deployId(s) or 'dd' to use the dd.router list.
-     * @param {Object} options - Runner options (uses options.deployId for SSH host lookup).
+     * @param {UnderpostRunDefaultOptions} options - Runner options (uses options.deployId for SSH host lookup).
      * @param {string} options.deployId - Deployment identifier used for SSH config lookup.
      * @param {string} options.user - SSH user for credential lookup.
      * @param {boolean} options.dev - Development mode flag.
@@ -902,7 +1139,7 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
      * @method tz
      * @description Sets the system timezone using `timedatectl set-timezone` command.
      * @param {string} path - The input value, identifier, or path for the operation (used as the timezone string).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     tz: (path, options = DEFAULT_OPTION) => {
@@ -923,7 +1160,7 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
      * @method get-proxy
      * @description Retrieves and logs the HTTPProxy resources in the specified namespace using `kubectl get HTTPProxy`.
      * @param {string} path - The input value, identifier, or path for the operation (used as an optional filter for the HTTPProxy resources).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'get-proxy': async (path = '', options = DEFAULT_OPTION) => {
@@ -989,8 +1226,19 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
             // pathRewritePolicy,
           });
         if (options.tls) {
-          shellExec(`sudo kubectl delete Certificate ${_host} -n ${options.namespace} --ignore-not-found`);
-          proxyYaml += Underpost.deploy.buildCertManagerCertificate({ ...options, host: _host });
+          if (options.test) {
+            const sslDir = `./engine-private/ssl/${_host}`;
+            const nameSafe = _host.replace(/[^a-zA-Z0-9_.-]/g, '_');
+            fs.mkdirpSync(sslDir);
+            shellExec(`bash ./scripts/ssl.sh "${sslDir}" "${_host}"`);
+            shellExec(`kubectl delete secret ${_host} -n ${options.namespace} --ignore-not-found`);
+            shellExec(
+              `kubectl create secret tls ${_host} --cert="${sslDir}/${nameSafe}.pem" --key="${sslDir}/${nameSafe}-key.pem" -n ${options.namespace}`,
+            );
+          } else {
+            shellExec(`sudo kubectl delete Certificate ${_host} -n ${options.namespace} --ignore-not-found`);
+            proxyYaml += Underpost.deploy.buildCertManagerCertificate({ ...options, host: _host });
+          }
         }
         // console.log(proxyYaml);
         shellExec(`kubectl delete HTTPProxy ${_host} --namespace ${options.namespace} --ignore-not-found`);
@@ -1019,7 +1267,7 @@ EOF
     /**
      * @method instance
      * @param {string} path - The input value, identifier, or path for the operation (used as a comma-separated string containing workflow parameters).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     instance: async (path = '', options = DEFAULT_OPTION) => {
@@ -1059,6 +1307,7 @@ EOF
         // Examples images:
         // `underpost/underpost-engine:${Underpost.version}`
         // `localhost/rockylinux9-underpost:${Underpost.version}`
+        if (options.imageName) _image = options.imageName;
         if (!_image) _image = `underpost/underpost-engine:${Underpost.version}`;
 
         if (_image && !_image.startsWith('localhost'))
@@ -1087,9 +1336,16 @@ EOF
               deployId: _deployId,
               env,
               version: targetTraffic,
-              nodeName: options.nodeName,
+              nodeName: Underpost.deploy.resolveDeployNode({
+                node: options.nodeName,
+                kind: options.kind,
+                kubeadm: options.kubeadm,
+                k3s: options.k3s,
+                env,
+              }),
               clusterContext: options.k3s ? 'k3s' : options.kubeadm ? 'kubeadm' : 'kind',
               gitClean: options.gitClean || false,
+              sshKeyPath: options.sshKeyPath || '',
             });
         // Regenerate the parent deploy's gRPC ClusterIP service pointing to the
         // parent's current traffic colour and apply it before the instance pod starts so
@@ -1154,12 +1410,16 @@ EOF
 `,
           { disableLog: true },
         );
-        const { ready, readyPods } = await Underpost.deploy.monitorReadyRunner(
+        // Custom instances run a bare binary (no `underpost start` / internal
+        // HTTP endpoint): Kubernetes readiness is the running signal and
+        // container-status is read via exec. See `Deploy custom instance to K8S.md`.
+        const { ready, readyPods } = await Underpost.monitor.monitorReadyRunner(
           _deployId,
           env,
           targetTraffic,
           ignorePods,
           options.namespace,
+          { readyGate: 'kubernetes', statusTransport: 'exec' },
         );
 
         if (!ready) {
@@ -1169,7 +1429,7 @@ EOF
         shellExec(
           `${baseCommand} run${baseClusterCommand} --namespace ${options.namespace}` +
             `${options.nodeName ? ` --node-name ${options.nodeName}` : ''}` +
-            `${options.tls ? ` --tls` : ''}` +
+            `${options.tls ? ` --tls ${options.test ? '--test' : ''}` : ''}` +
             ` instance-promote '${path}'`,
         );
       }
@@ -1179,12 +1439,39 @@ EOF
       }
     },
 
+    /**
+     * @method deploy-key
+     * @description Copies the deploy key for a specific user and deployId to a temporary location on the local machine.
+     * @param {string} path - The input value, identifier, or path for the operation (not used in this method).
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
+     * @param {string} options.user - The user for which to copy the deploy key.
+     * @param {string} options.deployId - The deployment identifier associated with the deploy key.
+     * @memberof UnderpostRun
+     */
+    'deploy-key': (path, options = DEFAULT_OPTION) => {
+      const prefix = 'dd-key';
+      if (options.reset) {
+        shellExec(`rm -rf /home/dd/tmp/${prefix}_*`);
+        return;
+      }
+      if (!options.user || !options.deployId) {
+        logger.error('Both --user and --deploy-id options are required to copy the deploy key.');
+        return;
+      }
+      const targetPath = `/home/dd/tmp/${prefix}_${s4()}${s4()}`;
+      fs.mkdirSync('/home/dd/tmp', { recursive: true });
+      fs.copyFileSync(`./engine-private/conf/${options.deployId}/users/${options.user}/id_rsa`, targetPath);
+      logger.info(`Copied deploy key to ${targetPath}`);
+      if (options.copy) pbcopy(targetPath);
+    },
+
     /**
      * @method instance-build-manifest
      * @description Builds a Kubernetes Deployment + Service manifest for a specific instance entry
-     * from `conf.instances.json` and writes it to a file.
-     * Traffic colour is automatically chosen as the opposite of the current live colour (blue/green),
-     * defaulting to `blue` when no deployment is running yet.
+     * from `conf.instances.json` and writes it to a file. This is a purely local
+     * artifact generator: it never probes a live cluster. Traffic colour defaults
+     * to the canonical initial `blue` and can be overridden with `--traffic`; the
+     * real blue/green swap is resolved at deploy time (`deploy --sync`).
      *
      * If `--build` is supplied the image is built from the project Dockerfile and loaded into the
      * cluster before the manifest is written (kind by default; `--kubeadm` / `--k3s` override).
@@ -1195,7 +1482,7 @@ EOF
      *   `<projectPath>/manifests/<env>/deployment.yaml`.
      *   In production, files are also copied to `<projectPath>/Dockerfile` and
      *   `<projectPath>/deployment.yaml`.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'instance-build-manifest': (path, options = DEFAULT_OPTION) => {
@@ -1274,7 +1561,7 @@ EOF
           path: projectPath,
           imageName: _image,
           podmanSave: true,
-          imagePath: projectPath,
+          imageOutPath: projectPath,
           kind: isKind,
           kubeadm: !!options.kubeadm,
           k3s: !!options.k3s,
@@ -1287,15 +1574,8 @@ EOF
         });
       }
 
-      // Determine target traffic: opposite of current, or 'blue' if nothing is running yet.
-      const currentTraffic = Underpost.deploy.getCurrentTraffic(_deployId, {
-        hostTest: _host,
-        namespace: options.namespace,
-      });
-      const targetTraffic = currentTraffic ? (currentTraffic === 'blue' ? 'green' : 'blue') : 'blue';
-
-      // Resolve {{grpc-service-dns}} using the parent deploy's current (or default) traffic.
-      const parentTraffic = Underpost.deploy.getCurrentTraffic(deployId, { namespace: options.namespace }) || 'blue';
+      const targetTraffic = options.traffic || 'blue';
+      const parentTraffic = targetTraffic;
       const resolvedCmd = _cmd[env].map((c) =>
         c.replaceAll(
           '{{grpc-service-dns}}',
@@ -1343,11 +1623,86 @@ EOF
         image: _image,
       });
 
+      // --- Sibling manifests (pv-pvc, proxy, grpc-service) ------------------
+      // Emit the same apply-able set the parent deploy writes to build/<env>,
+      // scoped to this instance so the project repo and engine-private ship
+      // more than just deployment.yaml. Content matches what `run instance`
+      // creates dynamically at deploy time (deployVolume / instance-promote /
+      // the parent gRPC ClusterIP), so a static `kubectl apply` is equivalent.
+      const pvDataNode = Underpost.deploy.resolveDeployNode({
+        node: options.nodeName,
+        kind: options.kind,
+        kubeadm: options.kubeadm,
+        k3s: options.k3s,
+        env,
+      });
+
+      // pv-pvc.yaml — one PV+PVC per instance volume; names mirror deployVolume.
+      let pvPvcYaml = '';
+      for (const volume of _volumes || []) {
+        if (!volume.claimName) continue;
+        const pvcId = `${volume.claimName}-${_deployId}-${env}-${targetTraffic}`;
+        const pvId = `${volume.claimName.replace('pvc-', 'pv-')}-${_deployId}-${env}-${targetTraffic}`;
+        pvPvcYaml += `---\n${Underpost.deploy.persistentVolumeFactory({
+          pvcId,
+          namespace: options.namespace,
+          hostPath: `/home/dd/engine/volume/${pvId}`,
+          nodeName: pvDataNode,
+        })}\n`;
+      }
+
+      // proxy.yaml — HTTPProxy for the instance host (mirrors instance-promote).
+      const proxyYaml =
+        Underpost.deploy.baseProxyYamlFactory({ host: _host, env, options }) +
+        Underpost.deploy.deploymentYamlServiceFactory({
+          path: _path,
+          port: _fromPort,
+          deployId: _deployId,
+          env,
+          deploymentVersions: [targetTraffic],
+        });
+
+      // grpc-service.yaml — the parent deploy's gRPC ClusterIP (shared; the
+      // instance cmd resolves {{grpc-service-dns}} to it). Reuse the parent's
+      // generated manifest when present rather than regenerating it here.
+      const parentGrpcServicePath = `./engine-private/conf/${deployId}/build/${env}/grpc-service.yaml`;
+      const grpcServiceYaml = fs.existsSync(parentGrpcServicePath)
+        ? fs.readFileSync(parentGrpcServicePath, 'utf8')
+        : '';
+
+      // Write the sibling set next to deployment.yaml (project) and into the
+      // engine-private per-instance build dir (mirrors instances/<id>/ layout).
+      const instanceBuildDir = `./engine-private/conf/${deployId}/instances/${_id}/build/${env}`;
+      fs.mkdirpSync(instanceBuildDir);
+      fs.writeFileSync(`${instanceBuildDir}/deployment.yaml`, deploymentYaml, 'utf8');
+      const siblingManifests = {
+        'pv-pvc.yaml': pvPvcYaml,
+        'proxy.yaml': proxyYaml,
+        'grpc-service.yaml': grpcServiceYaml,
+      };
+      for (const [name, content] of Object.entries(siblingManifests)) {
+        if (!content) continue;
+        fs.writeFileSync(`${envManifestPath}/${name}`, content, 'utf8');
+        fs.writeFileSync(`${instanceBuildDir}/${name}`, content, 'utf8');
+      }
+      logger.info('[instance-build-manifest] Sibling manifests written', {
+        project: envManifestPath,
+        enginePrivate: instanceBuildDir,
+        pvPvc: !!pvPvcYaml,
+        proxy: !!proxyYaml,
+        grpcService: !!grpcServiceYaml,
+      });
+
       if (env === 'production') {
         if (fs.existsSync(dockerfileManifestPath)) {
           fs.copyFileSync(dockerfileManifestPath, `${rootPath}/Dockerfile`);
         }
         fs.copyFileSync(outputPath, `${rootPath}/deployment.yaml`);
+        // Sibling manifests alongside deployment.yaml at the project root.
+        for (const name of ['pv-pvc.yaml', 'proxy.yaml', 'grpc-service.yaml']) {
+          const src = `${envManifestPath}/${name}`;
+          if (fs.existsSync(src)) fs.copyFileSync(src, `${rootPath}/${name}`);
+        }
         logger.info('[instance-build-manifest] Production artifacts copied to project root', {
           rootPath,
           dockerfile: `${rootPath}/Dockerfile`,
@@ -1359,6 +1714,25 @@ EOF
           fs.copyFileSync(ciSrc, `${rootPath}/.github/workflows/docker-image.${_runtime}.ci.yml`);
           logger.info(`[instance-build-manifest] CI workflow copied`, { src: ciSrc });
         }
+
+        // Ship the development variant alongside production so the instance repo
+        // is self-contained: the dev Dockerfile (built by the -dev CI workflow
+        // into underpost/<runtime>-dev, consumed by the development compose
+        // stack) and its dispatchable workflow. Both are optional — synced only
+        // when the source-of-truth files exist in the engine repo.
+        if (_runtime) {
+          const devDockerfileSrc = `src/runtime/${_runtime}/Dockerfile.dev`;
+          if (fs.existsSync(devDockerfileSrc)) {
+            fs.copyFileSync(devDockerfileSrc, `${rootPath}/Dockerfile.dev`);
+            logger.info('[instance-build-manifest] Dev Dockerfile copied', { src: devDockerfileSrc });
+          }
+          const devCiSrc = `./.github/workflows/docker-image.${_runtime}.dev.ci.yml`;
+          if (fs.existsSync(devCiSrc)) {
+            if (!fs.existsSync(`${rootPath}/.github/workflows`)) fs.mkdirpSync(`${rootPath}/.github/workflows`);
+            fs.copyFileSync(devCiSrc, `${rootPath}/.github/workflows/docker-image.${_runtime}.dev.ci.yml`);
+            logger.info(`[instance-build-manifest] Dev CI workflow copied`, { src: devCiSrc });
+          }
+        }
       }
     },
 
@@ -1366,7 +1740,7 @@ EOF
      * @method ls-deployments
      * @description Retrieves and logs a table of Kubernetes deployments using `Underpost.deploy.get`.
      * @param {string} path - The input value, identifier, or path for the operation (used as an optional deployment name filter).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'ls-deployments': async (path, options = DEFAULT_OPTION) => {
@@ -1377,7 +1751,7 @@ EOF
      * @method host-update
      * @description Executes the `rocky-setup.sh` script to update the host system configuration.
      * @param {string} path - The input value, identifier, or path for the operation.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'host-update': async (path, options = DEFAULT_OPTION) => {
@@ -1393,7 +1767,7 @@ EOF
      * the systemd cgroup driver, enables the `crio` service, and writes `/etc/crictl.yaml`
      * so that `crictl` targets the CRI-O socket by default.
      * @param {string} path - Unused.
-     * @param {Object} options - The default underpost runner options for customizing workflow.
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow.
      * @memberof UnderpostRun
      */
     'install-crio': (path, options = DEFAULT_OPTION) => {
@@ -1407,8 +1781,24 @@ gpgcheck=1
 gpgkey=https://download.opensuse.org/repositories/isv:/cri-o:/stable:/v1.33/rpm/repodata/repomd.xml.key
 EOF`);
       shellExec(`sudo dnf -y install cri-o`);
-      // crictl is in the kubernetes repo but excluded by default — install it explicitly
-      shellExec(`sudo yum install -y cri-tools --disableexcludes=kubernetes`);
+      // Add the Kubernetes repo so cri-tools (crictl CLI) is available.
+      // The repo has exclude=kubelet kubeadm kubectl cri-tools kubernetes-cni, so we
+      // use --disableexcludes=kubernetes to override and install cri-tools.
+      shellExec(`cat <<EOF | sudo tee /etc/yum.repos.d/kubernetes.repo
+[kubernetes]
+name=Kubernetes
+baseurl=https://pkgs.k8s.io/core:/stable:/v1.36/rpm/
+enabled=1
+gpgcheck=1
+gpgkey=https://pkgs.k8s.io/core:/stable:/v1.36/rpm/repodata/repomd.xml.key
+exclude=kubelet kubeadm kubectl cri-tools kubernetes-cni
+EOF`);
+      shellExec(`sudo yum install -y cri-tools --disableexcludes=kubernetes 2>/dev/null || {
+        CRICTL_VERSION="v1.31.0"
+        ARCH="amd64"
+        log "Kubernetes repo not available, downloading crictl binary directly..."
+        curl -sL "https://github.com/kubernetes-sigs/cri-tools/releases/download/\${CRICTL_VERSION}/crictl-\${CRICTL_VERSION}-\${ARCH}.tar.gz" | sudo tar -C /usr/local/bin -xz
+      }`);
       // Ensure CRI-O uses systemd cgroup driver (matches kubelet default)
       shellExec(`sudo sed -i 's/^#\?cgroup_manager =.*/cgroup_manager = "systemd"/' /etc/crio/crio.conf`, {
         silentOnError: true,
@@ -1428,7 +1818,7 @@ EOF`);
      * @method dd-container
      * @description Deploys a development or debug container tasks jobs, setting up necessary volumes and images, and running specified commands within the container.
      * @param {string} path - The input value, identifier, or path for the operation (used as the command to run inside the container).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'dd-container': async (path = '', options = DEFAULT_OPTION) => {
@@ -1436,8 +1826,8 @@ EOF`);
       const baseClusterCommand = options.dev ? ' --dev' : '';
       const currentImage = options.imageName
         ? options.imageName
-        : Underpost.deploy
-            .getCurrentLoadedImages(options.nodeName ? options.nodeName : 'kind-worker', false)
+        : Underpost.image
+            .getCurrentLoaded(options.nodeName ? options.nodeName : 'kind-worker', false)
             .find((o) => o.IMAGE.match('underpost'));
       const podName = options.podName || `underpost-dev-container`;
       const volumeHostPath = options.claimName || '/home/dd';
@@ -1455,7 +1845,11 @@ EOF`);
       }
 
       if (!currentImage)
-        shellExec(`${baseCommand} image${baseClusterCommand} --pull-base ${options.dev ? '--kind' : '--kubeadm'}`);
+        shellExec(
+          `${baseCommand} image${baseClusterCommand} --pull-base --build --path ${
+            options.dev ? '.' : options.underpostRoot
+          } ${options.dev ? '--kind' : '--kubeadm'}`,
+        );
       // shellExec(`kubectl delete pod ${podName} --ignore-not-found`);
 
       const payload = {
@@ -1483,7 +1877,7 @@ EOF`);
      * @method ip-info
      * @description Executes the `ip-info.sh` script to display IP-related information for the specified path.
      * @param {string} path - The input value, identifier, or path for the operation (used as an argument to the script).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'ip-info': (path, options = DEFAULT_OPTION) => {
@@ -1496,7 +1890,7 @@ EOF`);
      * @method db-client
      * @description Deploys and exposes the Adminer database client application (using `adminer:4.7.6-standalone` image) on the cluster.
      * @param {string} path - The input value, identifier, or path for the operation.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'db-client': async (path, options = DEFAULT_OPTION) => {
@@ -1523,7 +1917,7 @@ EOF`);
      * @method git-conf
      * @description Configures Git global and local user name and email settings based on the provided `path` (formatted as `username,email`), or defaults to environment variables.
      * @param {string} path - The input value, identifier, or path for the operation (used as a comma-separated string: `username,email`).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'git-conf': (path = '', options = DEFAULT_OPTION) => {
@@ -1571,7 +1965,7 @@ EOF`);
      * TLS config, deletes stale Certificate resources, then reapplies the proxy and secret.yaml
      * (cert-manager Certificate resources) for each affected deployment.
      * @param {string} path - The input value, identifier, or path for the operation (used as a comma-separated string: `deployId,env,replicas`).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     promote: async (path, options = DEFAULT_OPTION) => {
@@ -1625,10 +2019,31 @@ EOF`);
      * @method metrics
      * @description Deploys Prometheus and Grafana for metrics monitoring, targeting the hosts defined in the deployment configuration files.
      * @param {string} path - The input value, identifier, or path for the operation.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     metrics: async (path, options = DEFAULT_OPTION) => {
+      if (path === 'server') {
+        shellExec(
+          `kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/high-availability-1.21+.yaml`,
+        );
+        await timer(2000);
+
+        shellExec(`kubectl patch deployment metrics-server -n kube-system \
+  --type='json' \
+  -p='[
+    {
+      "op":"add",
+      "path":"/spec/template/spec/containers/0/args/-",
+      "value":"--kubelet-insecure-tls"
+    }
+  ]'`);
+        shellExec(`kubectl scale deployment metrics-server \
+  -n kube-system \
+  --replicas=1`);
+
+        return;
+      }
       const deployList = fs.readFileSync(`./engine-private/deploy/dd.router`, 'utf8').split(',');
       let hosts = [];
       for (const deployId of deployList) {
@@ -1642,7 +2057,7 @@ EOF`);
      * @method cluster
      * @description Deploys a full production/development ready Kubernetes cluster environment including MongoDB, MariaDB, Valkey, Contour (Ingress), and Cert-Manager, and deploys all services.
      * @param {string} path - The input value, identifier, or path for the operation.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     cluster: async (path = '', options = DEFAULT_OPTION) => {
@@ -1704,7 +2119,7 @@ EOF`);
      * @method deploy
      * @description Deploys a specified service (identified by `path`) using blue/green strategy, monitors its status, and switches traffic upon readiness.
      * @param {string} path - The input value, identifier, or path for the operation (used as the deployment ID to deploy).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     deploy: async (path, options = DEFAULT_OPTION) => {
@@ -1714,20 +2129,13 @@ EOF`);
       const currentTraffic = Underpost.deploy.getCurrentTraffic(deployId, { namespace: options.namespace });
       const targetTraffic = currentTraffic === 'blue' ? 'green' : 'blue';
       const env = options.dev ? 'development' : 'production';
-      const ignorePods = Underpost.deploy
+      const ignorePods = Underpost.kubectl
         .get(`${deployId}-${env}-${targetTraffic}`, 'pods', options.namespace)
         .map((p) => p.NAME);
 
       shellExec(`sudo kubectl rollout restart deployment/${deployId}-${env}-${targetTraffic} -n ${options.namespace}`);
 
-      await Underpost.deploy.monitorReadyRunner(
-        deployId,
-        env,
-        targetTraffic,
-        ignorePods,
-        options.namespace,
-        'underpost',
-      );
+      await Underpost.monitor.monitorReadyRunner(deployId, env, targetTraffic, ignorePods, options.namespace);
 
       Underpost.deploy.switchTraffic(deployId, env, targetTraffic, options.replicas, options.namespace, options);
     },
@@ -1736,7 +2144,7 @@ EOF`);
      * @method disk-clean
      * @description Executes the `disk-clean-sh` script to perform disk cleanup operations.
      * @param {string} path - The input value, identifier, or path for the operation.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'disk-clean': async (path, options = DEFAULT_OPTION) => {
@@ -1749,7 +2157,7 @@ EOF`);
      * @method disk-devices
      * @description Executes the `disk-devices.sh` script to display information about disk devices.
      * @param {string} path - The input value, identifier, or path for the operation.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'disk-devices': async (path = '/', options = DEFAULT_OPTION) => {
@@ -1762,7 +2170,7 @@ EOF`);
      * @method disk-usage
      * @description Displays disk usage statistics using the `du` command, sorted by size.
      * @param {string} path - The input value, identifier, or path for the operation.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'disk-usage': async (path = '/', options = DEFAULT_OPTION) => {
@@ -1777,7 +2185,7 @@ EOF`);
      * @method dev
      * @description Starts development servers for client, API, and proxy based on provided parameters (deployId, host, path, clientHostPort).
      * @param {string} path - The input value, identifier, or path for the operation (formatted as `deployId,subConf,host,path,clientHostPort`).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     dev: async (path = '', options = DEFAULT_OPTION) => {
@@ -1819,7 +2227,7 @@ EOF`);
         }`;
         shellExec(cmd, { async: true });
       }
-      await awaitDeployMonitor();
+      if ((await awaitDeployMonitor()) !== true) return;
       {
         const cmd = `npm run dev:client ${deployId} ${subConf} ${host} ${_path} proxy${options.tls ? ' tls' : ''}`;
 
@@ -1827,7 +2235,7 @@ EOF`);
           async: true,
         });
       }
-      await awaitDeployMonitor();
+      if ((await awaitDeployMonitor()) !== true) return;
       shellExec(
         `NODE_ENV=development node src/proxy proxy ${deployId} ${subConf} ${host} ${_path}${options.tls ? ' tls' : ''}`,
       );
@@ -1837,7 +2245,7 @@ EOF`);
      * @method service
      * @description Deploys and exposes specific services (like `mongo-express-service`) on the cluster, updating deployment configurations and monitoring status.
      * @param {string} path - The input value, identifier, or path for the operation (formatted as `deployId,serviceId,host,path,replicas,image,node`).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     service: async (path = '', options = DEFAULT_OPTION) => {
@@ -1933,7 +2341,7 @@ EOF`);
      * @method etc-hosts
      * @description Generates and logs the contents for the `/etc/hosts` file based on provided hosts or deployment configurations.
      * @param {string} path - The input value, identifier, or path for the operation (used as a comma-separated list of hosts).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'etc-hosts': async (path = '', options = DEFAULT_OPTION) => {
@@ -1950,7 +2358,7 @@ EOF`);
      * @method sh
      * @description Enables remote control for the Kitty terminal emulator.
      * @param {string} path - The input value, identifier, or path for the operation.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     sh: async (path = '', options = DEFAULT_OPTION) => {
@@ -1970,7 +2378,7 @@ EOF`);
      * @method log
      * @description Searches and highlights keywords in a specified log file, optionally showing surrounding lines.
      * @param {string} path - The input value, identifier, or path for the operation (formatted as `filePath,keywords,lines`).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     log: async (path, options = DEFAULT_OPTION) => {
@@ -1987,7 +2395,7 @@ EOF`);
      * @method ps
      * @description Displays running processes that match a specified path or keyword.
      * @param {string} path - The input value, identifier, or path for the operation (used as a keyword to filter processes).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     ps: async (path = '', options = DEFAULT_OPTION) => {
@@ -2012,7 +2420,7 @@ EOF`);
      * @method pid-info
      * @description Displays detailed information about a process by PID, including service details, command line, executable path, working directory, environment variables, and parent process tree.
      * @param {string} path - The PID of the process to inspect.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'pid-info': (path, options = DEFAULT_OPTION) => {
@@ -2052,7 +2460,7 @@ EOF`);
      * @method background
      * @description Runs a custom command in the background using nohup, logging output to `/var/log/<id>.log` and saving the PID to `/var/run/<id>.pid`.
      * @param {string} path - The command to run in the background (e.g. 'npm run prod:container dd-cyberia-r3').
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     background: (path, options = DEFAULT_OPTION) => {
@@ -2072,7 +2480,7 @@ EOF`);
      * @method ports
      * @description Set on ~/.bashrc alias: ports <port> Command to list listening ports that match the given keyword.
      * @param {string} path - The input value, identifier, or path for the operation (used as a keyword to filter listening ports).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     ports: async (path = '', options = DEFAULT_OPTION) => {
@@ -2084,7 +2492,7 @@ EOF`);
      * @method deploy-test
      * @description Deploys a test deployment (`dd-test`) in either development or production mode, setting up necessary secrets and starting the deployment.
      * @param {string} path - The input value, identifier, or path for the operation (used as the deployment ID).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'deploy-test': async (path, options = DEFAULT_OPTION) => {
@@ -2109,7 +2517,7 @@ EOF`);
      * @method tf-vae-test
      * @description Creates and runs a job pod (`tf-vae-test`) that installs TensorFlow dependencies, clones the TensorFlow docs, and runs the CVAE tutorial script, with a terminal monitor attached.
      * @param {string} path - The input value, identifier, or path for the operation.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'tf-vae-test': async (path, options = DEFAULT_OPTION) => {
@@ -2209,7 +2617,7 @@ EOF`);
      * @method spark-template
      * @description Creates a new Spark template project using `sbt new` in `/home/dd/spark-template`, initializes a Git repository, and runs `replace_params.sh` and `build.sh`.
      * @param {string} path - The input value, identifier, or path for the operation.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'spark-template': (path, options = DEFAULT_OPTION) => {
@@ -2237,7 +2645,7 @@ EOF`);
      * @method pull-rocky-image
      * @description Pulls the base `rockylinux:9` image from Docker Hub via Podman.
      * @param {string} path - The input value, identifier, or path for the operation.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'pull-rocky-image': (path, options = DEFAULT_OPTION) => {
@@ -2247,7 +2655,7 @@ EOF`);
      * @method rmi
      * @description Forces the removal of all local Podman images (`podman rmi $(podman images -qa) --force`).
      * @param {string} path - The input value, identifier, or path for the operation.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     rmi: (path, options = DEFAULT_OPTION) => {
@@ -2257,7 +2665,7 @@ EOF`);
      * @method kill
      * @description Kills processes listening on the specified port(s). If the `path` contains a `+`, it treats it as a range of ports to kill.
      * @param {string} path - The input value, identifier, or path for the operation (used as the port number).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     kill: (path = '', options = DEFAULT_OPTION) => {
@@ -2289,7 +2697,7 @@ EOF`);
      * constraints (lowercase, uppercase, digit, special char, min 8 chars). Logs the plain password
      * to the console or, when `--copy` is set, copies it to the clipboard via pbcopy.
      * @param {string} path - Optional password length (default: 16).
-     * @param {Object} options - The default underpost runner options for customizing workflow.
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow.
      * @param {boolean} options.copy - When true, copies to clipboard instead of logging.
      * @memberof UnderpostRun
      */
@@ -2323,7 +2731,7 @@ EOF`);
      * @method secret
      * @description Creates an Underpost secret named 'underpost' from a file, defaulting to `/home/dd/engine/engine-private/conf/dd-cron/.env.production` if no path is provided.
      * @param {string} path - The input value, identifier, or path for the operation (used as the optional path to the secret file).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     secret: (path, options = DEFAULT_OPTION) => {
@@ -2336,7 +2744,7 @@ EOF`);
      * @method underpost-config
      * @description Calls `Underpost.deploy.configMap` to create a Kubernetes ConfigMap, defaulting to the 'production' environment.
      * @param {string} path - The input value, identifier, or path for the operation (used as the optional configuration name/environment).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'underpost-config': (path = '', options = DEFAULT_OPTION) => {
@@ -2346,7 +2754,7 @@ EOF`);
      * @method gpu-env
      * @description Sets up a dedicated GPU development environment cluster, resetting and then setting up the cluster with `--dedicated-gpu` and monitoring the pods.
      * @param {string} path - The input value, identifier, or path for the operation.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'gpu-env': (path, options = DEFAULT_OPTION) => {
@@ -2359,7 +2767,7 @@ EOF`);
      * @method tf-gpu-test
      * @description Deletes existing `tf-gpu-test-script` ConfigMap and `tf-gpu-test-pod`, and applies the test manifest from `manifests/deployment/tensorflow/tf-gpu-test.yaml`.
      * @param {string} path - The input value, identifier, or path for the operation.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'tf-gpu-test': (path, options = DEFAULT_OPTION) => {
@@ -2373,7 +2781,7 @@ EOF`);
      * @method deploy-job
      * @description Creates and applies a custom Kubernetes Pod manifest (Job) for running arbitrary commands inside a container image (defaulting to a TensorFlow/NVIDIA image).
      * @param {string} path - The input value, identifier, or path for the operation (used as the optional script path or job argument).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'deploy-job': async (path, options = DEFAULT_OPTION) => {
@@ -2504,16 +2912,17 @@ EOF`;
     /**
      * @method push-bundle
      * @description Builds the client zip for the specified deployment, splits it into parts, and uploads to file storage.
-     *   Steps: set env, build+split zip, switch to cron env, upload parts to storage.
+     *   Steps: set env, build+split zip, upload only the zip parts belonging to the deploy-id's hosts (from conf.server.json).
+     *   Only files matching `<host>-<route>.zip.part*` or `<host>-<route>.zip` for each non-skipped route are uploaded.
      * @param {string} path - Optional `fsPath.splitOption` string.
      *   Examples: `build` (default split 8), `build.16` (split 16 MB), `build.none-split` (no split flag).
-     * @param {Object} options - The default underpost runner options for customizing workflow.
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow.
      * @param {string} [options.deployId] - Override deploy ID.
      * @param {boolean} [options.dev] - Use development environment; defaults to production.
      * @memberof UnderpostRun
      */
     'push-bundle': (path = '', options = DEFAULT_OPTION) => {
-      const baseCommand = options.dev ? 'node bin' : 'underpost';
+      const baseCommand = 'node bin'; // options.dev ? 'node bin' : 'underpost';
       const env = options.dev ? 'development' : 'production';
       const deployId = options.deployId || 'dd-default';
       const pathParts = (path || '').split('.');
@@ -2537,11 +2946,54 @@ EOF`;
         }
       }
 
+      const confServerPath = `./engine-private/conf/${deployId}/conf.server.json`;
+      const confServer = fs.existsSync(confServerPath)
+        ? loadReplicas(deployId, loadConfServerJson(confServerPath))
+        : {};
+      const storageFilePath = `engine-private/conf/${deployId}/storage.bundle.json`;
+
       shellExec(`${baseCommand} env ${deployId} ${env}`);
       shellExec(`${baseCommand} client ${deployId} --build-zip${splitFlag ? ` ${splitFlag}` : ''}`);
-      shellExec(
-        `${baseCommand} fs ${fsPath} --recursive --deploy-id ${deployId} --storage-file-path engine-private/conf/${deployId}/storage.bundle.json --force`,
-      );
+
+      const pushBundleFiles = (host, routePath) => {
+        const buildId = `${host}-${routePath.replaceAll('/', '')}`;
+        const buildDir = `./${fsPath}`;
+        if (!fs.existsSync(buildDir)) return;
+        const partFiles = fs
+          .readdirSync(buildDir)
+          .filter(
+            (name) =>
+              name.startsWith(`${buildId}.zip.part`) ||
+              name.startsWith(`${buildId}.zip-part`) ||
+              name === `${buildId}.zip`,
+          )
+          .map((name) => `${fsPath}/${name}`);
+        if (partFiles.length === 0) {
+          logger.warn(`push-bundle: no bundle files found for '${host}${routePath}'`, { buildId });
+          return;
+        }
+        for (const partFile of partFiles) {
+          shellExec(
+            `${baseCommand} fs ${partFile} --deploy-id ${deployId} --storage-file-path ${storageFilePath} --force`,
+          );
+        }
+      };
+
+      for (const host of Object.keys(confServer)) {
+        for (const routePath of Object.keys(confServer[host])) {
+          const routeConf = confServer[host][routePath] || {};
+          if (routeConf.redirect || routeConf.disabledRebuild) continue;
+          if (routeConf.singleReplica) {
+            if (routeConf.replicas) {
+              for (const replica of routeConf.replicas) {
+                pushBundleFiles(host, replica);
+              }
+            }
+            continue;
+          }
+          pushBundleFiles(host, routePath);
+        }
+      }
     },
 
     /**
@@ -2552,17 +3004,19 @@ EOF`;
      *   so that multi-path deployments are handled correctly.
      * @param {string} path - Optional comma-separated host name(s) to restrict processing (e.g. 'underpost.net' or 'a.com,b.com').
      *   If omitted, all hosts from `engine-private/conf/<deployId>/conf.server.json` are used.
-     * @param {Object} options - The default underpost runner options for customizing workflow.
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow.
      * @param {string} [options.deployId] - Deploy ID for storage lookup (defaults to 'dd-default').
      * @param {boolean} [options.dev] - Use development environment; defaults to production.
      * @memberof UnderpostRun
      */
     'pull-bundle': (path = '', options = DEFAULT_OPTION) => {
-      const baseCommand = options.dev ? 'node bin' : 'underpost';
+      const baseCommand = 'node bin'; // options.dev ? 'node bin' : 'underpost';
       const env = options.dev ? 'development' : 'production';
       const deployId = options.deployId || 'dd-default';
       const confServerPath = `./engine-private/conf/${deployId}/conf.server.json`;
-      const confServer = fs.existsSync(confServerPath) ? loadConfServerJson(confServerPath) : {};
+      const confServer = fs.existsSync(confServerPath)
+        ? loadReplicas(deployId, loadConfServerJson(confServerPath))
+        : {};
       const hostsArg = path
         ? path
             .split(',')
@@ -2581,70 +3035,81 @@ EOF`;
         `${baseCommand} fs build --recursive --deploy-id ${deployId} --storage-file-path engine-private/conf/${deployId}/storage.bundle.json --pull --omit-unzip`,
       );
 
+      const pullBundleRoute = (host, routePath) => {
+        const buildId = `${host}-${routePath.replaceAll('/', '')}`;
+        const zipPath = `build/${buildId}.zip`;
+        const buildDir = './build';
+        const hasZip = fs.existsSync(zipPath);
+        const hasParts =
+          fs.existsSync(buildDir) &&
+          fs
+            .readdirSync(buildDir)
+            .some((name) => name.startsWith(`${buildId}.zip.part`) || name.startsWith(`${buildId}.zip-part`));
+
+        if (!hasZip && !hasParts) {
+          logger.warn(`Bundle not found for '${host}${routePath}'. Skipping.`, { zipPath, deployId });
+          return;
+        }
+
+        if (hasParts) shellExec(`${baseCommand} client --merge-zip ${zipPath}`);
+        shellExec(`${baseCommand} client --unzip ${zipPath}`);
+        shellExec(`sudo rm -rf ${zipPath}`);
+
+        if (fs.existsSync(buildDir)) {
+          fs.readdirSync(buildDir)
+            .filter((name) => name.startsWith(`${buildId}.zip.part`) || name.startsWith(`${buildId}.zip-part`))
+            .forEach((partFile) => shellExec(`sudo rm -rf ${buildDir}/${partFile}`));
+        }
+
+        const extractedDir = `build/${buildId.replace(/-$/, '')}`;
+        if (!fs.existsSync(extractedDir)) {
+          logger.warn(`Extracted build dir not found: ${extractedDir}. Skipping move for '${host}${routePath}'.`);
+          return;
+        }
+
+        const publicDestPath = routePath === '/' ? `public/${host}` : `public/${host}${routePath}`;
+        if (fs.existsSync(publicDestPath)) shellExec(`sudo rm -rf ${publicDestPath}`);
+        if (routePath !== '/') shellExec(`sudo mkdir -p public/${host}`);
+        fs.copySync(`${extractedDir}`, `${publicDestPath}`);
+      };
+
       for (const host of hostsArg) {
-        // Gather all routes for this host; fall back to root '/' when host is not in confServer
-        // (e.g. when hosts were provided explicitly via the path argument).
         const routePaths = confServer[host] ? Object.keys(confServer[host]) : ['/'];
 
         for (const routePath of routePaths) {
           const routeConf = confServer[host] ? confServer[host][routePath] || {} : {};
-          // Skip routes that are not built by buildClient (mirrors buildClient skip conditions)
-          if (routeConf.singleReplica || routeConf.redirect || routeConf.disabledRebuild) continue;
-
-          // buildClient names the zip as "<host>-<path-no-slashes>.zip"
-          // e.g. host="underpost.net", path="/" → buildId="underpost.net-", zip="build/underpost.net-.zip"
-          // e.g. host="app.net", path="/admin" → buildId="app.net-admin", zip="build/app.net-admin.zip"
-          const buildId = `${host}-${routePath.replaceAll('/', '')}`;
-          const zipPath = `build/${buildId}.zip`;
-          const buildDir = './build';
-          const hasZip = fs.existsSync(zipPath);
-          const hasParts =
-            fs.existsSync(buildDir) &&
-            fs
-              .readdirSync(buildDir)
-              .some((name) => name.startsWith(`${buildId}.zip.part`) || name.startsWith(`${buildId}.zip-part`));
-
-          if (!hasZip && !hasParts) {
-            logger.warn(`Bundle not found for '${host}${routePath}'. Skipping.`, { zipPath, deployId });
-            continue;
-          }
-
-          if (hasParts) shellExec(`${baseCommand} client --merge-zip ${zipPath}`);
-          shellExec(`${baseCommand} client --unzip ${zipPath}`);
-          shellExec(`sudo rm -rf ${zipPath}`);
-
-          // Clean up downloaded part wrapper zips left by --omit-unzip pull
-          if (fs.existsSync(buildDir)) {
-            fs.readdirSync(buildDir)
-              .filter((name) => name.startsWith(`${buildId}.zip.part`) || name.startsWith(`${buildId}.zip-part`))
-              .forEach((partFile) => shellExec(`sudo rm -rf ${buildDir}/${partFile}`));
-          }
-
-          // unzipClientBuild extracts to buildId with trailing '-' stripped
-          // e.g. "build/underpost.net-" → "build/underpost.net"
-          // e.g. "build/app.net-admin" → "build/app.net-admin" (no trailing dash, no change)
-          const extractedDir = `build/${buildId.replace(/-$/, '')}`;
-          if (!fs.existsSync(extractedDir)) {
-            logger.warn(`Extracted build dir not found: ${extractedDir}. Skipping move for '${host}${routePath}'.`);
+          if (routeConf.redirect || routeConf.disabledRebuild) continue;
+          if (routeConf.singleReplica) {
+            if (routeConf.replicas) {
+              for (const replica of routeConf.replicas) {
+                pullBundleRoute(host, replica);
+              }
+            }
             continue;
           }
-
-          // Destination mirrors the public directory layout used by the server
-          const publicDestPath = routePath === '/' ? `public/${host}` : `public/${host}${routePath}`;
-          if (fs.existsSync(publicDestPath)) shellExec(`sudo rm -rf ${publicDestPath}`);
-          // Ensure parent directory exists for sub-paths
-          if (routePath !== '/') shellExec(`sudo mkdir -p public/${host}`);
-          shellExec(`sudo mv ${extractedDir} ${publicDestPath}`);
+          pullBundleRoute(host, routePath);
         }
       }
     },
 
+    /**
+     * @method build-cluster-deployment-manifests
+     * @description Builds deployment manifests for both production and development environments using `node bin deploy --build-manifest`, syncing them, and setting replicas to 1 for the `dd` deployment.
+     * @param {string} path - Unused.
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow.
+     * @memberof UnderpostRun
+     */
+    'build-cluster-deployment-manifests': (path = '', options = DEFAULT_OPTION) => {
+      shellExec(`node bin deploy --build-manifest --sync --info-router --replicas 1 dd development`);
+      shellExec(`node bin deploy --build-manifest --sync --info-router --replicas 1 dd production --cert`);
+    },
+
     /**
      * @method monitor-ui
      * @description Installs and enables the Cockpit KVM Dashboard (cockpit, cockpit-machines, libvirt)
      * and opens the cockpit firewall service. With `--remove`, closes the firewall service instead.
      * @param {string} path - Unused.
-     * @param {Object} options - The default underpost runner options for customizing workflow.
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow.
      *   `options.remove` — when true, removes the cockpit firewall rule instead of adding it.
      * @memberof UnderpostRun
      */
@@ -2668,7 +3133,7 @@ EOF`;
      * Use `reload-shared-dir` for subsequent permission repairs without recreating the group.
      * @param {string} path - Target directory to set up (defaults to `/home/dd/engine`).
      *   Customise via the `path` argument or leave empty to use the default.
-     * @param {Object} options - The default underpost runner options for customizing workflow.
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow.
      *   Key fields: `options.user` (default `'admin'`), `options.group` (default `'engine-dev'`).
      * @memberof UnderpostRun
      */
@@ -2689,6 +3154,33 @@ EOF`;
 
       logger.info(`[setup-shared-dir] Shared directory setup complete: ${dir}`);
     },
+    /**
+     * @method shared-dir-add-user
+     * @description Add a user to an existing shared directory without changing
+     * file owners. Grants recursive group/ACL access so the user can read and
+     * write throughout the shared workspace while preserving existing ownership.
+     *
+     * @param {string} path - Shared directory (defaults to `/home/dd/engine`).
+     * @param {UnderpostRunDefaultOptions} options - Underpost runner options.
+     *   Key fields:
+     *     - options.user  (default: 'admin')
+     *     - options.group (default: 'engine-dev')
+     *
+     * @memberof UnderpostRun
+     */ 'shared-dir-add-user': (path = '/home/dd/engine', options = DEFAULT_OPTION) => {
+      const dir = path || '/home/dd/engine';
+      const user = options.user || 'admin';
+
+      logger.info(`[shared-dir-add-user] dir=${dir} user=${user}`);
+
+      // Give the user direct access without changing owners or group ownership.
+      shellExec(`sudo setfacl -R -m u:${user}:rwx ${dir}`);
+
+      // Make future files/directories inherit the same user ACL.
+      shellExec(`sudo find ${dir} -type d -exec setfacl -d -m u:${user}:rwx {} \\;`);
+
+      logger.info(`[shared-dir-add-user] User '${user}' added to shared directory: ${dir}`);
+    },
   };
 
   static API = {
@@ -2717,7 +3209,7 @@ EOF`;
      * @description Executes a specified runner function from the UnderpostRun class with the provided path and options.
      * @param {string} runner - The name of the runner to execute.
      * @param {string} path - The input value, identifier, or path for the operation.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      * @returns {Promise<any>} The result of the runner execution.
      */
@@ -2730,7 +3222,7 @@ EOF`;
      * @description Initiates the execution of a specified CLI command (runner) with the given input value (`path`) and processed options.
      * @param {string} runner - The name of the runner to execute.
      * @param {string} path - The input value, identifier, or path for the operation.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      * @returns {Promise<any>} The result of the callback execution.
      */