@underpostnet/underpost 2.99.0 → 2.99.4

package/src/cli/image.js

@@ -8,7 +8,7 @@ import fs from 'fs-extra';
 import dotenv from 'dotenv';
 import { loggerFactory } from '../server/logger.js';
 import Underpost from '../index.js';
-import { getUnderpostRootPath } from '../server/conf.js';
+import { getNpmRootPath, getUnderpostRootPath } from '../server/conf.js';
 import { shellExec } from '../server/process.js';
 
 dotenv.config();
@@ -32,7 +32,7 @@ class UnderpostImage {
      * @param {boolean} [options.kind=false] - If true, load image into Kind cluster.
      * @param {boolean} [options.kubeadm=false] - If true, load image into Kubeadm cluster.
      * @param {boolean} [options.k3s=false] - If true, load image into K3s cluster.
-     * @param {string} [options.path=false] - Path to the Dockerfile context.
+     * @param {string} [options.path=''] - Path to the Dockerfile context.
      * @param {boolean} [options.dev=false] - If true, use development mode.
      * @param {string} [options.version=''] - Version tag for the image.
      * @param {string} [options.imageName=''] - Custom name for the image.
@@ -43,7 +43,7 @@ class UnderpostImage {
         kind: false,
         kubeadm: false,
         k3s: false,
-        path: false,
+        path: '',
         dev: false,
         version: '',
         imageName: '',
@@ -137,7 +137,7 @@ class UnderpostImage {
           ),
         );
         for (const key of Object.keys(envObj)) {
-          secretsInput += ` && export ${key}="${envObj[key]}" `; // Example: $(cat gitlab-token.txt)
+          secretsInput += ` && export ${key}="${envObj[key]}" `;
           secretDockerInput += ` --secret id=${key},env=${key} \ `;
         }
       }

package/src/cli/index.js

@@ -456,6 +456,9 @@ program
   .option('--retry-count <count>', 'Sets HTTPProxy per-route retry count (e.g., 3).')
   .option('--retry-per-try-timeout <duration>', 'Sets HTTPProxy retry per-try timeout (e.g., "150ms").')
   .option('--disable-private-conf-update', 'Disables updates to private configuration during execution.')
+  .option('--versions <deployment-versions>', 'Specifies the deployment versions to monitor. eg. "blue,green", "green"')
+  .option('--ready-deployment', 'Run in ready deployment monitor mode.')
+  .option('--promote', 'Promotes the deployment after monitoring.')
   .description('Manages health server monitoring for specified deployments.')
   .action(Underpost.monitor.callback);
 
@@ -554,6 +557,20 @@ program
   .option('--retry-count <count>', 'Sets HTTPProxy per-route retry count (e.g., 3).')
   .option('--retry-per-try-timeout <duration>', 'Sets HTTPProxy retry per-try timeout (e.g., "150ms").')
   .option('--disable-private-conf-update', 'Disables updates to private configuration during execution.')
+  .option('--logs', 'Streams logs during the runner execution.')
+  .option('--monitor-status <status>', 'Sets the status to monitor for pod/resource (default: "Running").')
+  .option(
+    '--monitor-status-kind-type <kind-type>',
+    'Sets the Kubernetes resource kind type to monitor (default: "pods").',
+  )
+  .option(
+    '--monitor-status-delta-ms <milliseconds>',
+    'Sets the polling interval in milliseconds for status monitoring (default: 1000).',
+  )
+  .option(
+    '--monitor-status-max-attempts <attempts>',
+    'Sets the maximum number of status check attempts (default: 600).',
+  )
   .description('Runs specified scripts using various runners.')
   .action(Underpost.run.callback);
 
@@ -593,7 +610,13 @@ program
   .action(Underpost.lxd.callback);
 
 program
-  .command('baremetal [workflow-id] [ip-address] [hostname] [ip-file-server] [ip-config] [netmask] [dns-server]')
+  .command('baremetal [workflow-id]')
+  .option('--ip-address <ip-address>', 'The IP address of the control server or the local machine.')
+  .option('--hostname <hostname>', 'The hostname of the target baremetal machine.')
+  .option('--ip-file-server <ip-file-server>', 'The IP address of the file server (NFS/TFTP).')
+  .option('--ip-config <ip-config>', 'IP configuration string for the baremetal machine.')
+  .option('--netmask <netmask>', 'Netmask of network.')
+  .option('--dns-server <dns-server>', 'DNS server IP address.')
   .option('--control-server-install', 'Installs the baremetal control server.')
   .option('--control-server-uninstall', 'Uninstalls the baremetal control server.')
   .option('--control-server-restart', 'Restarts the baremetal control server.')

package/src/cli/monitor.js

@@ -42,6 +42,9 @@ class UnderpostMonitor {
      * @param {string} [options.timeoutIdle=''] - Timeout for idle connections.
      * @param {string} [options.retryCount=''] - Number of retry attempts for health checks.
      * @param {string} [options.retryPerTryTimeout=''] - Timeout per retry attempt.
+     * @param {boolean} [options.promote=false] - Promote the deployment after monitoring.
+     * @param {boolean} [options.readyDeployment=false] - Monitor until the deployment is ready.
+     * @param {string} [options.versions=''] - Specific version of the deployment to monitor.
      * @param {object} [commanderOptions] - Options passed from the command line interface.
      * @param {object} [auxRouter] - Optional router configuration for the deployment.
      * @memberof UnderpostMonitor
@@ -61,6 +64,9 @@ class UnderpostMonitor {
         timeoutIdle: '',
         retryCount: '',
         retryPerTryTimeout: '',
+        promote: false,
+        readyDeployment: false,
+        versions: '',
       },
       commanderOptions,
       auxRouter,
@@ -79,6 +85,17 @@ class UnderpostMonitor {
         return;
       }
 
+      if (options.readyDeployment) {
+        for (const version of options.versions.split(',')) {
+          (async () => {
+            await Underpost.deploy.monitorReadyRunner(deployId, env, version, [], options.namespace, 'underpost');
+            if (options.promote)
+              Underpost.deploy.switchTraffic(deployId, env, version, options.replicas, options.namespace, options);
+          })();
+        }
+        return;
+      }
+
       const router = auxRouter ?? (await Underpost.deploy.routerFactory(deployId, env));
 
       const confServer = loadReplicas(

package/src/cli/run.js

@@ -80,6 +80,11 @@ const logger = loggerFactory(import.meta);
  * @property {string} user - The user to run as.
  * @property {string} pid - The process ID.
  * @property {boolean} disablePrivateConfUpdate - Whether to disable private configuration updates.
+ * @property {string} monitorStatus - The monitor status option.
+ * @property {string} monitorStatusKindType - The monitor status kind type option.
+ * @property {string} monitorStatusDeltaMs - The monitor status delta in milliseconds.
+ * @property {string} monitorStatusMaxAttempts - The maximum number of attempts for monitor status.
+ * @property {boolean} logs - Whether to enable logs.
  * @memberof UnderpostRun
  */
 const DEFAULT_OPTION = {
@@ -134,6 +139,11 @@ const DEFAULT_OPTION = {
   user: '',
   pid: '',
   disablePrivateConfUpdate: false,
+  monitorStatus: '',
+  monitorStatusKindType: '',
+  monitorStatusDeltaMs: '',
+  monitorStatusMaxAttempts: '',
+  logs: false,
 };
 
 /**
@@ -255,17 +265,25 @@ class UnderpostRun {
     },
 
     /**
-     * @method ssh-cluster-info
-     * @description Executes the `ssh-cluster-info.sh` script to display cluster connection information.
+     * @method ssh-deploy-info
+     * @description Retrieves deployment status and pod information from a remote server via SSH.
      * @param {string} path - The input value, identifier, or path for the operation.
      * @param {Object} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
-    'ssh-cluster-info': async (path, options = DEFAULT_OPTION) => {
-      const { underpostRoot } = options;
-      if (options.deployId && options.user) await Underpost.ssh.setDefautlSshCredentials(options);
-      shellExec(`chmod +x ${underpostRoot}/scripts/ssh-cluster-info.sh`);
-      shellExec(`${underpostRoot}/scripts/ssh-cluster-info.sh`);
+    'ssh-deploy-info': async (path = '', options = DEFAULT_OPTION) => {
+      const env = options.dev ? 'development' : 'production';
+      await Underpost.ssh.sshRemoteRunner(
+        `node bin deploy ${path ? path : 'dd'} ${env} --status && kubectl get pods -A`,
+        {
+          deployId: options.deployId,
+          user: options.user,
+          dev: options.dev,
+          remote: true,
+          useSudo: true,
+          cd: '/home/dd/engine',
+        },
+      );
     },
 
     /**
@@ -494,19 +512,19 @@ class UnderpostRun {
       if (targetTraffic) versions = versions ? versions : targetTraffic;
 
       const timeoutFlags = Underpost.deploy.timeoutFlagsFactory(options);
+      const cmdString = options.cmd
+        ? ' --cmd ' + (options.cmd.find((c) => c.match('"')) ? '"' + options.cmd + '"' : "'" + options.cmd + "'")
+        : '';
 
       shellExec(
         `${baseCommand} deploy --kubeadm --build-manifest --sync --info-router --replicas ${replicas} --node ${node}${
           image ? ` --image ${image}` : ''
         }${versions ? ` --versions ${versions}` : ''}${
           options.namespace ? ` --namespace ${options.namespace}` : ''
-        }${timeoutFlags} dd ${env}`,
+        }${timeoutFlags}${cmdString} dd ${env}`,
       );
 
       if (isDeployRunnerContext(path, options)) {
-        const cmdString = options.cmd
-          ? ` --cmd ${options.cmd.find((c) => c.match('"')) ? `"${options.cmd}"` : `'${options.cmd}'`}`
-          : '';
         shellExec(
           `${baseCommand} deploy --kubeadm${cmdString} --replicas ${replicas} --disable-update-proxy ${deployId} ${env} --versions ${versions}${
             options.namespace ? ` --namespace ${options.namespace}` : ''
@@ -552,29 +570,121 @@ class UnderpostRun {
      * @memberof UnderpostRun
      */
     'ssh-deploy-stop': async (path, options = DEFAULT_OPTION) => {
-      const env = options.dev ? 'development' : 'production';
       const baseCommand = options.dev ? 'node bin' : 'underpost';
       const baseClusterCommand = options.dev ? ' --dev' : '';
-      await Underpost.ssh.setDefautlSshCredentials(options);
-      shellExec(`#!/usr/bin/env bash
-set -euo pipefail
 
-REMOTE_USER=$(node bin config get --plain DEFAULT_SSH_USER)
-REMOTE_HOST=$(node bin config get --plain DEFAULT_SSH_HOST)
-REMOTE_PORT=$(node bin config get --plain DEFAULT_SSH_PORT)
-SSH_KEY=$(node bin config get --plain DEFAULT_SSH_KEY_PATH)
-
-chmod 600 "$SSH_KEY"
-
-ssh -i "$SSH_KEY" -o BatchMode=yes "$REMOTE_USER@$REMOTE_HOST" -p $REMOTE_PORT sh <<EOF
-cd /home/dd/engine
-sudo -n -- /bin/bash -lc "${[
+      const remoteCommand = [
         `${baseCommand} run${baseClusterCommand} stop${path ? ` ${path}` : ''}`,
         ` --deploy-id ${options.deployId}${options.instanceId ? ` --instance-id ${options.instanceId}` : ''}`,
         ` --namespace ${options.namespace}${options.hosts ? ` --hosts ${options.hosts}` : ''}`,
-      ].join('')}"
-EOF
-`);
+      ].join('');
+
+      await Underpost.ssh.sshRemoteRunner(remoteCommand, {
+        deployId: options.deployId,
+        user: options.user,
+        dev: options.dev,
+        remote: true,
+        useSudo: true,
+        cd: '/home/dd/engine',
+      });
+    },
+
+    /**
+     * @method ssh-deploy-db-rollback
+     * @description Performs a database rollback on remote deployment via SSH.
+     * @param {string} path - Comma-separated deployId and optional number of commits to reset (format: "deployId,nCommits")
+     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {string} options.deployId - The deployment identifier
+     * @param {string} options.user - The SSH user for credential lookup
+     * @param {boolean} options.dev - Development mode flag
+     * @memberof UnderpostRun
+     */
+    'ssh-deploy-db-rollback': async (path = '', options = DEFAULT_OPTION) => {
+      const baseCommand = options.dev ? 'node bin' : 'underpost';
+      let [deployId, nCommitsReset] = path.split(',');
+      if (!nCommitsReset) nCommitsReset = 1;
+
+      const remoteCommand = `${baseCommand} db ${deployId} --git --kubeadm --primary-pod --force-clone --macro-rollback-export ${nCommitsReset}${options.namespace ? ` --ns ${options.namespace}` : ''}`;
+
+      await Underpost.ssh.sshRemoteRunner(remoteCommand, {
+        deployId: options.deployId,
+        user: options.user,
+        dev: options.dev,
+        remote: true,
+        useSudo: true,
+        cd: '/home/dd/engine',
+      });
+    },
+
+    /**
+     * @method ssh-deploy-db
+     * @description Imports/restores a database on remote deployment via SSH.
+     * @param {string} path - The deployment ID for database import
+     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {string} options.deployId - The deployment identifier
+     * @param {string} options.user - The SSH user for credential lookup
+     * @param {boolean} options.dev - Development mode flag
+     * @memberof UnderpostRun
+     */
+    'ssh-deploy-db': async (path, options = DEFAULT_OPTION) => {
+      const baseCommand = options.dev ? 'node bin' : 'underpost';
+
+      const remoteCommand = `${baseCommand} db ${path} --import --drop --preserveUUID --git --kubeadm --primary-pod --force-clone${options.namespace ? ` --ns ${options.namespace}` : ''}`;
+
+      await Underpost.ssh.sshRemoteRunner(remoteCommand, {
+        deployId: options.deployId,
+        user: options.user,
+        dev: options.dev,
+        remote: true,
+        useSudo: true,
+        cd: '/home/dd/engine',
+      });
+    },
+
+    /**
+     * @method ssh-deploy-db-status
+     * @description Retrieves database status/stats for a deployment (or all deployments from dd.router) via SSH.
+     * @param {string} path - Comma-separated deployId(s) or 'dd' to use the dd.router list.
+     * @param {Object} options - Runner options (uses options.deployId for SSH host lookup).
+     * @param {string} options.deployId - Deployment identifier used for SSH config lookup.
+     * @param {string} options.user - SSH user for credential lookup.
+     * @param {boolean} options.dev - Development mode flag.
+     * @param {string} [options.namespace] - Kubernetes namespace to pass to the db check.
+     * @memberof UnderpostRun
+     */
+    'ssh-deploy-db-status': async (path = '', options = DEFAULT_OPTION) => {
+      const baseCommand = options.dev ? 'node bin' : 'underpost';
+
+      let deployList = [];
+      if (!path || path === 'dd') {
+        if (!fs.existsSync('./engine-private/deploy/dd.router')) {
+          logger.warn('dd.router not found; nothing to run');
+          return;
+        }
+        deployList = fs
+          .readFileSync('./engine-private/deploy/dd.router', 'utf8')
+          .split(',')
+          .map((d) => d.trim())
+          .filter(Boolean);
+      } else {
+        deployList = path
+          .split(',')
+          .map((d) => d.trim())
+          .filter(Boolean);
+      }
+
+      for (const deployId of deployList) {
+        const remoteCommand = `${baseCommand} db ${deployId} --stats --kubeadm --primary-pod${options.namespace ? ` --ns ${options.namespace}` : ''}`;
+
+        await Underpost.ssh.sshRemoteRunner(remoteCommand, {
+          deployId: options.deployId,
+          user: options.user,
+          dev: options.dev,
+          remote: true,
+          useSudo: true,
+          cd: '/home/dd/engine',
+        });
+      }
     },
 
     /**
@@ -897,86 +1007,6 @@ EOF
       shellExec(`${underpostRoot}/scripts/ip-info.sh ${path}`);
     },
 
-    /**
-     * @method monitor
-     * @description Monitors a specific pod (identified by `path`) for the existence of a file (`/await`), and performs conditional actions (like file copying and opening Firefox) when the file is removed.
-     * @param {string} path - The input value, identifier, or path for the operation (used as the name of the pod to monitor).
-     * @param {Object} options - The default underpost runner options for customizing workflow
-     * @memberof UnderpostRun
-     */
-    monitor: (path, options = DEFAULT_OPTION) => {
-      const pid = getTerminalPid();
-      logger.info('monitor pid', pid);
-      const checkPath = '/await';
-      const _monitor = async () => {
-        const result = Underpost.deploy.existsContainerFile({ podName: path, path: checkPath });
-        logger.info('monitor', result);
-        if (result === true) {
-          switch (path) {
-            case 'tf-vae-test':
-              {
-                const nameSpace = options.namespace;
-                const podName = path;
-                const basePath = '/home/dd';
-                const scriptPath = '/site/en/tutorials/generative/cvae.py';
-                // shellExec(
-                //   `sudo kubectl cp ${nameSpace}/${podName}:${basePath}/docs${scriptPath} ${basePath}/lab/src/${scriptPath
-                //     .split('/')
-                //     .pop()}`,
-                // );
-                // const file = fs.readFileSync(`${basePath}/lab/src/${scriptPath.split('/').pop()}`, 'utf8');
-                //                 fs.writeFileSync(
-                //                   `${basePath}/lab/src/${scriptPath.split('/').pop()}`,
-                //                   file.replace(
-                //                     `import time`,
-                //                     `import time
-                // print('=== SCRIPT UPDATE TEST ===')`,
-                //                   ),
-                //                   'utf8',
-                //                 );
-                shellExec(
-                  `sudo kubectl cp ${basePath}/lab/src/${scriptPath
-                    .split('/')
-                    .pop()} ${nameSpace}/${podName}:${basePath}/docs${scriptPath}`,
-                );
-                // shellExec(`sudo kubectl exec -i ${podName} -- sh -c "ipython ${basePath}/docs${scriptPath}"`);
-                shellExec(`sudo kubectl exec -i ${podName} -- sh -c "rm -rf ${checkPath}"`);
-
-                {
-                  const checkPath = `/latent_space_plot.png`;
-                  const outsPaths = [];
-                  logger.info('monitor', checkPath);
-                  while (!Underpost.deploy.existsContainerFile({ podName, path: `/home/dd/docs${checkPath}` }))
-                    await timer(1000);
-
-                  {
-                    const toPath = `${basePath}/lab${checkPath}`;
-                    outsPaths.push(toPath);
-                    shellExec(`sudo kubectl cp ${nameSpace}/${podName}:${basePath}/docs${checkPath} ${toPath}`);
-                  }
-
-                  for (let i of range(1, 10)) {
-                    i = `/image_at_epoch_${setPad(i, '0', 4)}.png`;
-                    const toPath = `${basePath}/lab/${i}`;
-                    outsPaths.push(toPath);
-                    shellExec(`sudo kubectl cp ${nameSpace}/${podName}:${basePath}/docs${i} ${toPath}`);
-                  }
-                  openTerminal(`firefox ${outsPaths.join(' ')}`, { single: true });
-                }
-                shellExec(`sudo kill -9 ${pid}`);
-              }
-              break;
-
-            default:
-              break;
-          }
-          return;
-        }
-        await timer(1000);
-        _monitor();
-      };
-      _monitor();
-    },
     /**
      * @method db-client
      * @description Deploys and exposes the Adminer database client application (using `adminer:4.7.6-standalone` image) on the cluster.
@@ -1034,15 +1064,20 @@ EOF
           `git config user.email '${email}' && ` +
           `git config credential.interactive always &&` +
           `git config pull.rebase false`,
+        {
+          disableLog: true,
+          silent: true,
+        },
       );
 
-      console.log(
-        shellExec(`git config list`, { silent: true, stdout: true })
-          .replaceAll('user.email', 'user.email'.yellow)
-          .replaceAll(username, username.green)
-          .replaceAll('user.name', 'user.name'.yellow)
-          .replaceAll(email, email.green),
-      );
+      if (options.logs)
+        console.log(
+          shellExec(`git config list`, { silent: true, stdout: true })
+            .replaceAll('user.email', 'user.email'.yellow)
+            .replaceAll(username, username.green)
+            .replaceAll('user.name', 'user.name'.yellow)
+            .replaceAll(email, email.green),
+        );
     },
 
     /**
@@ -1134,7 +1169,9 @@ EOF
       }
       await timer(5000);
       for (const deployId of deployList) {
-        shellExec(`${baseCommand} db ${deployId} --import --git --drop --preserveUUID --primary-pod`);
+        shellExec(
+          `${baseCommand} db ${deployId} --import --git --drop --preserveUUID --primary-pod${options.namespace ? ` --ns ${options.namespace}` : ''}`,
+        );
       }
       await timer(5000);
       shellExec(`${baseCommand} cluster${baseClusterCommand} --${clusterType} --pull-image --valkey`);
@@ -1148,7 +1185,7 @@ EOF
         shellExec(
           `${baseCommand} deploy ${deployId} ${env} --${clusterType}${env === 'production' ? ' --cert' : ''}${
             env === 'development' ? ' --etc-hosts' : ''
-          }`,
+          }${options.namespace ? ` --namespace ${options.namespace}` : ''}`,
         );
       }
     },
@@ -1165,7 +1202,7 @@ EOF
       if (!validVersion) throw new Error('Version mismatch');
       const currentTraffic = Underpost.deploy.getCurrentTraffic(deployId, { namespace: options.namespace });
       const targetTraffic = currentTraffic === 'blue' ? 'green' : 'blue';
-      const env = 'production';
+      const env = options.dev ? 'development' : 'production';
       const ignorePods = Underpost.deploy
         .get(`${deployId}-${env}-${targetTraffic}`, 'pods', options.namespace)
         .map((p) => p.NAME);
@@ -1268,7 +1305,11 @@ EOF
             });
       }
       await awaitDeployMonitor(true);
-      shellExec(`npm run dev-proxy ${deployId} ${subConf} ${host} ${_path}${options.tls ? ' tls' : ''}`);
+      shellExec(
+        `./node_modules/.bin/env-cmd -f .env.development node src/proxy proxy ${deployId} ${subConf} ${host} ${_path}${
+          options.tls ? ' tls' : ''
+        }`,
+      );
     },
 
     /**
@@ -1538,15 +1579,74 @@ EOF
      * @memberof UnderpostRun
      */
     'tf-vae-test': async (path, options = DEFAULT_OPTION) => {
-      const { underpostRoot } = options;
       const podName = 'tf-vae-test';
       await Underpost.run.CALL('deploy-job', '', {
+        logs: options.logs,
         podName,
         // volumeMountPath: '/custom_images',
         // volumeHostPath: '/home/dd/engine/src/client/public/cyberia/assets/skin',
         on: {
           init: async () => {
-            openTerminal(`node bin run --dev monitor ${podName}`);
+            // const pid = getTerminalPid();
+            // shellExec(`sudo kill -9 ${pid}`);
+            (async () => {
+              const nameSpace = options.namespace;
+              const basePath = '/home/dd';
+              const scriptPath = '/site/en/tutorials/generative/cvae.py';
+
+              const { close } = await (async () => {
+                const checkAwaitPath = '/await';
+                while (!Underpost.deploy.existsContainerFile({ podName, path: checkAwaitPath })) {
+                  logger.info('monitor', checkAwaitPath);
+                  await timer(1000);
+                }
+
+                return {
+                  close: () => shellExec(`sudo kubectl exec -i ${podName} -- sh -c "rm -rf ${checkAwaitPath}"`),
+                };
+              })();
+
+              const localScriptPath = `${basePath}/lab/src/${scriptPath.split('/').pop()}`;
+              if (!fs.existsSync(localScriptPath)) {
+                throw new Error(`Local override script not found: ${localScriptPath}`);
+              }
+
+              shellExec(`sudo kubectl cp ${localScriptPath} ${nameSpace}/${podName}:${basePath}/docs${scriptPath}`);
+
+              close();
+
+              {
+                const checkPath = `/latent_space_plot.png`;
+                const outsPaths = [];
+                const labDir = `${basePath}/lab`;
+
+                logger.info('monitor', checkPath);
+                {
+                  const checkAwaitPath = `/home/dd/docs${checkPath}`;
+                  while (!Underpost.deploy.existsContainerFile({ podName, path: checkAwaitPath })) {
+                    logger.info('waiting for', checkAwaitPath);
+                    await timer(1000);
+                  }
+                }
+
+                {
+                  const toPath = `${labDir}${checkPath}`;
+                  outsPaths.push(toPath);
+                  shellExec(`sudo kubectl cp ${nameSpace}/${podName}:${basePath}/docs${checkPath} ${toPath}`);
+                }
+
+                for (let i of range(1, 10)) {
+                  const fileName = `image_at_epoch_${setPad(i, '0', 4)}.png`;
+                  const fromPath = `/${fileName}`;
+                  const toPath = `${labDir}/${fileName}`;
+                  outsPaths.push(toPath);
+                  shellExec(`sudo kubectl cp ${nameSpace}/${podName}:${basePath}/docs${fromPath} ${toPath}`);
+                }
+
+                openTerminal(`firefox ${outsPaths.join(' ')}`, { single: true });
+                process.exit(0);
+              }
+            })();
           },
         },
         args: [
@@ -1771,10 +1871,16 @@ EOF`;
       shellExec(`kubectl delete pod ${podName} -n ${namespace} --ignore-not-found`);
       console.log(cmd);
       shellExec(cmd, { disableLog: true });
-      const successInstance = await Underpost.test.statusMonitor(podName);
+      const successInstance = await Underpost.test.statusMonitor(
+        podName,
+        options.monitorStatus || 'Running',
+        options.monitorStatusKindType || 'pods',
+        options.monitorStatusDeltaMs || 1000,
+        options.monitorStatusMaxAttempts || 600,
+      );
       if (successInstance) {
         options.on?.init ? await options.on.init() : null;
-        shellExec(`kubectl logs -f ${podName} -n ${namespace}`);
+        if (options.logs) shellExec(`kubectl logs -f ${podName} -n ${namespace}`, { async: true });
       }
     },
   };

package/src/cli/ssh.js

@@ -496,6 +496,55 @@ EOF`);
       if (options.status) shellExec('service sshd status');
     },
 
+    /**
+     * Generic SSH remote command runner that centralizes SSH execution logic.
+     * Executes arbitrary shell commands on a remote server via SSH with proper credential handling.
+     * @async
+     * @function sshRemoteRunner
+     * @param {string} remoteCommand - The command to execute on the remote server
+     * @param {Object} options - Configuration options for SSH execution
+     * @param {string} [options.deployId] - Deployment ID for credential lookup
+     * @param {string} [options.user] - SSH user for credential lookup
+     * @param {boolean} [options.dev=false] - Development mode flag
+     * @param {string} [options.cd='/home/dd/engine'] - Working directory on remote server
+     * @param {boolean} [options.useSudo=true] - Whether to use sudo for command execution
+     * @param {boolean} [options.remote=true] - Whether to execute as remote command (if false, runs locally)
+     * @returns {Promise<string>} Output from the shell execution
+     * @memberof UnderpostSSH
+     */
+    sshRemoteRunner: async (remoteCommand, options = {}) => {
+      const { deployId = '', user = '', dev = false, cd = '/home/dd/engine', useSudo = true, remote = true } = options;
+
+      // If not executing remotely, just run locally
+      if (!remote) {
+        return shellExec(remoteCommand);
+      }
+
+      // Set up SSH credentials from config
+      if (deployId && user) {
+        await Underpost.ssh.setDefautlSshCredentials({ deployId, user });
+      }
+
+      // Build the complete SSH command
+      const sshScript = `#!/usr/bin/env bash
+set -euo pipefail
+
+REMOTE_USER=$(node bin config get --plain DEFAULT_SSH_USER)
+REMOTE_HOST=$(node bin config get --plain DEFAULT_SSH_HOST)
+REMOTE_PORT=$(node bin config get --plain DEFAULT_SSH_PORT)
+SSH_KEY=$(node bin config get --plain DEFAULT_SSH_KEY_PATH)
+
+chmod 600 "$SSH_KEY"
+
+ssh -i "$SSH_KEY" -o BatchMode=yes "$REMOTE_USER@$REMOTE_HOST" -p $REMOTE_PORT sh <<EOF
+${cd ? `cd ${cd}` : ''}
+${useSudo ? `sudo -n -- /bin/bash -lc "${remoteCommand}"` : remoteCommand}
+EOF
+`;
+
+      return shellExec(sshScript, { stdout: true });
+    },
+
     /**
      * Loads saved SSH credentials from config and sets them in the UnderpostRootEnv API.
      * @async

package/src/cli/test.js

@@ -59,7 +59,19 @@ class UnderpostTest {
      * @param {number} options.maxAttempts - The maximum number of attempts.
      * @memberof UnderpostTest
      */
-    async callback(deployList = '', options = { itc: false, sh: false, logs: false }) {
+    async callback(
+      deployList = '',
+      options = {
+        itc: false,
+        sh: false,
+        logs: false,
+        podName: '',
+        podStatus: '',
+        kindType: '',
+        deltaMs: 1000,
+        maxAttempts: 60 * 5,
+      },
+    ) {
       if (
         options.podName &&
         typeof options.podName === 'string' &&