cyberia 3.1.3 → 3.2.9

package/src/cli/run.js

@@ -4,7 +4,8 @@
  * @namespace UnderpostRun
  */
 
-import { daemonProcess, getTerminalPid, shellCd, shellExec } from '../server/process.js';
+import { daemonProcess, getTerminalPid, pbcopy, shellCd, shellExec } from '../server/process.js';
+import crypto from 'crypto';
 import {
   awaitDeployMonitor,
   buildKindPorts,
@@ -17,12 +18,31 @@ import {
 import { actionInitLog, loggerFactory } from '../server/logger.js';
 
 import fs from 'fs-extra';
+import net from 'net';
 import { range, setPad, timer } from '../client/components/core/CommonJs.js';
 
 import os from 'os';
 import Underpost from '../index.js';
 import dotenv from 'dotenv';
 
+const waitForPort = (port, host = '127.0.0.1', { maxAttempts = 30, interval = 2000 } = {}) =>
+  new Promise((resolve, reject) => {
+    let attempts = 0;
+    const tryConnect = () => {
+      attempts++;
+      const socket = net.createConnection({ port, host }, () => {
+        socket.destroy();
+        resolve();
+      });
+      socket.on('error', () => {
+        socket.destroy();
+        if (attempts >= maxAttempts) return reject(new Error(`Port ${port} not ready after ${maxAttempts} attempts`));
+        setTimeout(tryConnect, interval);
+      });
+    };
+    tryConnect();
+  });
+
 const logger = loggerFactory(import.meta);
 
 /**
@@ -73,7 +93,6 @@ const logger = loggerFactory(import.meta);
  * @property {boolean} kubeadm - Whether to run in kubeadm mode.
  * @property {boolean} kind - Whether to run in kind mode.
  * @property {boolean} k3s - Whether to run in k3s mode.
- * @property {string} logType - The type of log to generate.
  * @property {string} hosts - The hosts to use.
  * @property {string} deployId - The deployment ID.
  * @property {string} instanceId - The instance ID.
@@ -91,6 +110,10 @@ const logger = loggerFactory(import.meta);
  * @property {string|Array<{ip: string, hostnames: string[]}>} hostAliases - Adds entries to the Pod /etc/hosts via Kubernetes hostAliases.
  *   As a string (CLI): semicolon-separated entries of "ip=hostname1,hostname2" (e.g., "127.0.0.1=foo.local,bar.local;10.1.2.3=foo.remote").
  *   As an array (programmatic): objects with `ip` and `hostnames` fields (e.g., [{ ip: "127.0.0.1", hostnames: ["foo.local"] }]).
+ * @property {boolean} gitClean - Whether to perform a `git clean` before running.
+ * @property {boolean} copy - Whether to copy the command to the clipboard instead of executing it.
+ * @property {boolean} skipFullBuild - Whether to skip the full client bundle build during deployment (supported by: sync, template-deploy).
+ * @property {boolean} pullBundle - Whether to pull the bundle before running. Use together with --skip-full-build to skip the local build entirely (supported by: sync, template-deploy).
  * @memberof UnderpostRun
  */
 const DEFAULT_OPTION = {
@@ -138,7 +161,6 @@ const DEFAULT_OPTION = {
   kubeadm: false,
   kind: false,
   k3s: false,
-  logType: '',
   hosts: '',
   deployId: '',
   instanceId: '',
@@ -154,6 +176,10 @@ const DEFAULT_OPTION = {
   createJobNow: false,
   fromNCommit: 0,
   hostAliases: '',
+  gitClean: false,
+  copy: false,
+  skipFullBuild: false,
+  pullBundle: false,
 };
 
 /**
@@ -246,8 +272,15 @@ class UnderpostRun {
       const ports = '6379,27017';
       shellExec(`node bin run kill '${ports}'`);
       shellExec(`node bin run dev-cluster --dev --expose --namespace ${options.namespace}`, { async: true });
-      console.log('Loading fordward services...');
-      await timer(5000);
+      logger.info('Waiting for port-forward services to be ready...');
+      try {
+        await Promise.all([waitForPort(27017), waitForPort(6379)]);
+        logger.info('Port-forward services are ready');
+      } catch (err) {
+        logger.error('Port-forward services failed to become ready', { error: err.message });
+        shellExec(`node bin run kill '${ports}'`);
+        throw err;
+      }
       shellExec(`node bin metadata --generate ${path}`);
       shellExec(`node bin db --dev --clean-fs-collection dd`);
       shellExec(`node bin run kill '${ports}'`);
@@ -373,8 +406,12 @@ class UnderpostRun {
       }
       shellExec(`${baseCommand} run pull`);
 
-      // Capture last N commit messages for propagation (default: last 1 commit)
-      const fromN = options.fromNCommit && parseInt(options.fromNCommit) > 0 ? parseInt(options.fromNCommit) : 1;
+      // Capture last N commit messages for propagation.
+      // When --from-n-commit is not set, auto-detect unpushed commit count (same as --unpush flag).
+      const fromN =
+        options.fromNCommit && parseInt(options.fromNCommit) > 0
+          ? parseInt(options.fromNCommit)
+          : Underpost.repo.getUnpushedCount('.').count;
       const message = shellExec(`node bin cmt --changelog ${fromN} --changelog-no-hash`, {
         silent: true,
         stdout: true,
@@ -404,6 +441,15 @@ class UnderpostRun {
         deployType = 'init';
       }
 
+      // If --build is set and path is a sync-engine-* target, push the pre-built client bundle
+      // to Cloudinary so the remote container can pull it instead of rebuilding from source.
+      if (options.build && deployConfId && deployConfId.startsWith('engine-')) {
+        const confName = deployConfId.replace(/^engine-/, '');
+        const pushDeployId = options.deployId || `dd-${confName}`;
+        logger.info(`[template-deploy] Running push-bundle for deployId: ${pushDeployId}`);
+        shellExec(`${baseCommand} run push-bundle --deploy-id ${pushDeployId}`);
+      }
+
       // Dispatch npmpkg CI workflow — this builds pwa-microservices-template first.
       // If deployConfId is set, npmpkg.ci.yml will dispatch the engine-<conf-id> CI
       // with sync=true after template build completes. The engine CI then dispatches
@@ -424,16 +470,50 @@ class UnderpostRun {
     },
 
     /**
-     * @method template-deploy-image
-     * @description Dispatches the Docker image CI workflow for the `engine` repository.
+     * @method template-deploy-local
+     * @description Similar to `template-deploy` but runs the workflow locally without dispatching GitHub Actions. It pulls the latest changes, pushes to GitHub, builds the template, and optionally triggers a local release with CI push.
+     * @param {string} path - The deployment path identifier (e.g., 'sync-engine-core', 'init-engine-core', or empty for build-only).
+     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @memberof UnderpostRun
+     */
+    'template-deploy-local': async (path, options = DEFAULT_OPTION) => {
+      const baseCommand = options.dev ? 'node bin' : 'underpost';
+      shellExec(`npm run security:secrets`);
+      const reportPath = './gitleaks-report.json';
+      if (fs.existsSync(reportPath) && JSON.parse(fs.readFileSync(reportPath, 'utf8')).length > 0) {
+        logger.error('Secrets detected in gitleaks-report.json, aborting template-deploy');
+        return;
+      }
+      shellExec(`${baseCommand} run pull`);
+
+      // Capture last N commit messages from the engine repo.
+      // When --from-n-commit is not set, auto-detect unpushed commit count (same as --unpush flag).
+      const fromN =
+        options.fromNCommit && parseInt(options.fromNCommit) > 0
+          ? parseInt(options.fromNCommit)
+          : Underpost.repo.getUnpushedCount('.').count;
+      const rawMessage = shellExec(`node bin cmt --changelog ${fromN} --changelog-no-hash`, {
+        silent: true,
+        stdout: true,
+      }).trim();
+      const sanitizedMessage = Underpost.repo.sanitizeChangelogMessage(rawMessage);
+
+      const { triggerCmd } = path
+        ? await Underpost.release.ci(path, sanitizedMessage, options)
+        : await Underpost.release.pwa(sanitizedMessage, options);
+      pbcopy(triggerCmd + ' && cd /home/dd/engine');
+    },
+    /**
+     * @method docker-image
+     * @description Dispatches the Docker image CI workflow (`docker-image.ci.yml`) for the `engine` repository via `workflow_dispatch`.
      * @param {string} path - The input value, identifier, or path for the operation.
      * @param {Object} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
-    'template-deploy-image': (path, options = DEFAULT_OPTION) => {
+    'docker-image': (path, options = DEFAULT_OPTION) => {
       Underpost.repo.dispatchWorkflow({
         repo: `${process.env.GITHUB_USERNAME}/engine`,
-        workflowFile: 'docker-image.ci.yml',
+        workflowFile: `docker-image${path ? `.${path}` : ''}.ci.yml`,
         ref: 'master',
         inputs: {},
       });
@@ -570,8 +650,9 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
     sync: async (path, options = DEFAULT_OPTION) => {
       // Dev usage: node bin run --dev --build sync dd-default
       const env = options.dev ? 'development' : 'production';
-      const baseCommand = options.dev ? 'node bin' : 'underpost';
+      const baseCommand = 'node bin'; // options.dev ? 'node bin' : 'underpost';
       const baseClusterCommand = options.dev ? ' --dev' : '';
+      const clusterFlag = options.k3s ? ' --k3s' : options.kind ? ' --kind' : ' --kubeadm';
       const defaultPath = [
         'dd-default',
         options.replicas,
@@ -586,13 +667,23 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
       image = image ? image : defaultPath[3];
       node = node ? node : defaultPath[4];
       shellExec(`${baseCommand} cluster --ns-use ${options.namespace}`);
+
+      if (image && !image.startsWith('localhost'))
+        Underpost.image.pullDockerHubImage({
+          dockerhubImage: image,
+          kind: options.kind || (!options.nodeName && !options.kubeadm && !options.k3s),
+          kubeadm: options.nodeName || options.kubeadm,
+          k3s: options.k3s,
+        });
+
       if (isDeployRunnerContext(path, options)) {
         if (!options.disablePrivateConfUpdate) {
           const { validVersion } = Underpost.repo.privateConfUpdate(deployId);
           if (!validVersion) throw new Error('Version mismatch');
         }
         if (options.timezone !== 'none') shellExec(`${baseCommand} run${baseClusterCommand} tz`);
-        if (options.deployIdCronJobs !== 'none') shellExec(`node bin cron --dev --setup-start --apply`);
+        if (options.deployIdCronJobs !== 'none')
+          shellExec(`node bin cron${baseClusterCommand}${clusterFlag} --setup-start --git --apply`);
       }
 
       const currentTraffic = isDeployRunnerContext(path, options)
@@ -605,20 +696,28 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
       const cmdString = options.cmd
         ? ' --cmd ' + (options.cmd.find((c) => c.match('"')) ? '"' + options.cmd + '"' : "'" + options.cmd + "'")
         : '';
+      const gitCleanFlag = options.gitClean ? ' --git-clean' : '';
+
+      const skipFullBuildFlag = options.skipFullBuild ? ' --skip-full-build' : '';
+      const pullBundleFlag = options.pullBundle ? ' --pull-bundle' : '';
 
       shellExec(
-        `${baseCommand} deploy --kubeadm --build-manifest --sync --info-router --replicas ${replicas} --node ${node}${
+        `${baseCommand} deploy${clusterFlag} --build-manifest --sync --info-router --replicas ${replicas} --node ${node}${
           image ? ` --image ${image}` : ''
         }${versions ? ` --versions ${versions}` : ''}${
           options.namespace ? ` --namespace ${options.namespace}` : ''
-        }${timeoutFlags}${cmdString} ${deployId} ${env}`,
+        }${timeoutFlags}${cmdString}${gitCleanFlag}${skipFullBuildFlag}${pullBundleFlag} ${deployId} ${env}`,
       );
 
       if (isDeployRunnerContext(path, options)) {
+        // Backup app/services repositories with repo-backup configured
         shellExec(
-          `${baseCommand} deploy --kubeadm${cmdString} --replicas ${replicas} --disable-update-proxy ${deployId} ${env} --versions ${versions}${
+          `${baseCommand} db ${deployId} ${clusterFlag}${baseClusterCommand} --repo-backup --primary-pod --git --force-clone --preserveUUID ${options.namespace ? ` --ns ${options.namespace}` : ''}`,
+        );
+        shellExec(
+          `${baseCommand} deploy${clusterFlag}${cmdString} --replicas ${replicas} --disable-update-proxy ${deployId} ${env} --versions ${versions}${
             options.namespace ? ` --namespace ${options.namespace}` : ''
-          }${timeoutFlags}`,
+          }${timeoutFlags}${gitCleanFlag}`,
         );
         if (!targetTraffic)
           targetTraffic = Underpost.deploy.getCurrentTraffic(deployId, { namespace: options.namespace });
@@ -830,6 +929,7 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
       const confInstances = JSON.parse(
         fs.readFileSync(`./engine-private/conf/${deployId}/conf.instances.json`, 'utf8'),
       );
+      let promotedTraffic = '';
       for (const instance of confInstances) {
         let {
           id: _id,
@@ -838,17 +938,23 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
           image: _image,
           fromPort: _fromPort,
           toPort: _toPort,
+          fromDebugPort: _fromDebugPort,
+          toDebugPort: _toDebugPort,
           cmd: _cmd,
           volumes: _volumes,
           metadata: _metadata,
         } = instance;
         if (id !== _id) continue;
         const _deployId = `${deployId}-${_id}`;
+        // Use debug ports in development when defined, fall back to production ports.
+        if (env === 'development' && _fromDebugPort) _fromPort = _fromDebugPort;
+        if (env === 'development' && _toDebugPort) _toPort = _toDebugPort;
         const currentTraffic = Underpost.deploy.getCurrentTraffic(_deployId, {
           hostTest: _host,
           namespace: options.namespace,
         });
         const targetTraffic = currentTraffic ? (currentTraffic === 'blue' ? 'green' : 'blue') : 'blue';
+        promotedTraffic = targetTraffic;
         let proxyYaml =
           Underpost.deploy.baseProxyYamlFactory({ host: _host, env: options.tls ? 'production' : env, options }) +
           Underpost.deploy.deploymentYamlServiceFactory({
@@ -874,6 +980,18 @@ EOF
           { disableLog: true },
         );
       }
+      // Refresh the gRPC service to ensure it points to the parent deploy's current traffic.
+      if (promotedTraffic) {
+        const parentTraffic = Underpost.deploy.getCurrentTraffic(deployId, { namespace: options.namespace }) || 'blue';
+        const grpcServicePath = Underpost.deploy.buildGrpcServiceManifest({
+          deployId,
+          env,
+          confServer: loadConfServerJson(`./engine-private/conf/${deployId}/conf.server.json`),
+          namespace: options.namespace,
+          traffic: [parentTraffic],
+        });
+        if (grpcServicePath) shellExec(`kubectl apply -f ${grpcServicePath} -n ${options.namespace}`);
+      }
     },
 
     /**
@@ -900,12 +1018,17 @@ EOF
           image: _image,
           fromPort: _fromPort,
           toPort: _toPort,
+          fromDebugPort: _fromDebugPort,
+          toDebugPort: _toDebugPort,
           cmd: _cmd,
           volumes: _volumes,
           metadata: _metadata,
         } = instance;
         if (id !== _id) continue;
         const _deployId = `${deployId}-${_id}`;
+        // Use debug ports in development when defined, fall back to production ports.
+        if (env === 'development' && _fromDebugPort) _fromPort = _fromDebugPort;
+        if (env === 'development' && _toDebugPort) _toPort = _toDebugPort;
         etcHosts.push(_host);
         if (options.expose) continue;
         // Examples images:
@@ -913,12 +1036,13 @@ EOF
         // `localhost/rockylinux9-underpost:${Underpost.version}`
         if (!_image) _image = `underpost/underpost-engine:${Underpost.version}`;
 
-        if (options.nodeName) {
-          shellExec(`sudo crictl pull ${_image}`);
-        } else {
-          shellExec(`docker pull ${_image}`);
-          shellExec(`sudo kind load docker-image ${_image}`);
-        }
+        if (_image && !_image.startsWith('localhost'))
+          Underpost.image.pullDockerHubImage({
+            dockerhubImage: _image,
+            kind: options.kind || (!options.nodeName && !options.kubeadm && !options.k3s),
+            kubeadm: options.nodeName || options.kubeadm,
+            k3s: options.k3s,
+          });
 
         const currentTraffic = Underpost.deploy.getCurrentTraffic(_deployId, {
           hostTest: _host,
@@ -927,7 +1051,7 @@ EOF
 
         const targetTraffic = currentTraffic ? (currentTraffic === 'blue' ? 'green' : 'blue') : 'blue';
         const podId = `${_deployId}-${env}-${targetTraffic}`;
-        const ignorePods = Underpost.deploy.get(podId, 'pods', options.namespace).map((p) => p.NAME);
+        const ignorePods = Underpost.kubectl.get(podId, 'pods', options.namespace).map((p) => p.NAME);
         Underpost.deploy.configMap(env, options.namespace);
         shellExec(`kubectl delete service ${podId}-service --namespace ${options.namespace} --ignore-not-found`);
         shellExec(`kubectl delete deployment ${podId} --namespace ${options.namespace} --ignore-not-found`);
@@ -939,7 +1063,30 @@ EOF
               env,
               version: targetTraffic,
               nodeName: options.nodeName,
+              clusterContext: options.k3s ? 'k3s' : options.kubeadm ? 'kubeadm' : 'kind',
+              gitClean: options.gitClean || false,
             });
+        // Regenerate the parent deploy's gRPC ClusterIP service pointing to the
+        // parent's current traffic colour and apply it before the instance pod starts so
+        // DNS is resolvable the moment the pod boots.
+        const parentTraffic = Underpost.deploy.getCurrentTraffic(deployId, { namespace: options.namespace }) || 'blue';
+        const grpcServicePath = Underpost.deploy.buildGrpcServiceManifest({
+          deployId,
+          env,
+          confServer: loadConfServerJson(`./engine-private/conf/${deployId}/conf.server.json`),
+          namespace: options.namespace,
+          traffic: [targetTraffic],
+          host: _host,
+        });
+        if (grpcServicePath) shellExec(`kubectl apply -f ${grpcServicePath} -n ${options.namespace}`);
+
+        const resolvedCmd = _cmd[env].map((c) =>
+          c.replaceAll(
+            '{{grpc-service-dns}}',
+            `${deployId}-grpc-service-${env}-${parentTraffic}.${options.namespace || 'default'}.svc.cluster.local:50051`,
+          ),
+        );
+
         let deploymentYaml = `---
 ${Underpost.deploy
   .deploymentYamlPartsFactory({
@@ -951,7 +1098,7 @@ ${Underpost.deploy
     image: _image,
     namespace: options.namespace,
     volumes: _volumes,
-    cmd: _cmd[env],
+    cmd: resolvedCmd,
   })
   .replace('{{ports}}', buildKindPorts(_fromPort, _toPort))}
 `;
@@ -969,7 +1116,6 @@ EOF
           targetTraffic,
           ignorePods,
           options.namespace,
-          options.logType,
         );
 
         if (!ready) {
@@ -989,6 +1135,153 @@ EOF
       }
     },
 
+    /**
+     * @method instance-build-manifest
+     * @description Builds a Kubernetes Deployment + Service manifest for a specific instance entry
+     * from `conf.instances.json` and writes it to a file.
+     * Traffic colour is automatically chosen as the opposite of the current live colour (blue/green),
+     * defaulting to `blue` when no deployment is running yet.
+     *
+     * If `--build` is supplied the image is built from the project Dockerfile and loaded into the
+     * cluster before the manifest is written (kind by default; `--kubeadm` / `--k3s` override).
+     *
+     * @param {string} path - Comma-separated: `deployId,instanceId[,projectPath]`.
+     *   `projectPath` is the root directory that contains the `Dockerfile` (e.g. `./cyberia-client`).
+     *   Artifacts are written to `<projectPath>/manifests/<env>/Dockerfile` and
+     *   `<projectPath>/manifests/<env>/deployment.yaml`.
+     *   In production, files are also copied to `<projectPath>/Dockerfile` and
+     *   `<projectPath>/deployment.yaml`.
+     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @memberof UnderpostRun
+     */
+    'instance-build-manifest': (path, options = DEFAULT_OPTION) => {
+      const env = options.dev ? 'development' : 'production';
+      let [deployId, id, projectPath] = path.split(',');
+      const rootPath = projectPath ? projectPath : '.';
+      const envManifestPath = `${rootPath}/manifests/deployments/${id}-${env}`;
+      const outputPath = `${envManifestPath}/deployment.yaml`;
+      const dockerfileManifestPath = `${envManifestPath}/Dockerfile`;
+
+      fs.mkdirpSync(envManifestPath);
+
+      const confInstances = JSON.parse(
+        fs.readFileSync(`./engine-private/conf/${deployId}/conf.instances.json`, 'utf8'),
+      );
+
+      const instance = confInstances.find((i) => i.id === id);
+      if (!instance) {
+        logger.error(`Instance with id '${id}' not found in conf.instances.json for deployId '${deployId}'`);
+        return;
+      }
+
+      let {
+        id: _id,
+        host: _host,
+        path: _path,
+        image: _image,
+        fromPort: _fromPort,
+        toPort: _toPort,
+        fromDebugPort: _fromDebugPort,
+        toDebugPort: _toDebugPort,
+        cmd: _cmd,
+        volumes: _volumes,
+        metadata: _metadata,
+        runtime: _runtime,
+      } = instance;
+
+      // Resolve Dockerfile source: use runtime-specific path when instance defines a runtime.
+      const dockerfileSourcePath = _runtime ? `src/runtime/${_runtime}/Dockerfile` : `${rootPath}/Dockerfile`;
+      if (fs.existsSync(dockerfileSourcePath)) {
+        fs.copyFileSync(dockerfileSourcePath, dockerfileManifestPath);
+      } else {
+        logger.warn(`[instance-build-manifest] Dockerfile not found at ${dockerfileSourcePath}`);
+      }
+
+      const _deployId = `${deployId}-${_id}`;
+      if (!_image) _image = `underpost/underpost-engine:${Underpost.version}`;
+      // Use debug ports in development when defined, fall back to production ports.
+      if (env === 'development' && _fromDebugPort) _fromPort = _fromDebugPort;
+      if (env === 'development' && _toDebugPort) _toPort = _toDebugPort;
+
+      // Build image from projectPath Dockerfile and load into cluster when --build is set.
+      if (options.build && projectPath) {
+        const isKind = !options.kubeadm && !options.k3s;
+        Underpost.image.build({
+          path: projectPath,
+          imageName: _image,
+          podmanSave: true,
+          imagePath: projectPath,
+          kind: isKind,
+          kubeadm: !!options.kubeadm,
+          k3s: !!options.k3s,
+          reset: !!options.reset,
+          dev: options.dev,
+        });
+        logger.info(`[instance-build-manifest] Image built and loaded`, {
+          image: _image,
+          cluster: isKind ? 'kind' : options.kubeadm ? 'kubeadm' : 'k3s',
+        });
+      }
+
+      // Determine target traffic: opposite of current, or 'blue' if nothing is running yet.
+      const currentTraffic = Underpost.deploy.getCurrentTraffic(_deployId, {
+        hostTest: _host,
+        namespace: options.namespace,
+      });
+      const targetTraffic = currentTraffic ? (currentTraffic === 'blue' ? 'green' : 'blue') : 'blue';
+
+      // Resolve {{grpc-service-dns}} using the parent deploy's current (or default) traffic.
+      const parentTraffic = Underpost.deploy.getCurrentTraffic(deployId, { namespace: options.namespace }) || 'blue';
+      const resolvedCmd = _cmd[env].map((c) =>
+        c.replaceAll(
+          '{{grpc-service-dns}}',
+          `${deployId}-grpc-service-${env}-${parentTraffic}.${options.namespace || 'default'}.svc.cluster.local:50051`,
+        ),
+      );
+
+      const deploymentYaml =
+        `---\n` +
+        Underpost.deploy
+          .deploymentYamlPartsFactory({
+            deployId: _deployId,
+            env,
+            suffix: targetTraffic,
+            resources: Underpost.deploy.resourcesFactory(options),
+            replicas: options.replicas,
+            image: _image,
+            namespace: options.namespace,
+            volumes: _volumes,
+            cmd: resolvedCmd,
+          })
+          .replace('{{ports}}', buildKindPorts(_fromPort, _toPort));
+
+      fs.writeFileSync(outputPath, deploymentYaml, 'utf8');
+      logger.info(`[instance-build-manifest] Manifest written to ${outputPath}`, {
+        deployId: _deployId,
+        env,
+        traffic: targetTraffic,
+        image: _image,
+      });
+
+      if (env === 'production') {
+        if (fs.existsSync(dockerfileManifestPath)) {
+          fs.copyFileSync(dockerfileManifestPath, `${rootPath}/Dockerfile`);
+        }
+        fs.copyFileSync(outputPath, `${rootPath}/deployment.yaml`);
+        logger.info('[instance-build-manifest] Production artifacts copied to project root', {
+          rootPath,
+          dockerfile: `${rootPath}/Dockerfile`,
+          deployment: `${rootPath}/deployment.yaml`,
+        });
+        const ciSrc = `./.github/workflows/docker-image.${_runtime}.ci.yml`;
+        if (fs.existsSync(ciSrc)) {
+          if (!fs.existsSync(`${rootPath}/.github/workflows`)) fs.mkdirpSync(`${rootPath}/.github/workflows`);
+          fs.copyFileSync(ciSrc, `${rootPath}/.github/workflows/docker-image.${_runtime}.ci.yml`);
+          logger.info(`[instance-build-manifest] CI workflow copied`, { src: ciSrc });
+        }
+      }
+    },
+
     /**
      * @method ls-deployments
      * @description Retrieves and logs a table of Kubernetes deployments using `Underpost.deploy.get`.
@@ -997,7 +1290,7 @@ EOF
      * @memberof UnderpostRun
      */
     'ls-deployments': async (path, options = DEFAULT_OPTION) => {
-      console.table(await Underpost.deploy.get(path, 'deployments', options.namespace));
+      console.table(await Underpost.kubectl.get(path, 'deployments', options.namespace));
     },
 
     /**
@@ -1013,6 +1306,44 @@ EOF
       shellExec(`${options.underpostRoot}/scripts/rocky-setup.sh${options.dev ? ' --install-dev' : ``}`);
     },
 
+    /**
+     * @method install-crio
+     * @description Installs and configures CRI-O as the container runtime for kubeadm clusters.
+     * Adds the stable v1.33 CRI-O yum repository, installs the `cri-o` package, configures
+     * the systemd cgroup driver, enables the `crio` service, and writes `/etc/crictl.yaml`
+     * so that `crictl` targets the CRI-O socket by default.
+     * @param {string} path - Unused.
+     * @param {Object} options - The default underpost runner options for customizing workflow.
+     * @memberof UnderpostRun
+     */
+    'install-crio': (path, options = DEFAULT_OPTION) => {
+      logger.info('Installing CRI-O...');
+      shellExec(`cat <<EOF | sudo tee /etc/yum.repos.d/cri-o.repo
+[cri-o]
+name=CRI-O
+baseurl=https://download.opensuse.org/repositories/isv:/cri-o:/stable:/v1.33/rpm/
+enabled=1
+gpgcheck=1
+gpgkey=https://download.opensuse.org/repositories/isv:/cri-o:/stable:/v1.33/rpm/repodata/repomd.xml.key
+EOF`);
+      shellExec(`sudo dnf -y install cri-o`);
+      // crictl is in the kubernetes repo but excluded by default — install it explicitly
+      shellExec(`sudo yum install -y cri-tools --disableexcludes=kubernetes`);
+      // Ensure CRI-O uses systemd cgroup driver (matches kubelet default)
+      shellExec(
+        `sudo sed -i 's/^#\?cgroup_manager =.*/cgroup_manager = "systemd"/' /etc/crio/crio.conf 2>/dev/null || true`,
+      );
+      shellExec(`sudo systemctl enable --now crio`);
+      logger.info('CRI-O installed and started.');
+      // Write crictl config so all crictl calls default to the CRI-O socket
+      shellExec(`cat <<EOF | sudo tee /etc/crictl.yaml
+runtime-endpoint: unix:///var/run/crio/crio.sock
+image-endpoint: unix:///var/run/crio/crio.sock
+timeout: 10
+debug: false
+EOF`);
+    },
+
     /**
      * @method dd-container
      * @description Deploys a development or debug container tasks jobs, setting up necessary volumes and images, and running specified commands within the container.
@@ -1091,15 +1422,13 @@ EOF
     'db-client': async (path, options = DEFAULT_OPTION) => {
       const { underpostRoot } = options;
 
-      const image = 'adminer:4.7.6-standalone';
-
-      if (!options.kubeadm && !options.k3s) {
-        // Only load if not kubeadm/k3s (Kind needs it)
-        shellExec(`docker pull ${image}`);
-        shellExec(`sudo kind load docker-image ${image}`);
-      } else if (options.kubeadm || options.k3s)
-        // For kubeadm/k3s, ensure it's available for containerd
-        shellExec(`sudo crictl pull ${image}`);
+      Underpost.image.pullDockerHubImage({
+        dockerhubImage: 'adminer',
+        version: '4.7.6-standalone',
+        kind: options.kind,
+        kubeadm: options.kubeadm,
+        k3s: options.k3s,
+      });
 
       shellExec(`kubectl delete deployment adminer -n ${options.namespace} --ignore-not-found`);
       shellExec(`kubectl apply -k ${underpostRoot}/manifests/deployment/adminer/. -n ${options.namespace}`);
@@ -1157,6 +1486,9 @@ EOF
     /**
      * @method promote
      * @description Switches traffic between blue/green deployments for a specified deployment ID(s) (uses `dd.router` for 'dd', or a specific ID).
+     * When `--tls` is set, rebuilds the proxy manifest with `--cert` so the HTTPProxy includes
+     * TLS config, deletes stale Certificate resources, then reapplies the proxy and secret.yaml
+     * (cert-manager Certificate resources) for each affected deployment.
      * @param {string} path - The input value, identifier, or path for the operation (used as a comma-separated string: `deployId,env,replicas`).
      * @param {Object} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
@@ -1165,11 +1497,34 @@ EOF
       let [inputDeployId, inputEnv, inputReplicas] = path.split(',');
       if (!inputEnv) inputEnv = 'production';
       if (!inputReplicas) inputReplicas = 1;
+      // TODO: normalize: --tls maps to --cert for deploy.js isValidTLSContext compatibility
+      if (options.tls) options.cert = true;
+
+      const applyCerts = (deployId, targetTraffic) => {
+        if (!options.tls) return;
+        // Rebuild proxy.yaml with --cert so the HTTPProxy includes TLS virtualhost config
+        shellExec(
+          `node bin deploy --build-manifest --cert --traffic ${targetTraffic} --replicas ${inputReplicas} --namespace ${options.namespace} ${deployId} ${inputEnv}`,
+        );
+        // Delete stale Certificate resources before reapplying
+        const confServerPath = `./engine-private/conf/${deployId}/conf.server.json`;
+        if (fs.existsSync(confServerPath)) {
+          for (const host of Object.keys(JSON.parse(fs.readFileSync(confServerPath, 'utf8'))))
+            shellExec(`sudo kubectl delete Certificate ${host} -n ${options.namespace} --ignore-not-found`);
+        }
+        shellExec(
+          `sudo kubectl apply -f ./engine-private/conf/${deployId}/build/${inputEnv}/proxy.yaml -n ${options.namespace}`,
+        );
+        const secretPath = `./engine-private/conf/${deployId}/build/${inputEnv}/secret.yaml`;
+        if (fs.existsSync(secretPath)) shellExec(`kubectl apply -f ${secretPath} -n ${options.namespace}`);
+      };
+
       if (inputDeployId === 'dd') {
         for (const deployId of fs.readFileSync(`./engine-private/deploy/dd.router`, 'utf8').split(',')) {
           const currentTraffic = Underpost.deploy.getCurrentTraffic(deployId, { namespace: options.namespace });
           const targetTraffic = currentTraffic === 'blue' ? 'green' : 'blue';
           Underpost.deploy.switchTraffic(deployId, inputEnv, targetTraffic, inputReplicas, options.namespace, options);
+          applyCerts(deployId, targetTraffic);
         }
       } else {
         const currentTraffic = Underpost.deploy.getCurrentTraffic(inputDeployId, { namespace: options.namespace });
@@ -1182,6 +1537,7 @@ EOF
           options.namespace,
           options,
         );
+        applyCerts(inputDeployId, targetTraffic);
       }
     },
     /**
@@ -1662,7 +2018,7 @@ EOF
         : [
             `npm install -g npm@11.2.0`,
             `npm install -g underpost`,
-            `${baseCommand} secret underpost --create-from-file /etc/config/.env.${env}`,
+            `${baseCommand} secret underpost --create-from-env`,
             `${baseCommand} start --build --run ${deployId} ${env}`,
           ];
       shellExec(`node bin run sync${baseClusterCommand} --deploy-id-cron-jobs none dd-test --cmd "${cmd}"`);
@@ -1693,7 +2049,7 @@ EOF
 
               const { close } = await (async () => {
                 const checkAwaitPath = '/await';
-                while (!Underpost.deploy.existsContainerFile({ podName, path: checkAwaitPath })) {
+                while (!Underpost.kubectl.existsFile({ podName, path: checkAwaitPath })) {
                   logger.info('monitor', checkAwaitPath);
                   await timer(1000);
                 }
@@ -1720,7 +2076,7 @@ EOF
                 logger.info('monitor', checkPath);
                 {
                   const checkAwaitPath = `/home/dd/docs${checkPath}`;
-                  while (!Underpost.deploy.existsContainerFile({ podName, path: checkAwaitPath })) {
+                  while (!Underpost.kubectl.existsFile({ podName, path: checkAwaitPath })) {
                     logger.info('waiting for', checkAwaitPath);
                     await timer(1000);
                   }
@@ -1786,7 +2142,8 @@ EOF
 
       shellCd(dir);
 
-      shellExec(`git init && git add . && git commit -m "Base implementation"`);
+      Underpost.repo.initLocalRepo({ path: dir });
+      shellExec(`git add . && git commit -m "Base implementation"`);
       shellExec(`chmod +x ./replace_params.sh`);
       shellExec(`chmod +x ./build.sh`);
 
@@ -1795,6 +2152,16 @@ EOF
 
       shellCd('/home/dd/engine');
     },
+    /**
+     * @method pull-rocky-image
+     * @description Pulls the base `rockylinux:9` image from Docker Hub via Podman.
+     * @param {string} path - The input value, identifier, or path for the operation.
+     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @memberof UnderpostRun
+     */
+    'pull-rocky-image': (path, options = DEFAULT_OPTION) => {
+      shellExec(`sudo podman pull docker.io/library/rockylinux:9`);
+    },
     /**
      * @method rmi
      * @description Forces the removal of all local Podman images (`podman rmi $(podman images -qa) --force`).
@@ -1824,6 +2191,42 @@ EOF
         } else shellExec(`sudo kill -9 $(lsof -t -i:${_path})`);
       }
     },
+    /**
+     * @method generate-pass
+     * @description Generates a cryptographically secure random password that satisfies all validatePassword
+     * constraints (lowercase, uppercase, digit, special char, min 8 chars). Logs the plain password
+     * to the console or, when `--copy` is set, copies it to the clipboard via pbcopy.
+     * @param {string} path - Optional password length (default: 16).
+     * @param {Object} options - The default underpost runner options for customizing workflow.
+     * @param {boolean} options.copy - When true, copies to clipboard instead of logging.
+     * @memberof UnderpostRun
+     */
+    'generate-pass': (path, options = DEFAULT_OPTION) => {
+      const length = path && parseInt(path) > 0 ? parseInt(path) : 16;
+      const lower = 'abcdefghijklmnopqrstuvwxyz';
+      const upper = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ';
+      const digits = '0123456789';
+      const special = '@#$%^&*()_+';
+      const all = lower + upper + digits + special;
+      const buf = crypto.randomBytes(length + 4);
+      // Guarantee at least one character from each required class
+      const chars = [
+        lower[buf[0] % lower.length],
+        upper[buf[1] % upper.length],
+        digits[buf[2] % digits.length],
+        special[buf[3] % special.length],
+      ];
+      for (let i = 4; i < length; i++) chars.push(all[buf[i] % all.length]);
+      // Fisher-Yates shuffle using an independent random buffer
+      const shuf = crypto.randomBytes(length);
+      for (let i = chars.length - 1; i > 0; i--) {
+        const j = shuf[i % shuf.length] % (i + 1);
+        [chars[i], chars[j]] = [chars[j], chars[i]];
+      }
+      const password = chars.join('');
+      if (options.copy) pbcopy(password);
+      else console.log(password);
+    },
     /**
      * @method secret
      * @description Creates an Underpost secret named 'underpost' from a file, defaulting to `/home/dd/engine/engine-private/conf/dd-cron/.env.production` if no path is provided.
@@ -1833,9 +2236,7 @@ EOF
      */
     secret: (path, options = DEFAULT_OPTION) => {
       const secretPath = path ? path : `/home/dd/engine/engine-private/conf/dd-cron/.env.production`;
-      const command = options.dev
-        ? `node bin secret underpost --create-from-file ${secretPath}`
-        : `underpost secret underpost --create-from-file ${secretPath}`;
+      const command = `${options.dev ? 'node bin' : 'underpost'} secret underpost --create-from-file ${secretPath}`;
       shellExec(command);
     },
     /**
@@ -2006,6 +2407,144 @@ EOF`;
         if (options.logs) shellExec(`kubectl logs -f ${podName} -n ${namespace}`, { async: true });
       }
     },
+
+    /**
+     * @method push-bundle
+     * @description Builds the client zip for the specified deployment, splits it into parts, and uploads to file storage.
+     *   Steps: set env, build+split zip, switch to cron env, upload parts to storage.
+     * @param {string} path - Optional `fsPath.splitOption` string.
+     *   Examples: `build` (default split 8), `build.16` (split 16 MB), `build.none-split` (no split flag).
+     * @param {Object} options - The default underpost runner options for customizing workflow.
+     * @param {string} [options.deployId] - Override deploy ID.
+     * @param {boolean} [options.dev] - Use development environment; defaults to production.
+     * @memberof UnderpostRun
+     */
+    'push-bundle': (path = '', options = DEFAULT_OPTION) => {
+      const baseCommand = options.dev ? 'node bin' : 'underpost';
+      const env = options.dev ? 'development' : 'production';
+      const deployId = options.deployId || 'dd-default';
+      const pathParts = (path || '').split('.');
+      const fsPath = (pathParts[0] || '').trim() || 'build';
+      const splitOption = (pathParts[1] || '').trim();
+
+      let splitFlag = '--split 8';
+      if (splitOption) {
+        if (splitOption === 'none-split') {
+          splitFlag = '';
+        } else {
+          const splitMb = Number(splitOption);
+          if (Number.isFinite(splitMb) && splitMb > 0) {
+            splitFlag = `--split ${splitMb}`;
+          } else {
+            logger.warn('push-bundle: invalid split option, using default split 8', {
+              path,
+              splitOption,
+            });
+          }
+        }
+      }
+
+      shellExec(`${baseCommand} env ${deployId} ${env}`);
+      shellExec(`${baseCommand} client ${deployId} --build-zip${splitFlag ? ` ${splitFlag}` : ''}`);
+      shellExec(
+        `${baseCommand} fs ${fsPath} --recursive --deploy-id ${deployId} --storage-file-path engine-private/conf/${deployId}/storage.bundle.json --force`,
+      );
+    },
+
+    /**
+     * @method pull-bundle
+     * @description Downloads split zip parts from file storage, merges and extracts them, and moves the result into the public directory.
+     *   Steps: set env, download parts (omit-unzip), merge zip, unzip, remove zip + parts, move to public/<host>[/path].
+     *   Iterates over every non-singleReplica, non-redirect, non-disabledRebuild route in conf.server.json
+     *   so that multi-path deployments are handled correctly.
+     * @param {string} path - Optional comma-separated host name(s) to restrict processing (e.g. 'underpost.net' or 'a.com,b.com').
+     *   If omitted, all hosts from `engine-private/conf/<deployId>/conf.server.json` are used.
+     * @param {Object} options - The default underpost runner options for customizing workflow.
+     * @param {string} [options.deployId] - Deploy ID for storage lookup (defaults to 'dd-default').
+     * @param {boolean} [options.dev] - Use development environment; defaults to production.
+     * @memberof UnderpostRun
+     */
+    'pull-bundle': (path = '', options = DEFAULT_OPTION) => {
+      const baseCommand = options.dev ? 'node bin' : 'underpost';
+      const env = options.dev ? 'development' : 'production';
+      const deployId = options.deployId || 'dd-default';
+      const confServerPath = `./engine-private/conf/${deployId}/conf.server.json`;
+      const confServer = fs.existsSync(confServerPath) ? loadConfServerJson(confServerPath) : {};
+      const hostsArg = path
+        ? path
+            .split(',')
+            .map((h) => h.trim())
+            .filter(Boolean)
+        : Object.keys(confServer);
+
+      if (hostsArg.length === 0) {
+        logger.error('pull-bundle: no hosts resolved', { deployId, path, confServerPath });
+        return;
+      }
+
+      shellExec(`${baseCommand} env ${deployId} ${env}`);
+      if (!fs.existsSync('./build')) fs.mkdirSync('./build', { recursive: true });
+      shellExec(
+        `${baseCommand} fs build --recursive --deploy-id ${deployId} --storage-file-path engine-private/conf/${deployId}/storage.bundle.json --pull --omit-unzip`,
+      );
+
+      for (const host of hostsArg) {
+        // Gather all routes for this host; fall back to root '/' when host is not in confServer
+        // (e.g. when hosts were provided explicitly via the path argument).
+        const routePaths = confServer[host] ? Object.keys(confServer[host]) : ['/'];
+
+        for (const routePath of routePaths) {
+          const routeConf = confServer[host] ? confServer[host][routePath] || {} : {};
+          // Skip routes that are not built by buildClient (mirrors buildClient skip conditions)
+          if (routeConf.singleReplica || routeConf.redirect || routeConf.disabledRebuild) continue;
+
+          // buildClient names the zip as "<host>-<path-no-slashes>.zip"
+          // e.g. host="underpost.net", path="/" → buildId="underpost.net-", zip="build/underpost.net-.zip"
+          // e.g. host="app.net", path="/admin" → buildId="app.net-admin", zip="build/app.net-admin.zip"
+          const buildId = `${host}-${routePath.replaceAll('/', '')}`;
+          const zipPath = `build/${buildId}.zip`;
+          const buildDir = './build';
+          const hasZip = fs.existsSync(zipPath);
+          const hasParts =
+            fs.existsSync(buildDir) &&
+            fs
+              .readdirSync(buildDir)
+              .some((name) => name.startsWith(`${buildId}.zip.part`) || name.startsWith(`${buildId}.zip-part`));
+
+          if (!hasZip && !hasParts) {
+            logger.warn(`Bundle not found for '${host}${routePath}'. Skipping.`, { zipPath, deployId });
+            continue;
+          }
+
+          if (hasParts) shellExec(`${baseCommand} client --merge-zip ${zipPath}`);
+          shellExec(`${baseCommand} client --unzip ${zipPath}`);
+          shellExec(`sudo rm -rf ${zipPath}`);
+
+          // Clean up downloaded part wrapper zips left by --omit-unzip pull
+          if (fs.existsSync(buildDir)) {
+            fs.readdirSync(buildDir)
+              .filter((name) => name.startsWith(`${buildId}.zip.part`) || name.startsWith(`${buildId}.zip-part`))
+              .forEach((partFile) => shellExec(`sudo rm -rf ${buildDir}/${partFile}`));
+          }
+
+          // unzipClientBuild extracts to buildId with trailing '-' stripped
+          // e.g. "build/underpost.net-" → "build/underpost.net"
+          // e.g. "build/app.net-admin" → "build/app.net-admin" (no trailing dash, no change)
+          const extractedDir = `build/${buildId.replace(/-$/, '')}`;
+          if (!fs.existsSync(extractedDir)) {
+            logger.warn(`Extracted build dir not found: ${extractedDir}. Skipping move for '${host}${routePath}'.`);
+            continue;
+          }
+
+          // Destination mirrors the public directory layout used by the server
+          const publicDestPath = routePath === '/' ? `public/${host}` : `public/${host}${routePath}`;
+          if (fs.existsSync(publicDestPath)) shellExec(`sudo rm -rf ${publicDestPath}`);
+          // Ensure parent directory exists for sub-paths
+          if (routePath !== '/') shellExec(`sudo mkdir -p public/${host}`);
+          shellExec(`sudo mv ${extractedDir} ${publicDestPath}`);
+        }
+      }
+    },
   };
 
   static API = {