cyberia 2.8.885

package/src/cli/env.js

@@ -0,0 +1,101 @@
+/**
+ * Environment module for managing the environment variables of the underpost root
+ * @module src/cli/env.js
+ * @namespace UnderpostEnv
+ */
+
+import { getNpmRootPath, writeEnv } from '../server/conf.js';
+import fs from 'fs-extra';
+import { loggerFactory } from '../server/logger.js';
+import dotenv from 'dotenv';
+
+dotenv.config();
+
+const logger = loggerFactory(import.meta);
+
+/**
+ * @class UnderpostEnv
+ * @description Manages the environment variables of the underpost root.
+ * @memberof UnderpostEnv
+ */
+class UnderpostRootEnv {
+  static API = {
+    /**
+     * @method set
+     * @description Sets an environment variable in the underpost root environment.
+     * @param {string} key - The key of the environment variable to set.
+     * @param {string} value - The value of the environment variable to set.
+     * @memberof UnderpostEnv
+     */
+    set(key, value) {
+      const exeRootPath = `${getNpmRootPath()}/underpost`;
+      const envPath = `${exeRootPath}/.env`;
+      let env = {};
+      if (fs.existsSync(envPath)) env = dotenv.parse(fs.readFileSync(envPath, 'utf8'));
+      env[key] = value;
+      writeEnv(envPath, env);
+    },
+    /**
+     * @method delete
+     * @description Deletes an environment variable from the underpost root environment.
+     * @param {string} key - The key of the environment variable to delete.
+     * @memberof UnderpostEnv
+     */
+    delete(key) {
+      const exeRootPath = `${getNpmRootPath()}/underpost`;
+      const envPath = `${exeRootPath}/.env`;
+      let env = {};
+      if (fs.existsSync(envPath)) env = dotenv.parse(fs.readFileSync(envPath, 'utf8'));
+      delete env[key];
+      writeEnv(envPath, env);
+    },
+    /**
+     * @method get
+     * @description Gets an environment variable from the underpost root environment.
+     * @param {string} key - The key of the environment variable to get.
+     * @param {string} value - The value of the environment variable to get.
+     * @param {object} options - Options for getting the environment variable.
+     * @param {boolean} [options.plain=false] - If true, returns the environment variable value as a string.
+     * @memberof UnderpostEnv
+     */
+    get(key, value, options = { plain: false }) {
+      const exeRootPath = `${getNpmRootPath()}/underpost`;
+      const envPath = `${exeRootPath}/.env`;
+      if (!fs.existsSync(envPath)) {
+        logger.error(`Unable to find underpost root environment`);
+        return undefined;
+      }
+      const env = dotenv.parse(fs.readFileSync(envPath, 'utf8'));
+      options?.plain === true ? console.log(env[key]) : logger.info(`${key}(${typeof env[key]})`, env[key]);
+      return env[key];
+    },
+    /**
+     * @method list
+     * @description Lists all environment variables in the underpost root environment.
+     * @memberof UnderpostEnv
+     */
+    list() {
+      const exeRootPath = `${getNpmRootPath()}/underpost`;
+      const envPath = `${exeRootPath}/.env`;
+      if (!fs.existsSync(envPath)) {
+        logger.error(`Unable to find underpost root environment`);
+        return {};
+      }
+      const env = dotenv.parse(fs.readFileSync(envPath, 'utf8'));
+      logger.info('underpost root', env);
+      return env;
+    },
+    /**
+     * @method clean
+     * @description Cleans the underpost root environment by removing the environment file.
+     * @memberof UnderpostEnv
+     */
+    clean() {
+      const exeRootPath = `${getNpmRootPath()}/underpost`;
+      const envPath = `${exeRootPath}/.env`;
+      fs.removeSync(envPath);
+    },
+  };
+}
+
+export default UnderpostRootEnv;

package/src/cli/fs.js

@@ -0,0 +1,256 @@
+/**
+ * File storage module for managing file operations using Cloudinary.
+ * @module src/cli/fs.js
+ * @namespace UnderpostFileStorage
+ */
+
+import { v2 as cloudinary } from 'cloudinary';
+import { loggerFactory } from '../server/logger.js';
+import dotenv from 'dotenv';
+import AdmZip from 'adm-zip';
+import * as dir from 'path';
+import fs from 'fs-extra';
+import { Downloader } from '../server/downloader.js';
+import UnderpostRepository from './repository.js';
+import { shellExec } from '../server/process.js';
+dotenv.config();
+
+const logger = loggerFactory(import.meta);
+
+/**
+ * @class UnderpostFileStorage
+ * @description Manages file storage operations using Cloudinary.
+ * This class provides a set of static methods to upload, pull, and delete files
+ * from Cloudinary, as well as manage a local storage configuration file.
+ */
+class UnderpostFileStorage {
+  static API = {
+    /**
+     * @method cloudinaryConfig
+     * @description Configures the Cloudinary client with environment variables.
+     * @memberof UnderpostFileStorage
+     */
+    cloudinaryConfig() {
+      // https://console.cloudinary.com/
+      cloudinary.config({
+        cloud_name: process.env.CLOUDINARY_CLOUD_NAME,
+        api_key: process.env.CLOUDINARY_API_KEY,
+        api_secret: process.env.CLOUDINARY_API_SECRET,
+      });
+    },
+    /**
+     * @method getStorageConf
+     * @description Retrieves the storage configuration for a specific deployment.
+     * @param {object} options - An object containing deployment-specific options.
+     * @param {string} options.deployId - The identifier for the deployment.
+     * @param {string} [options.storageFilePath] - The path to the storage configuration file.
+     * @returns {object} An object containing the storage configuration and storage file path.
+     * @memberof UnderpostFileStorage
+     */
+    getStorageConf(options) {
+      let storage, storageConf;
+      if (options.deployId && typeof options.deployId === 'string') {
+        storageConf = options.storageFilePath ?? `./engine-private/conf/${options.deployId}/storage.json`;
+        if (!fs.existsSync(storageConf)) fs.writeFileSync(storageConf, JSON.stringify({}), 'utf8');
+        storage = JSON.parse(fs.readFileSync(storageConf, 'utf8'));
+      }
+      return { storage, storageConf };
+    },
+    /**
+     * @method writeStorageConf
+     * @description Writes the storage configuration to a file.
+     * @param {object} storage - The storage configuration object.
+     * @param {string} storageConf - The path to the storage configuration file.
+     * @memberof UnderpostFileStorage
+     */
+    writeStorageConf(storage, storageConf) {
+      if (storage) fs.writeFileSync(storageConf, JSON.stringify(storage, null, 4), 'utf8');
+    },
+    /**
+     * @method recursiveCallback
+     * @description Recursively processes files and directories based on the provided options.
+     * @param {string} path - The path to the directory to process.
+     * @param {object} [options] - An object containing options for the recursive callback.
+     * @param {boolean} [options.rm=false] - Flag to remove files and directories.
+     * @param {boolean} [options.recursive=false] - Flag to process directories recursively.
+     * @param {string} [options.deployId=''] - The identifier for the deployment.
+     * @param {boolean} [options.force=false] - Flag to force file operations.
+     * @param {boolean} [options.pull=false] - Flag to pull files from storage.
+     * @param {boolean} [options.git=false] - Flag to use Git for file operations.
+     * @param {string} [options.storageFilePath=''] - The path to the storage configuration file.
+     * @returns {Promise<void>} A promise that resolves when the recursive callback is complete.
+     * @memberof UnderpostFileStorage
+     */
+    async recursiveCallback(
+      path,
+      options = {
+        rm: false,
+        recursive: false,
+        deployId: '',
+        force: false,
+        pull: false,
+        git: false,
+        storageFilePath: '',
+      },
+    ) {
+      const { storage, storageConf } = UnderpostFileStorage.API.getStorageConf(options);
+      const deleteFiles = options.pull === true ? [] : UnderpostRepository.API.getDeleteFiles(path);
+      for (const relativePath of deleteFiles) {
+        const _path = path + '/' + relativePath;
+        if (_path in storage) {
+          await UnderpostFileStorage.API.delete(_path);
+          delete storage[_path];
+        }
+      }
+      if (options.pull === true) {
+        for (const _path of Object.keys(storage)) {
+          if (!fs.existsSync(_path) || options.force === true) {
+            if (options.force === true && fs.existsSync(_path)) fs.removeSync(_path);
+            await UnderpostFileStorage.API.pull(_path, options);
+          } else logger.warn(`Pull path already exists`, _path);
+        }
+        shellExec(`cd ${path} && git init && git add . && git commit -m "Base pull state"`);
+      } else {
+        const files =
+          options.git === true
+            ? UnderpostRepository.API.getChangedFiles(path)
+            : await fs.readdir(path, { recursive: true });
+        for (const relativePath of files) {
+          const _path = path + '/' + relativePath;
+          if (fs.statSync(_path).isDirectory()) {
+            if (options.pull === true && !fs.existsSync(_path)) fs.mkdirSync(_path, { recursive: true });
+            continue;
+          } else if (!(_path in storage) || options.force === true) {
+            await UnderpostFileStorage.API.upload(_path, options);
+            if (storage) storage[_path] = {};
+          } else logger.warn('File already exists', _path);
+        }
+      }
+      UnderpostFileStorage.API.writeStorageConf(storage, storageConf);
+      if (options.git === true) {
+        shellExec(`cd ${path} && git add .`);
+        shellExec(`underpost cmt ${path} feat`);
+      }
+    },
+    /**
+     * @method callback
+     * @description Orchestrates file storage operations based on the provided options.
+     * This method handles file uploads, deletions, and recursive processing of directories.
+     * @param {string} path - The path to the file or directory to process.
+     * @param {object} [options] - An object containing options for the callback.
+     * @param {boolean} [options.rm=false] - Flag to remove files and directories.
+     * @param {boolean} [options.recursive=false] - Flag to process directories recursively.
+     * @param {string} [options.deployId=''] - The identifier for the deployment.
+     * @param {boolean} [options.force=false] - Flag to force file operations.
+     * @param {boolean} [options.pull=false] - Flag to pull files from storage.
+     * @param {boolean} [options.git=false] - Flag to use Git for file operations.
+     * @returns {Promise<void>} A promise that resolves when the callback is complete.
+     * @memberof UnderpostFileStorage
+     */
+    async callback(
+      path,
+      options = { rm: false, recursive: false, deployId: '', force: false, pull: false, git: false },
+    ) {
+      if (options.recursive === true || options.git === true)
+        return await UnderpostFileStorage.API.recursiveCallback(path, options);
+      if (options.pull === true) return await UnderpostFileStorage.API.pull(path, options);
+      if (options.rm === true) return await UnderpostFileStorage.API.delete(path, options);
+      return await UnderpostFileStorage.API.upload(path, options);
+    },
+    /**
+     * @method upload
+     * @description Uploads a file to Cloudinary.
+     * @param {string} path - The path to the file to upload.
+     * @param {object} [options] - An object containing options for the upload.
+     * @param {boolean} [options.force=false] - Flag to force file operations.
+     * @param {string} [options.storageFilePath=''] - The path to the storage configuration file.
+     * @returns {Promise<object>} A promise that resolves to the upload result.
+     * @memberof UnderpostFileStorage
+     */
+    async upload(
+      path,
+      options = { rm: false, recursive: false, deployId: '', force: false, pull: false, storageFilePath: '' },
+    ) {
+      UnderpostFileStorage.API.cloudinaryConfig();
+      const { storage, storageConf } = UnderpostFileStorage.API.getStorageConf(options);
+      // path = UnderpostFileStorage.API.file2Zip(path);
+      const uploadResult = await cloudinary.uploader
+        .upload(path, {
+          public_id: path,
+          resource_type: 'raw',
+          overwrite: options.force === true ? true : false,
+        })
+        .catch((error) => {
+          logger.error(error, { path, stack: error.stack });
+        });
+      logger.info('upload result', uploadResult);
+      if (storage) storage[path] = {};
+      UnderpostFileStorage.API.writeStorageConf(storage, storageConf);
+      return uploadResult;
+    },
+    /**
+     * @method pull
+     * @description Pulls a file from Cloudinary.
+     * @param {string} path - The path to the file to pull.
+     * @returns {Promise<void>} A promise that resolves when the file is pulled.
+     * @memberof UnderpostFileStorage
+     */
+    async pull(path) {
+      UnderpostFileStorage.API.cloudinaryConfig();
+      const folder = dir.dirname(path);
+      if (!fs.existsSync(folder)) fs.mkdirSync(folder, { recursive: true });
+      const downloadResult = await cloudinary.utils.download_archive_url({
+        public_ids: [path],
+        resource_type: 'raw',
+      });
+      logger.info('download result', downloadResult);
+      await Downloader(downloadResult, path + '.zip');
+      path = UnderpostFileStorage.API.zip2File(path + '.zip');
+      fs.removeSync(path + '.zip');
+    },
+    async delete(path) {
+      UnderpostFileStorage.API.cloudinaryConfig();
+      const deleteResult = await cloudinary.api
+        .delete_resources([path], { type: 'upload', resource_type: 'raw' })
+        .catch((error) => {
+          logger.error(error, { path, stack: error.stack });
+        });
+      logger.info('delete result', deleteResult);
+      return deleteResult;
+    },
+    /**
+     * @method file2Zip
+     * @description Converts a file to a zip file.
+     * @param {string} path - The path to the file to convert.
+     * @returns {string} The path to the zip file.
+     * @memberof UnderpostFileStorage
+     */
+    file2Zip(path) {
+      const zip = new AdmZip();
+      zip.addLocalFile(path, '/');
+      path = path + '.zip';
+      zip.writeZip(path);
+      return path;
+    },
+    /**
+     * @method zip2File
+     * @description Converts a zip file to a file.
+     * @param {string} path - The path to the zip file to convert.
+     * @returns {string} The path to the file.
+     * @memberof UnderpostFileStorage
+     */
+    zip2File(path) {
+      const zip = new AdmZip(path);
+      path = path.replaceAll('.zip', '');
+      zip.extractEntryTo(
+        /*entry name*/ path.split('/').pop(),
+        /*target path*/ dir.dirname(path),
+        /*maintainEntryPath*/ false,
+        /*overwrite*/ true,
+      );
+      return path;
+    },
+  };
+}
+
+export default UnderpostFileStorage;

package/src/cli/image.js

@@ -0,0 +1,156 @@
+/**
+ * Image management module for pull, build, creation of docker images and loading them into Kubernetes clusters
+ * @module src/cli/image.js
+ * @namespace UnderpostImage
+ */
+
+import fs from 'fs-extra';
+import dotenv from 'dotenv';
+import { loggerFactory } from '../server/logger.js';
+import Underpost from '../index.js';
+import { getUnderpostRootPath } from '../server/conf.js';
+import { shellExec } from '../server/process.js';
+
+dotenv.config();
+
+const logger = loggerFactory(import.meta);
+
+/**
+ * @class UnderpostImage
+ * @description Manages Docker image operations, including pulling, building, and loading images into Kubernetes clusters.
+ * This class provides a set of static methods to handle image operations, including pulling base images,
+ * building custom images, and loading them into specified Kubernetes clusters (Kind, Kubeadm, or K3s).
+ * @memberof UnderpostImage
+ */
+class UnderpostImage {
+  static API = {
+    dockerfile: {
+      /**
+       * @method pullBaseImages
+       * @description Pulls base images and builds a 'rockylinux9-underpost' image,
+       * then loads it into the specified Kubernetes cluster type (Kind, Kubeadm, or K3s).
+       * @param {object} options - Options for pulling and loading images.
+       * @param {boolean} [options.kindLoad=false] - If true, load image into Kind cluster.
+       * @param {boolean} [options.kubeadmLoad=false] - If true, load image into Kubeadm cluster.
+       * @param {boolean} [options.k3sLoad=false] - If true, load image into K3s cluster.
+       * @param {string} [options.path=false] - Path to the Dockerfile context.
+       * @param {string} [options.version=''] - Version tag for the image.
+       * @memberof UnderpostImage
+       */
+      pullBaseImages(
+        options = {
+          kindLoad: false,
+          kubeadmLoad: false,
+          k3sLoad: false,
+          path: false,
+          version: '',
+        },
+      ) {
+        // shellExec(`sudo podman pull docker.io/library/debian:buster`);
+        shellExec(`sudo podman pull docker.io/library/rockylinux:9`);
+        const IMAGE_NAME = `rockylinux9-underpost`;
+        const IMAGE_NAME_FULL = `${IMAGE_NAME}:${options.version ?? Underpost.version}`;
+        let LOAD_TYPE = '';
+        if (options.kindLoad === true) {
+          LOAD_TYPE = `--kind-load`;
+        } else if (options.kubeadmLoad === true) {
+          LOAD_TYPE = `--kubeadm-load`;
+        } else if (options.k3sLoad === true) {
+          // Handle K3s load type
+          LOAD_TYPE = `--k3s-load`;
+        }
+
+        shellExec(
+          `underpost dockerfile-image-build --podman-save --reset --image-path=. --path ${
+            options.path ?? getUnderpostRootPath()
+          } --image-name=${IMAGE_NAME_FULL} ${LOAD_TYPE}`,
+        );
+      },
+      /**
+       * @method build
+       * @description Builds a Docker image using Podman, optionally saves it as a tar archive,
+       * and loads it into a specified Kubernetes cluster (Kind, Kubeadm, or K3s).
+       * @param {object} options - Options for building and loading images.
+       * @param {string} [options.path=''] - The path to the directory containing the Dockerfile.
+       * @param {string} [options.imageName=''] - The name and tag for the image (e.g., 'my-app:latest').
+       * @param {string} [options.imagePath=''] - Directory to save the image tar file.
+       * @param {string} [options.dockerfileName=''] - Name of the Dockerfile (defaults to 'Dockerfile').
+       * @param {boolean} [options.podmanSave=false] - If true, save the image as a tar archive using Podman.
+       * @param {boolean} [options.kindLoad=false] - If true, load the image archive into a Kind cluster.
+       * @param {boolean} [options.kubeadmLoad=false] - If true, load the image archive into a Kubeadm cluster (uses 'ctr').
+       * @param {boolean} [options.k3sLoad=false] - If true, load the image archive into a K3s cluster (uses 'k3s ctr').
+       * @param {boolean} [options.secrets=false] - If true, load secrets from the .env file for the build.
+       * @param {string} [options.secretsPath=''] - Custom path to the .env file for secrets.
+       * @param {boolean} [options.reset=false] - If true, perform a no-cache build.
+       * @memberof UnderpostImage
+       */
+      build(
+        options = {
+          path: '',
+          imageName: '',
+          imagePath: '',
+          dockerfileName: '',
+          podmanSave: false,
+          kindLoad: false,
+          kubeadmLoad: false,
+          k3sLoad: false,
+          secrets: false,
+          secretsPath: '',
+          reset: false,
+        },
+      ) {
+        const {
+          path,
+          imageName,
+          imagePath,
+          dockerfileName,
+          podmanSave,
+          secrets,
+          secretsPath,
+          kindLoad,
+          kubeadmLoad,
+          k3sLoad,
+          reset,
+        } = options;
+        const podManImg = `localhost/${imageName}`;
+        if (imagePath && typeof imagePath === 'string' && !fs.existsSync(imagePath))
+          fs.mkdirSync(imagePath, { recursive: true });
+        const tarFile = `${imagePath}/${imageName.replace(':', '_')}.tar`;
+        let secretsInput = ' ';
+        let secretDockerInput = '';
+        let cache = '';
+        if (secrets === true) {
+          const envObj = dotenv.parse(
+            fs.readFileSync(
+              secretsPath && typeof secretsPath === 'string' ? secretsPath : `${getNpmRootPath()}/underpost/.env`,
+              'utf8',
+            ),
+          );
+          for (const key of Object.keys(envObj)) {
+            secretsInput += ` && export ${key}="${envObj[key]}" `; // Example: $(cat gitlab-token.txt)
+            secretDockerInput += ` --secret id=${key},env=${key} \ `;
+          }
+        }
+        if (reset === true) cache += ' --rm --no-cache';
+        if (path && typeof path === 'string')
+          shellExec(
+            `cd ${path}${secretsInput}&& sudo podman build -f ./${
+              dockerfileName && typeof dockerfileName === 'string' ? dockerfileName : 'Dockerfile'
+            } -t ${imageName} --pull=never --cap-add=CAP_AUDIT_WRITE${cache}${secretDockerInput} --network host`,
+          );
+
+        if (podmanSave === true) shellExec(`podman save -o ${tarFile} ${podManImg}`);
+        if (kindLoad === true) shellExec(`sudo kind load image-archive ${tarFile}`);
+        if (kubeadmLoad === true) {
+          // Use 'ctr' for Kubeadm
+          shellExec(`sudo ctr -n k8s.io images import ${tarFile}`);
+        }
+        if (k3sLoad === true) {
+          // Use 'k3s ctr' for K3s
+          shellExec(`sudo k3s ctr images import ${tarFile}`);
+        }
+      },
+    },
+  };
+}
+export default UnderpostImage;