@underpostnet/cyberia 3.2.80

package/src/cli/env.js

@@ -0,0 +1,177 @@
+/**
+ * Environment module for managing the environment variables of the underpost root
+ * @module src/cli/env.js
+ * @namespace UnderpostEnv
+ */
+
+import { getNpmRootPath, writeEnv } from '../server/conf.js';
+import fs from 'fs-extra';
+import { loggerFactory } from '../server/logger.js';
+import dotenv from 'dotenv';
+import { pbcopy } from '../server/process.js';
+
+const logger = loggerFactory(import.meta);
+
+/**
+ * Guards an env file path against stale directory artifacts.
+ * Removes the path if it exists as a directory (e.g. `.env/` created by a previous EISDIR bug).
+ * @param {string} envPath - The path to the environment file.
+ * @memberof UnderpostEnv
+ */
+const guardEnvPath = (envPath) => {
+  if (fs.existsSync(envPath) && !fs.statSync(envPath).isFile()) {
+    logger.warn(`Removing stale directory at env path: ${envPath}`);
+    fs.removeSync(envPath);
+  }
+};
+
+/**
+ * @class UnderpostRootEnv
+ * @description Manages the environment variables of the underpost root.
+ * @memberof UnderpostEnv
+ */
+class UnderpostRootEnv {
+  static API = {
+    /**
+     * @method set
+     * @description Sets an environment variable in the underpost root environment.
+     * @param {string} key - The key of the environment variable to set.
+     * @param {string} value - The value of the environment variable to set.
+     * @param {object} options - Options for setting the environment variable.
+     * @param {string} [options.deployId=''] - Deployment ID associated with the environment variable.
+     * @param {boolean} [options.build=false] - If true, sets the environment variable using custom --deploy-id or all dd.router deploy ids configured.
+     * @memberof UnderpostEnv
+     */
+    set(key, value, options = { deployId: '', build: false }) {
+      const _set = (envPath, key, value) => {
+        guardEnvPath(envPath);
+        let env = {};
+        if (fs.existsSync(envPath)) env = dotenv.parse(fs.readFileSync(envPath, 'utf8'));
+        env[key] = value;
+        writeEnv(envPath, env);
+      };
+      if (options.build) {
+        const deployIdList = options.deployId
+          ? [options.deployId]
+          : fs.existsSync(`./engine-private/deploy/dd.router`)
+            ? fs.readFileSync(`./engine-private/deploy/dd.router`, 'utf8').split(',')
+            : [DEFAULT_DEPLOY_ID];
+        for (const deployId of deployIdList)
+          for (const envFile of ['test', 'development', 'production'])
+            _set(`./engine-private/conf/${deployId}/.env.${envFile}`, key, value);
+        return;
+      }
+      const exeRootPath = `${getNpmRootPath()}/underpost`;
+      fs.ensureDirSync(exeRootPath);
+      const envPath = `${exeRootPath}/.env`;
+      _set(envPath, key, value);
+    },
+    /**
+     * @method delete
+     * @description Deletes an environment variable from the underpost root environment.
+     * @param {string} key - The key of the environment variable to delete.
+     * @memberof UnderpostEnv
+     */
+    delete(key) {
+      const exeRootPath = `${getNpmRootPath()}/underpost`;
+      const envPath = `${exeRootPath}/.env`;
+      guardEnvPath(envPath);
+      let env = {};
+      if (fs.existsSync(envPath)) env = dotenv.parse(fs.readFileSync(envPath, 'utf8'));
+      delete env[key];
+      writeEnv(envPath, env);
+    },
+    /**
+     * @method get
+     * @description Gets an environment variable from the underpost root environment.
+     * @param {string} key - The key of the environment variable to get.
+     * @param {string} value - The value of the environment variable to get.
+     * @param {object} options - Options for getting the environment variable.
+     * @param {boolean} [options.plain=false] - If true, returns the environment variable value as a string.
+     * @param {boolean} [options.disableLog=false] - If true, disables logging of the environment variable value.
+     * @param {boolean} [options.copy=false] - If true, copies the environment variable value to the clipboard.
+     * @memberof UnderpostEnv
+     */
+    get(key, value, options = { plain: false, disableLog: false, copy: false }) {
+      const exeRootPath = `${getNpmRootPath()}/underpost`;
+      const envPath = `${exeRootPath}/.env`;
+      if (!fs.existsSync(envPath) || !fs.statSync(envPath).isFile()) return undefined;
+      const env = dotenv.parse(fs.readFileSync(envPath, 'utf8'));
+      if (!options.disableLog)
+        options?.plain === true ? console.log(env[key]) : logger.info(`${key}(${typeof env[key]})`, env[key]);
+      if (options.copy === true) pbcopy(env[key]);
+      return env[key];
+    },
+    /**
+     * @method list
+     * @description Lists all environment variables in the underpost root environment.
+     * @param {string} key - Not used for list operation.
+     * @param {string} value - Not used for list operation.
+     * @param {object} options - Options for listing environment variables.
+     * @param {string} [options.filter] - Filter keyword to match against keys or values.
+     * @memberof UnderpostEnv
+     */
+    list(key, value, options = {}) {
+      const exeRootPath = `${getNpmRootPath()}/underpost`;
+      const envPath = `${exeRootPath}/.env`;
+      guardEnvPath(envPath);
+      if (!fs.existsSync(envPath)) {
+        logger.warn(`Empty environment variables`);
+        return {};
+      }
+      let env = dotenv.parse(fs.readFileSync(envPath, 'utf8'));
+
+      // Apply filter if provided
+      if (options.filter) {
+        const filterKeyword = options.filter.toLowerCase();
+        const filtered = {};
+        for (const [envKey, envValue] of Object.entries(env)) {
+          const keyMatch = envKey.toLowerCase().includes(filterKeyword);
+          const valueMatch = String(envValue).toLowerCase().includes(filterKeyword);
+          if (keyMatch || valueMatch) {
+            filtered[envKey] = envValue;
+          }
+        }
+        env = filtered;
+        logger.info(`underpost root (filtered by: ${options.filter})`, env);
+      } else {
+        logger.info('underpost root', env);
+      }
+
+      return env;
+    },
+    /**
+     * @method clean
+     * @description Cleans the underpost root environment by removing the environment file.
+     * @param {object} options - Options for cleaning the environment.
+     * @param {Array<string>} [options.keepKeys=[]] - List of keys to keep in the environment file. If provided, only these keys will be retained.
+     * @memberof UnderpostEnv
+     */
+    clean(options = { keepKeys: [] }) {
+      const { keepKeys } = options;
+      const exeRootPath = `${getNpmRootPath()}/underpost`;
+      const envPath = `${exeRootPath}/.env`;
+      if (keepKeys && keepKeys.length > 0 && fs.existsSync(envPath)) {
+        const env = dotenv.parse(fs.readFileSync(envPath, 'utf8'));
+        const filteredEnv = Object.fromEntries(Object.entries(env).filter(([key]) => keepKeys.includes(key)));
+        writeEnv(envPath, filteredEnv);
+      } else {
+        fs.removeSync(envPath);
+      }
+    },
+    /**
+     * @method isInsideContainer
+     * @description Detects whether the current process is running inside a container.
+     * Checks for Kubernetes service injection or Docker's .dockerenv marker.
+     * @returns {boolean} True if running inside a container.
+     * @memberof UnderpostEnv
+     */
+    isInsideContainer() {
+      return !!process.env.KUBERNETES_SERVICE_HOST || fs.existsSync('/.dockerenv');
+    },
+  };
+}
+
+export default UnderpostRootEnv;
+
+export { guardEnvPath };

package/src/cli/fs.js

@@ -0,0 +1,380 @@
+/**
+ * File storage module for managing file operations using Cloudinary.
+ * @module src/cli/fs.js
+ * @namespace UnderpostFileStorage
+ */
+
+import { v2 as cloudinary } from 'cloudinary';
+import { loggerFactory } from '../server/logger.js';
+import AdmZip from 'adm-zip';
+import * as dir from 'path';
+import fs from 'fs-extra';
+import Downloader from '../server/downloader.js';
+import { shellExec } from '../server/process.js';
+import Underpost from '../index.js';
+
+const logger = loggerFactory(import.meta);
+
+/**
+ * @class UnderpostFileStorage
+ * @description Manages file storage operations using Cloudinary.
+ * This class provides a set of static methods to upload, pull, and delete files
+ * from Cloudinary, as well as manage a local storage configuration file.
+ */
+class UnderpostFileStorage {
+  static API = {
+    /**
+     * @method cloudinaryConfig
+     * @description Configures the Cloudinary client with environment variables.
+     * @memberof UnderpostFileStorage
+     */
+    cloudinaryConfig() {
+      // https://console.cloudinary.com/
+      cloudinary.config({
+        cloud_name: process.env.CLOUDINARY_CLOUD_NAME,
+        api_key: process.env.CLOUDINARY_API_KEY,
+        api_secret: process.env.CLOUDINARY_API_SECRET,
+      });
+    },
+    /**
+     * @method getStorageConf
+     * @description Retrieves the storage configuration for a specific deployment.
+     * @param {object} options - An object containing deployment-specific options.
+     * @param {string} options.deployId - The identifier for the deployment.
+     * @param {string} [options.storageFilePath] - The path to the storage configuration file.
+     * @returns {object} An object containing the storage configuration and storage file path.
+     * @memberof UnderpostFileStorage
+     */
+    getStorageConf(options) {
+      let storage, storageConf;
+      if (options.deployId && typeof options.deployId === 'string') {
+        storageConf = options.storageFilePath ?? `./engine-private/conf/${options.deployId}/storage.json`;
+        if (!fs.existsSync(storageConf)) fs.writeFileSync(storageConf, JSON.stringify({}), 'utf8');
+        storage = JSON.parse(fs.readFileSync(storageConf, 'utf8'));
+      }
+      return { storage, storageConf };
+    },
+    /**
+     * @method writeStorageConf
+     * @description Writes the storage configuration to a file.
+     * @param {object} storage - The storage configuration object.
+     * @param {string} storageConf - The path to the storage configuration file.
+     * @memberof UnderpostFileStorage
+     */
+    writeStorageConf(storage, storageConf) {
+      if (storage) fs.writeFileSync(storageConf, JSON.stringify(storage, null, 4), 'utf8');
+    },
+    /**
+     * @method gitTrack
+     * @description Optional, non-fatal Git tracking layer. Any Git error is logged and swallowed
+     * so it can never interrupt or roll back the canonical `storage.*.json` workflow.
+     * @param {string} gitPath - The working directory to stage/commit.
+     * @param {object} [options] - Tracking options.
+     * @param {boolean} [options.init=false] - If true, initialize a local repo before staging.
+     * @param {string} [options.message=''] - Explicit commit message; when omitted, `underpost cmt` is used.
+     * @memberof UnderpostFileStorage
+     */
+    gitTrack(gitPath, options = { init: false, message: '' }) {
+      try {
+        if (options.init === true) Underpost.repo.initLocalRepo({ path: gitPath });
+        shellExec(`cd ${gitPath} && git add .`, { silentOnError: true, silent: true, disableLog: true });
+        if (options.message)
+          shellExec(`cd ${gitPath} && git commit -m "${options.message}"`, {
+            silentOnError: true,
+            silent: true,
+            disableLog: true,
+          });
+        else shellExec(`underpost cmt ${gitPath} feat`, { silentOnError: true, silent: true, disableLog: true });
+      } catch (error) {
+        logger.warn('git tracking skipped (non-fatal)', { gitPath, error: error?.message });
+      }
+    },
+    /**
+     * @method recursiveCallback
+     * @description Recursively processes files and directories based on the provided options.
+     * @param {string} path - The path to the directory to process.
+     * @param {object} [options] - An object containing options for the recursive callback.
+     * @param {boolean} [options.rm=false] - Flag to remove files and directories.
+     * @param {boolean} [options.recursive=false] - Flag to process directories recursively.
+     * @param {string} [options.deployId=''] - The identifier for the deployment.
+     * @param {boolean} [options.force=false] - Flag to force file operations.
+     * @param {boolean} [options.pull=false] - Flag to pull files from storage.
+     * @param {boolean} [options.git=false] - Flag to use Git for file operations.
+     * @param {boolean} [options.omitUnzip=false] - If true, do not extract zip and keep downloaded zip file.
+     * @param {string} [options.storageFilePath=''] - The path to the storage configuration file.
+     * @returns {Promise<void>} A promise that resolves when the recursive callback is complete.
+     * @memberof UnderpostFileStorage
+     */
+    async recursiveCallback(
+      path,
+      options = {
+        rm: false,
+        recursive: false,
+        deployId: '',
+        force: false,
+        pull: false,
+        git: false,
+        omitUnzip: false,
+        storageFilePath: '',
+      },
+    ) {
+      const { storage, storageConf } = Underpost.fs.getStorageConf(options);
+
+      // ── Single-file handling: when path is a file (not a directory), use parent dir
+      //    as the git working directory and process just that one file. ──────────────
+      let isSingleFile = false;
+      let parentDir = path;
+      let singleFileName = '';
+      if (fs.existsSync(path) && !fs.statSync(path).isDirectory()) {
+        isSingleFile = true;
+        parentDir = dir.dirname(path);
+        singleFileName = path.split('/').pop();
+      }
+
+      // In recursive remove mode, delete every tracked storage key under the requested path,
+      // even when local files/directories are already missing.
+      if (options.rm === true) {
+        const normalizedPath = typeof path === 'string' ? path.trim() : '';
+        const basePath = normalizedPath.replace(/\/+$/, '');
+        const hasPathFilter = basePath.length > 0;
+
+        const associatedPaths = Object.keys(storage || {}).filter((storedPath) => {
+          if (!hasPathFilter) return true;
+          return storedPath === basePath || storedPath.startsWith(`${basePath}/`);
+        });
+
+        for (const associatedPath of associatedPaths) {
+          await Underpost.fs.delete(associatedPath);
+          if (storage) delete storage[associatedPath];
+        }
+
+        if (hasPathFilter && options.force === true && fs.existsSync(basePath)) fs.removeSync(basePath);
+
+        // Storage is canonical: persist the removal before any (optional) git tracking runs.
+        Underpost.fs.writeStorageConf(storage, storageConf);
+
+        if (associatedPaths.length === 0)
+          logger.warn('No associated tracked storage paths found', { path: hasPathFilter ? basePath : '*' });
+        else
+          logger.info('Removed associated tracked storage paths', {
+            path: hasPathFilter ? basePath : '*',
+            removed: associatedPaths.length,
+          });
+
+        if (options.git === true) {
+          const gitPath = !hasPathFilter ? '.' : isSingleFile ? parentDir : basePath;
+          Underpost.fs.gitTrack(gitPath);
+        }
+
+        return;
+      }
+
+      // For single files, run getDeleteFiles against the parent directory to avoid
+      // trying to `cd` into a file.
+      const gitContextPath = isSingleFile ? parentDir : path;
+      // Detecting locally-deleted files is a best-effort enhancement backed by git; if the path is
+      // not a repo (or git is unavailable) it must not block the canonical storage workflow.
+      let deleteFiles = [];
+      if (options.pull !== true) {
+        try {
+          deleteFiles = Underpost.repo.getDeleteFiles(gitContextPath);
+        } catch (error) {
+          logger.warn('delete detection skipped (git unavailable)', { path: gitContextPath, error: error?.message });
+        }
+      }
+
+      // When processing a single file, only consider it for deletion
+      for (const relativePath of deleteFiles) {
+        const _path = isSingleFile ? (relativePath === singleFileName ? path : null) : path + '/' + relativePath;
+        if (_path && _path in storage) {
+          await Underpost.fs.delete(_path);
+          delete storage[_path];
+        }
+      }
+      if (options.pull === true) {
+        let pullSkipCount = 0;
+        for (const _path of Object.keys(storage)) {
+          if (!fs.existsSync(_path) || options.force === true) {
+            if (options.force === true && fs.existsSync(_path)) fs.removeSync(_path);
+            await Underpost.fs.pull(_path, options);
+          } else pullSkipCount++;
+        }
+        if (pullSkipCount > 0) logger.warn(`Pull skipped ${pullSkipCount} files that already exist`);
+        // Only run git init/commit when the caller explicitly requests git tracking (--git flag).
+        // For bundle pulls into ./build the git step is unwanted and would error on a non-repo path.
+        if (options.git === true) Underpost.fs.gitTrack(gitContextPath, { init: true, message: 'Base pull state' });
+      } else {
+        let files;
+        if (isSingleFile) {
+          // Single file: treat the file itself as the sole item to process
+          files = [singleFileName];
+        } else {
+          files =
+            options.git === true
+              ? Underpost.repo.getChangedFiles(gitContextPath)
+              : await fs.readdir(path, { recursive: true });
+        }
+        for (const relativePath of files) {
+          const _path = isSingleFile ? path : path + '/' + relativePath;
+          if (fs.existsSync(_path) && fs.statSync(_path).isDirectory()) {
+            if (options.pull === true && !fs.existsSync(_path)) fs.mkdirSync(_path, { recursive: true });
+            continue;
+          } else if (!(_path in storage) || options.force === true) {
+            await Underpost.fs.upload(_path, options);
+            if (storage) storage[_path] = {};
+          } else logger.warn('File already exists', _path);
+        }
+      }
+      // Storage is canonical and always persisted; git is an optional layer on top.
+      Underpost.fs.writeStorageConf(storage, storageConf);
+      if (options.git === true) Underpost.fs.gitTrack(gitContextPath);
+    },
+    /**
+     * @method callback
+     * @description Orchestrates file storage operations based on the provided options.
+     * This method handles file uploads, deletions, and recursive processing of directories.
+     * @param {string} path - The path to the file or directory to process.
+     * @param {object} [options] - An object containing options for the callback.
+     * @param {boolean} [options.rm=false] - Flag to remove files and directories.
+     * @param {boolean} [options.recursive=false] - Flag to process directories recursively.
+     * @param {string} [options.deployId=''] - The identifier for the deployment.
+     * @param {boolean} [options.force=false] - Flag to force file operations.
+     * @param {boolean} [options.pull=false] - Flag to pull files from storage.
+     * @param {boolean} [options.git=false] - Flag to use Git for file operations.
+     * @param {boolean} [options.omitUnzip=false] - If true, do not extract zip and keep downloaded zip file.
+     * @returns {Promise<void>} A promise that resolves when the callback is complete.
+     * @memberof UnderpostFileStorage
+     */
+    async callback(
+      path,
+      options = { rm: false, recursive: false, deployId: '', force: false, pull: false, git: false, omitUnzip: false },
+    ) {
+      // rm always routes through recursiveCallback so storage.*.json is updated regardless of
+      // --recursive/--git. The bare `delete` primitive only removes the remote asset and would
+      // otherwise leave the tracked storage key orphaned.
+      if (options.recursive === true || options.git === true || options.rm === true)
+        return await Underpost.fs.recursiveCallback(path, options);
+      if (options.pull === true) return await Underpost.fs.pull(path, options);
+      return await Underpost.fs.upload(path, options);
+    },
+    /**
+     * @method upload
+     * @description Uploads a file to Cloudinary.
+     * @param {string} path - The path to the file to upload.
+     * @param {object} [options] - An object containing options for the upload.
+     * @param {string} [options.deployId=''] - The identifier for the deployment (used to locate the storage config file).
+     * @param {boolean} [options.force=false] - Flag to force file operations (overwrites existing remote asset).
+     * @param {string} [options.storageFilePath=''] - The path to the storage configuration file.
+     * @returns {Promise<object>} A promise that resolves to the upload result.
+     * @memberof UnderpostFileStorage
+     */
+
+    async upload(
+      path,
+      options = { rm: false, recursive: false, deployId: '', force: false, pull: false, storageFilePath: '' },
+    ) {
+      Underpost.fs.cloudinaryConfig();
+      const { storage, storageConf } = Underpost.fs.getStorageConf(options);
+      // path = Underpost.fs.file2Zip(path);
+      const uploadResult = await cloudinary.uploader
+        .upload(path, {
+          public_id: path,
+          resource_type: 'raw',
+          overwrite: options.force === true ? true : false,
+        })
+        .catch((error) => {
+          logger.error(error, { path, stack: error.stack });
+        });
+      logger.info('upload result', uploadResult);
+      if (storage) storage[path] = {};
+      Underpost.fs.writeStorageConf(storage, storageConf);
+      return uploadResult;
+    },
+    /**
+     * @method pull
+     * @description Pulls a file from Cloudinary.
+     * @param {string} path - The path to the file to pull.
+     * @param {object} [options] - Pull options.
+     * @param {boolean} [options.omitUnzip=false] - If true, do not extract zip and keep downloaded zip file.
+     * @param {boolean} [options.force=false] - If true, re-download even if the local zip already exists.
+     * @returns {Promise<void>} A promise that resolves when the file is pulled.
+     * @memberof UnderpostFileStorage
+     */
+    async pull(path, options = { omitUnzip: false, force: false }) {
+      Underpost.fs.cloudinaryConfig();
+      const folder = dir.dirname(path);
+      if (!fs.existsSync(folder)) fs.mkdirSync(folder, { recursive: true });
+      const zipPath = `${path}.zip`;
+
+      if (options.omitUnzip === true && options.force !== true && fs.existsSync(zipPath)) {
+        logger.warn('pull skipped, zip already exists and omit-unzip is enabled', { path, zipPath });
+        return;
+      }
+
+      const downloadResult = await cloudinary.utils.download_archive_url({
+        public_ids: [path],
+        resource_type: 'raw',
+      });
+      await Downloader.downloadFile(downloadResult, zipPath);
+
+      if (options.omitUnzip === true) {
+        logger.warn('omit unzip enabled, keeping downloaded zip file', { path, zipPath });
+        return;
+      }
+
+      path = Underpost.fs.zip2File(zipPath);
+      fs.removeSync(`${path}.zip`);
+    },
+    /**
+     * @method delete
+     * @description Deletes a file from Cloudinary by its public ID.
+     * @param {string} path - The path (public ID) of the file to delete.
+     * @returns {Promise<object>} A promise that resolves to the Cloudinary delete result.
+     * @memberof UnderpostFileStorage
+     */
+    async delete(path) {
+      Underpost.fs.cloudinaryConfig();
+      const deleteResult = await cloudinary.api
+        .delete_resources([path], { type: 'upload', resource_type: 'raw' })
+        .catch((error) => {
+          logger.error(error, { path, stack: error.stack });
+        });
+      logger.info('delete result', deleteResult);
+      return deleteResult;
+    },
+    /**
+     * @method file2Zip
+     * @description Converts a file to a zip file.
+     * @param {string} path - The path to the file to convert.
+     * @returns {string} The path to the zip file.
+     * @memberof UnderpostFileStorage
+     */
+    file2Zip(path) {
+      const zip = new AdmZip();
+      zip.addLocalFile(path, '/');
+      path = path + '.zip';
+      zip.writeZip(path);
+      return path;
+    },
+    /**
+     * @method zip2File
+     * @description Converts a zip file to a file.
+     * @param {string} path - The path to the zip file to convert.
+     * @returns {string} The path to the file.
+     * @memberof UnderpostFileStorage
+     */
+    zip2File(path) {
+      const zip = new AdmZip(path);
+      path = path.replaceAll('.zip', '');
+      zip.extractEntryTo(
+        /*entry name*/ path.split('/').pop(),
+        /*target path*/ dir.dirname(path),
+        /*maintainEntryPath*/ false,
+        /*overwrite*/ true,
+      );
+      return path;
+    },
+  };
+}
+
+export default UnderpostFileStorage;