@underpostnet/underpost 2.8.5 → 2.8.6

package/manifests/mongodb-4.4/service-deployment.yaml

@@ -0,0 +1,63 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: mongodb-deployment
+  namespace: default
+spec:
+  selector:
+    matchLabels:
+      app: mongodb
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: mongodb
+    spec:
+      hostname: mongo
+      containers:
+        - name: mongodb
+          image: docker.io/library/mongo:4.4
+          command: ['mongod', '--replSet', 'rs0', '--bind_ip_all']
+          # -- bash
+          # mongo
+          # use admin
+          # rs.initiate()
+          ports:
+            - containerPort: 27017
+
+          volumeMounts:
+            - name: mongo-persistent-storage
+              mountPath: /data/db
+
+      # resources:
+      #   requests:
+      #     memory: '500Mi'
+      #     cpu: '500m'
+      volumes:
+        - name: mongo-persistent-storage
+          persistentVolumeClaim:
+            claimName: mongodb-pvc
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: mongodb-service
+  namespace: default
+spec:
+  clusterIP: None
+  selector:
+    app: mongodb
+  ports:
+    - protocol: TCP
+      port: 27017
+#     targetPort: 27017
+# type: ClusterIP
+
+# rs.initiate();
+# use admin;
+# rs.initiate({ _id: "rs0", members: [ { _id: 0, host: "127.0.0.1:27017" } ] })
+# db.createUser({user: "your_user",pwd: "your_password", roles: [{role: "userAdminAnyDatabase", db: "admin" },{ role: "readWriteAnyDatabase", db: "admin" }]});
+# cfg = rs.conf();
+# cfg.members[0].host = "127.0.0.1:27027";
+# rs.reconfig(cfg);

package/package.json

@@ -2,7 +2,7 @@
     "type": "module",
     "main": "src/index.js",
     "name": "@underpostnet/underpost",
-    "version": "2.8.5",
+    "version": "2.8.6",
     "description": "pwa api rest template",
     "scripts": {
         "start": "env-cmd -f .env.production node --max-old-space-size=8192 src/server",
@@ -10,6 +10,7 @@
         "pm2": "env-cmd -f .env.production pm2 start src/server.js --node-args=\"--max-old-space-size=8192\" --name engine",
         "dev": "env-cmd -f .env.development node src/client.dev default",
         "dev-img": "env-cmd -f .env.development node src/server",
+        "prod-img": "env-cmd -f .env.production node src/server",
         "dev-api": "env-cmd -f .env.development nodemon --watch src --ignore src/client src/api",
         "dev-client": "env-cmd -f .env.development node src/client.dev",
         "proxy": "node src/proxy proxy",
@@ -56,9 +57,11 @@
         "ag-grid-community": "31.0.0",
         "axios": "^1.5.1",
         "chai": "^5.1.0",
+        "clean-jsdoc-theme": "^4.3.0",
         "cli-progress": "^3.12.0",
         "cli-spinners": "^3.0.0",
         "clipboardy": "^4.0.0",
+        "cloudinary": "^2.5.1",
         "color": "^4.2.3",
         "colors": "^1.4.0",
         "commander": "^12.1.0",
@@ -66,6 +69,7 @@
         "cors": "^2.8.5",
         "d3": "^7.9.0",
         "dotenv": "^16.3.1",
+        "easy-json-schema": "^0.0.2-beta",
         "easymde": "^2.18.0",
         "env-cmd": "^10.1.0",
         "express": "^4.18.2",
@@ -86,6 +90,7 @@
         "log-update": "^6.0.0",
         "mariadb": "^3.2.2",
         "marked": "^12.0.2",
+        "mocha": "^10.8.2",
         "mongoose": "^8.9.5",
         "morgan": "^1.10.0",
         "nodemailer": "^6.9.9",
@@ -94,6 +99,7 @@
         "peer": "^1.0.2",
         "peerjs": "^1.5.2",
         "pixi.js": "7.4.2",
+        "plantuml": "^0.0.2",
         "prom-client": "^15.1.2",
         "public-ip": "^6.0.1",
         "read": "^2.1.0",
@@ -105,17 +111,13 @@
         "socket.io": "^4.8.0",
         "sortablejs": "^1.15.0",
         "split-file": "^2.3.0",
+        "swagger-autogen": "^2.23.7",
         "swagger-ui-express": "^5.0.0",
         "systeminformation": "^5.23.7",
         "uglify-js": "^3.17.4",
         "validator": "^13.11.0",
         "vanilla-jsoneditor": "^2.3.2",
-        "winston": "^3.11.0",
-        "clean-jsdoc-theme": "^4.3.0",
-        "easy-json-schema": "^0.0.2-beta",
-        "mocha": "^10.8.2",
-        "plantuml": "^0.0.2",
-        "swagger-autogen": "^2.23.7"
+        "winston": "^3.11.0"
     },
     "devDependencies": {
         "clean-jsdoc-theme": "^4.3.0",

package/src/cli/cluster.js

@@ -1,18 +1,40 @@
 import { timer } from '../client/components/core/CommonJs.js';
-import { cliSpinner } from '../server/conf.js';
+import { cliSpinner, getNpmRootPath } from '../server/conf.js';
 import { loggerFactory } from '../server/logger.js';
 import { shellExec } from '../server/process.js';
+import UnderpostDeploy from './deploy.js';
+import UnderpostTest from './test.js';
 
 const logger = loggerFactory(import.meta);
 
 class UnderpostCluster {
   static API = {
-    async init(options = { valkey: false, mariadb: false, valkey: false, full: false, info: false, nsUse: '' }) {
-      if (options.nsUse) {
+    async init(
+      podName,
+      options = {
+        mongodb: false,
+        mongodb4: false,
+        mariadb: false,
+        valkey: false,
+        full: false,
+        info: false,
+        certManager: false,
+        listPods: false,
+        reset: false,
+        dev: false,
+        nsUse: '',
+      },
+    ) {
+      const npmRoot = getNpmRootPath();
+      const underpostRoot = options?.dev === true ? '.' : `${npmRoot}/underpost`;
+      if (options.reset === true) return await UnderpostCluster.API.reset();
+      if (options.listPods === true) return console.table(UnderpostDeploy.API.get(podName ?? undefined));
+
+      if (options.nsUse && typeof options.nsUse === 'string') {
         shellExec(`kubectl config set-context --current --namespace=${options.nsUse}`);
         return;
       }
-      if (options.info) {
+      if (options.info === true) {
         shellExec(`kubectl config get-contexts`); // config env persisente for manage multiple clusters
         shellExec(`kubectl config get-clusters`);
         shellExec(`kubectl get nodes -o wide`); // set of nodes of a cluster
@@ -40,28 +62,31 @@ class UnderpostCluster {
         logger.info('----------------------------------------------------------------');
         shellExec(`kubectl get secrets --all-namespaces -o wide`);
         shellExec(`docker secret ls`);
+        shellExec(`kubectl get crd --all-namespaces -o wide`);
         return;
       }
-      const testClusterInit = shellExec(`kubectl get pods --all-namespaces -o wide`, {
-        disableLog: true,
-        silent: true,
-        stdout: true,
-      });
-      if (!(testClusterInit.match('kube-system') && testClusterInit.match('kube-proxy'))) {
+
+      if (!UnderpostDeploy.API.get('kube-apiserver-kind-control-plane')[0]) {
         shellExec(`containerd config default > /etc/containerd/config.toml`);
         shellExec(`sed -i -e "s/SystemdCgroup = false/SystemdCgroup = true/g" /etc/containerd/config.toml`);
         // shellExec(`cp /etc/kubernetes/admin.conf ~/.kube/config`);
         shellExec(`sudo systemctl restart kubelet`);
         shellExec(`sudo service docker restart`);
-        shellExec(`cd ./manifests && kind create cluster --config kind-config.yaml`);
+        shellExec(`sudo systemctl enable --now containerd.service`);
+        shellExec(`sudo systemctl restart containerd`);
+        shellExec(
+          `cd ${underpostRoot}/manifests && kind create cluster --config kind-config${
+            options?.dev === true ? '-dev' : ''
+          }.yaml`,
+        );
         shellExec(`sudo chown $(id -u):$(id -g) $HOME/.kube/config**`);
       } else logger.warn('Cluster already initialized');
 
-      if (options.full || options.valkey) {
+      if (options.full === true || options.valkey === true) {
         shellExec(`kubectl delete statefulset service-valkey`);
-        shellExec(`kubectl apply -k ./manifests/valkey`);
+        shellExec(`kubectl apply -k ${underpostRoot}/manifests/valkey`);
       }
-      if (options.full || options.mariadb) {
+      if (options.full === true || options.mariadb === true) {
         shellExec(
           `sudo kubectl create secret generic mariadb-secret --from-file=username=/home/dd/engine/engine-private/mariadb-username --from-file=password=/home/dd/engine/engine-private/mariadb-password`,
         );
@@ -69,9 +94,31 @@ class UnderpostCluster {
           `sudo kubectl create secret generic github-secret --from-literal=GITHUB_TOKEN=${process.env.GITHUB_TOKEN}`,
         );
         shellExec(`kubectl delete statefulset mariadb-statefulset`);
-        shellExec(`kubectl apply -k ./manifests/mariadb`);
+        shellExec(`kubectl apply -k ${underpostRoot}/manifests/mariadb`);
       }
-      if (options.full || options.mongodb) {
+      if (options.mongodb4 === true) {
+        shellExec(`kubectl apply -k ${underpostRoot}/manifests/mongodb-4.4`);
+
+        const deploymentName = 'mongodb-deployment';
+
+        const successInstance = await UnderpostTest.API.statusMonitor(deploymentName);
+
+        if (successInstance) {
+          const mongoConfig = {
+            _id: 'rs0',
+            members: [{ _id: 0, host: '127.0.0.1:27017' }],
+          };
+
+          const [pod] = UnderpostDeploy.API.get(deploymentName);
+
+          shellExec(
+            `sudo kubectl exec -i ${pod.NAME} -- mongo --quiet \
+        --eval 'rs.initiate(${JSON.stringify(mongoConfig)})'`,
+          );
+        }
+
+        // await UnderpostTest.API.statusMonitor('mongodb-1');
+      } else if (options.full === true || options.mongodb === true) {
         shellExec(
           `sudo kubectl create secret generic mongodb-keyfile --from-file=/home/dd/engine/engine-private/mongodb-keyfile`,
         );
@@ -79,46 +126,47 @@ class UnderpostCluster {
           `sudo kubectl create secret generic mongodb-secret --from-file=username=/home/dd/engine/engine-private/mongodb-username --from-file=password=/home/dd/engine/engine-private/mongodb-password`,
         );
         shellExec(`kubectl delete statefulset mongodb`);
-        shellExec(`kubectl apply -k ./manifests/mongodb`);
-
-        await new Promise(async (resolve) => {
-          cliSpinner(3000, `[cluster.js] `, ` Load mongodb instance`, 'yellow', 'material');
-          await timer(3000);
-
-          const monitor = async () => {
-            cliSpinner(1000, `[cluster.js] `, ` Load mongodb instance`, 'yellow', 'material');
-            await timer(1000);
-            if (
-              shellExec(`kubectl get pods --all-namespaces -o wide`, {
-                silent: true,
-                stdout: true,
-                disableLog: true,
-              }).match(`mongodb-1                                    1/1     Running`)
-            )
-              return resolve();
-            return monitor();
-          };
-          await monitor();
-        });
+        shellExec(`kubectl apply -k ${underpostRoot}/manifests/mongodb`);
 
-        const mongoConfig = {
-          _id: 'rs0',
-          members: [
-            { _id: 0, host: 'mongodb-0.mongodb-service:27017', priority: 1 },
-            { _id: 1, host: 'mongodb-1.mongodb-service:27017', priority: 1 },
-          ],
-        };
+        const successInstance = await UnderpostTest.API.statusMonitor('mongodb-1');
 
-        shellExec(
-          `sudo kubectl exec -i mongodb-0 -- mongosh --quiet --json=relaxed \
-          --eval 'use admin' \
-          --eval 'rs.initiate(${JSON.stringify(mongoConfig)})' \
-          --eval 'rs.status()'`,
-        );
+        if (successInstance) {
+          const mongoConfig = {
+            _id: 'rs0',
+            members: [
+              { _id: 0, host: 'mongodb-0.mongodb-service:27017', priority: 1 },
+              { _id: 1, host: 'mongodb-1.mongodb-service:27017', priority: 1 },
+            ],
+          };
+
+          shellExec(
+            `sudo kubectl exec -i mongodb-0 -- mongosh --quiet --json=relaxed \
+        --eval 'use admin' \
+        --eval 'rs.initiate(${JSON.stringify(mongoConfig)})' \
+        --eval 'rs.status()'`,
+          );
+        }
       }
 
-      if (options.full || options.contour)
+      if (options.full === true || options.contour === true)
         shellExec(`kubectl apply -f https://projectcontour.io/quickstart/contour.yaml`);
+
+      if (options.full === true || options.certManager === true) {
+        if (!UnderpostDeploy.API.get('cert-manager').find((p) => p.STATUS === 'Running')) {
+          shellExec(`helm repo add jetstack https://charts.jetstack.io --force-update`);
+          shellExec(
+            `helm install cert-manager jetstack/cert-manager \
+--namespace cert-manager \
+--create-namespace \
+--version v1.17.0 \
+--set crds.enabled=true`,
+          );
+        }
+
+        const letsEncName = 'letsencrypt-prod';
+        shellExec(`sudo kubectl delete ClusterIssuer ${letsEncName}`);
+        shellExec(`sudo kubectl apply -f ${underpostRoot}/manifests/${letsEncName}.yaml`);
+      }
     },
     reset() {
       shellExec(`kind get clusters | xargs -t -n1 kind delete cluster --name`);

package/src/cli/cron.js

@@ -46,7 +46,7 @@ class UnderpostCron {
     callback: async function (
       deployList = 'default',
       jobList = Object.keys(UnderpostCron.JOB),
-      options = { disableKindCluster: false, init: false },
+      options = { itc: false, init: false, git: false },
     ) {
       if (options.init === true) {
         await Underpost.test.setUpInfo();

package/src/cli/db.js

@@ -2,14 +2,28 @@ import { mergeFile, splitFileFactory } from '../server/conf.js';
 import { loggerFactory } from '../server/logger.js';
 import { shellExec } from '../server/process.js';
 import fs from 'fs-extra';
+import UnderpostDeploy from './deploy.js';
 
 const logger = loggerFactory(import.meta);
 
 class UnderpostDB {
   static API = {
-    async callback(deployList = 'default', options = { import: false, export: false }) {
+    async callback(
+      deployList = 'default',
+      options = {
+        import: false,
+        export: false,
+        podName: false,
+        ns: false,
+        collection: '',
+        outPath: '',
+        drop: false,
+        preserveUUID: false,
+        git: false,
+      },
+    ) {
       const newBackupTimestamp = new Date().getTime();
-      const nameSpace = 'default';
+      const nameSpace = options.ns && typeof options.ns === 'string' ? options.ns : 'default';
       for (const _deployId of deployList.split(',')) {
         const deployId = _deployId.trim();
         if (!deployId) continue;
@@ -43,10 +57,12 @@ class UnderpostDB {
               logger.info('', { hostFolder, provider, dbName });
 
               const backUpPath = `../${repoName}/${hostFolder}`;
+              if (!fs.existsSync(backUpPath)) fs.mkdirSync(backUpPath, { recursive: true });
+              shellExec(`cd ${backUpPath} && find . -type d -empty -delete`); // delete empty folders
               const times = await fs.readdir(backUpPath);
-              const currentBackupTimestamp = Math.max(...times.map((t) => parseInt(t)));
+              const currentBackupTimestamp = Math.max(...times.map((t) => parseInt(t)).filter((t) => !isNaN(t)));
               dbs[provider][dbName].currentBackupTimestamp = currentBackupTimestamp;
-              const removeBackupTimestamp = Math.min(...times.map((t) => parseInt(t)));
+              const removeBackupTimestamp = Math.min(...times.map((t) => parseInt(t)).filter((t) => !isNaN(t)));
 
               const sqlContainerPath = `/home/${dbName}.sql`;
               const _fromPartsParts = `../${repoName}/${hostFolder}/${currentBackupTimestamp}/${dbName}-parths.json`;
@@ -68,57 +84,105 @@ class UnderpostDB {
               }
 
               if (options.export === true && times.length >= 5) {
+                logger.info('remove', `../${repoName}/${hostFolder}/${removeBackupTimestamp}`);
                 fs.removeSync(`../${repoName}/${hostFolder}/${removeBackupTimestamp}`);
+                logger.info('create', `../${repoName}/${hostFolder}/${newBackupTimestamp}`);
                 fs.mkdirSync(`../${repoName}/${hostFolder}/${newBackupTimestamp}`, { recursive: true });
               }
 
               switch (provider) {
                 case 'mariadb': {
-                  const podName = `mariadb-statefulset-0`;
+                  const podNames =
+                    options.podName && typeof options.podName === 'string'
+                      ? options.podName.split(',')
+                      : UnderpostDeploy.API.get('mariadb'); // `mariadb-statefulset-0`;
                   const serviceName = 'mariadb';
-                  if (options.import === true) {
-                    shellExec(`sudo kubectl cp ${_toSqlPath} ${nameSpace}/${podName}:/${dbName}.sql`);
-                    const cmd = `mariadb -u ${user} -p${password} ${dbName} < /${dbName}.sql`;
-                    shellExec(
-                      `kubectl exec -i ${podName} -- ${serviceName} -p${password} -e 'CREATE DATABASE ${dbName};'`,
-                    );
-                    shellExec(`sudo kubectl exec -i ${podName} -- sh -c "${cmd}"`);
-                  }
-                  if (options.export === true) {
-                    const cmd = `mariadb-dump --user=${user} --password=${password} --lock-tables ${dbName} > ${sqlContainerPath}`;
-                    shellExec(`sudo kubectl exec -i ${podName} -- sh -c "${cmd}"`);
-                    shellExec(`sudo kubectl cp ${nameSpace}/${podName}:${sqlContainerPath} ${_toNewSqlPath}`);
-                    await splitFileFactory(dbName, _toNewSqlPath);
+                  for (const podNameData of [podNames[0]]) {
+                    const podName = podNameData.NAME;
+                    if (options.import === true) {
+                      shellExec(`sudo kubectl exec -i ${podName} -- sh -c "rm -rf /${dbName}.sql"`);
+                      shellExec(`sudo kubectl cp ${_toSqlPath} ${nameSpace}/${podName}:/${dbName}.sql`);
+                      const cmd = `mariadb -u ${user} -p${password} ${dbName} < /${dbName}.sql`;
+                      shellExec(
+                        `kubectl exec -i ${podName} -- ${serviceName} -p${password} -e 'CREATE DATABASE ${dbName};'`,
+                      );
+                      shellExec(`sudo kubectl exec -i ${podName} -- sh -c "${cmd}"`);
+                    }
+                    if (options.export === true) {
+                      shellExec(`sudo kubectl exec -i ${podName} -- sh -c "rm -rf ${sqlContainerPath}"`);
+                      const cmd = `mariadb-dump --user=${user} --password=${password} --lock-tables ${dbName} > ${sqlContainerPath}`;
+                      shellExec(`sudo kubectl exec -i ${podName} -- sh -c "${cmd}"`);
+                      shellExec(
+                        `sudo kubectl cp ${nameSpace}/${podName}:${sqlContainerPath} ${
+                          options.outPath ? options.outPath : _toNewSqlPath
+                        }`,
+                      );
+                      await splitFileFactory(dbName, options.outPath ? options.outPath : _toNewSqlPath);
+                    }
                   }
                   break;
                 }
 
                 case 'mongoose': {
                   if (options.import === true) {
-                    const podName = `mongodb-0`;
-                    shellExec(`sudo kubectl cp ${_toBsonPath} ${nameSpace}/${podName}:/${dbName}`);
-                    const cmd = `mongorestore -d ${dbName} /${dbName}`;
-                    shellExec(`sudo kubectl exec -i ${podName} -- sh -c "${cmd}"`);
+                    const podNames =
+                      options.podName && typeof options.podName === 'string'
+                        ? options.podName.split(',')
+                        : UnderpostDeploy.API.get('mongo');
+                    // `mongodb-0`;
+                    for (const podNameData of [podNames[0]]) {
+                      const podName = podNameData.NAME;
+                      shellExec(`sudo kubectl exec -i ${podName} -- sh -c "rm -rf /${dbName}"`);
+                      shellExec(
+                        `sudo kubectl cp ${
+                          options.outPath ? options.outPath : _toBsonPath
+                        } ${nameSpace}/${podName}:/${dbName}`,
+                      );
+                      const cmd = `mongorestore -d ${dbName} /${dbName}${options.drop ? ' --drop' : ''}${
+                        options.preserveUUID ? ' --preserveUUID' : ''
+                      }`;
+                      shellExec(`sudo kubectl exec -i ${podName} -- sh -c "${cmd}"`);
+                    }
                   }
                   if (options.export === true) {
-                    const podName = `backup-access`;
-                    const containerBaseBackupPath = '/backup';
-                    let timeFolder = shellExec(
-                      `sudo kubectl exec -i ${podName} -- sh -c "cd ${containerBaseBackupPath} && ls -a"`,
-                      {
-                        stdout: true,
-                        disableLog: false,
-                        silent: true,
-                      },
-                    ).split(`\n`);
-                    timeFolder = timeFolder[timeFolder.length - 2];
-                    if (timeFolder === '..') {
-                      logger.warn(`Cannot backup available`, { timeFolder });
-                    } else {
+                    const podNames =
+                      options.podName && typeof options.podName === 'string'
+                        ? options.podName.split(',')
+                        : UnderpostDeploy.API.get('mongo'); // `backup-access`;
+                    for (const podNameData of [podNames[0]]) {
+                      const podName = podNameData.NAME;
+                      shellExec(`sudo kubectl exec -i ${podName} -- sh -c "rm -rf /${dbName}"`);
+                      if (options.collections)
+                        for (const collection of options.collections)
+                          shellExec(
+                            `sudo kubectl exec -i ${podName} -- sh -c "mongodump -d ${dbName} --collection ${collection} -o /${dbName}"`,
+                          );
+                      else shellExec(`sudo kubectl exec -i ${podName} -- sh -c "mongodump -d ${dbName} -o /${dbName}"`);
                       shellExec(
-                        `sudo kubectl cp ${nameSpace}/${podName}:${containerBaseBackupPath}/${timeFolder}/${dbName} ${_toNewBsonPath}`,
+                        `sudo kubectl cp ${nameSpace}/${podName}:/${dbName} ${
+                          options.outPath ? options.outPath : _toNewBsonPath
+                        }`,
                       );
                     }
+                    if (false) {
+                      const containerBaseBackupPath = '/backup';
+                      let timeFolder = shellExec(
+                        `sudo kubectl exec -i ${podName} -- sh -c "cd ${containerBaseBackupPath} && ls -a"`,
+                        {
+                          stdout: true,
+                          disableLog: false,
+                          silent: true,
+                        },
+                      ).split(`\n`);
+                      timeFolder = timeFolder[timeFolder.length - 2];
+                      if (timeFolder === '..') {
+                        logger.warn(`Cannot backup available`, { timeFolder });
+                      } else {
+                        shellExec(
+                          `sudo kubectl cp ${nameSpace}/${podName}:${containerBaseBackupPath}/${timeFolder}/${dbName} ${_toNewBsonPath}`,
+                        );
+                      }
+                    }
                   }
                   break;
                 }
@@ -129,7 +193,7 @@ class UnderpostDB {
             }
           }
         }
-        if (options.export === true) {
+        if (options.export === true && options.git === true) {
           shellExec(`cd ../${repoName} && git add .`);
           shellExec(
             `underpost cmt ../${repoName} backup '' '${new Date(newBackupTimestamp).toLocaleDateString()} ${new Date(