@underpostnet/underpost 2.8.4 → 2.8.5

package/manifests/valkey/statefulset.yaml

@@ -0,0 +1,39 @@
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: service-valkey
+  namespace: default
+spec:
+  serviceName: service-valkey
+  replicas: 1
+  selector:
+    matchLabels:
+      app: service-valkey
+  template:
+    metadata:
+      labels:
+        app: service-valkey
+    spec:
+      containers:
+        - name: service-valkey
+          image: docker.io/valkey/valkey:latest
+          env:
+            - name: TZ
+              value: Europe/Zurich
+          ports:
+            - containerPort: 6379
+          startupProbe:
+            tcpSocket:
+              port: 6379
+            failureThreshold: 30
+            periodSeconds: 5
+            timeoutSeconds: 5
+          livenessProbe:
+            tcpSocket:
+              port: 6379
+            failureThreshold: 2
+            periodSeconds: 30
+            timeoutSeconds: 5
+      restartPolicy: Always
+      automountServiceAccountToken: false

package/package.json

@@ -2,20 +2,27 @@
     "type": "module",
     "main": "src/index.js",
     "name": "@underpostnet/underpost",
-    "version": "2.8.4",
+    "version": "2.8.5",
     "description": "pwa api rest template",
     "scripts": {
         "start": "env-cmd -f .env.production node --max-old-space-size=8192 src/server",
-        "build": "node bin/deploy build-full-client",
+        "test": "env-cmd -f .env.test c8 mocha",
+        "pm2": "env-cmd -f .env.production pm2 start src/server.js --node-args=\"--max-old-space-size=8192\" --name engine",
         "dev": "env-cmd -f .env.development node src/client.dev default",
+        "dev-img": "env-cmd -f .env.development node src/server",
         "dev-api": "env-cmd -f .env.development nodemon --watch src --ignore src/client src/api",
+        "dev-client": "env-cmd -f .env.development node src/client.dev",
+        "proxy": "node src/proxy proxy",
         "docs": "jsdoc -c jsdoc.json",
         "install-global": "npm install -g pm2 && npm install -g jsdoc && npm install -g prettier && npm install -g env-cmd && npm install -g yarn && npm install -g auto-changelog",
         "install-test": "npm install -g mocha && npm install -g c8 && npm install -g nyc && npm install -g coveralls",
+        "install-underpost": "cp -a $(npm root -g)/underpost/node_modules ./node_modules && npm install --only=dev --ignore-scripts",
         "install": "npm run install-global && npm run install-test",
         "docker:start": "docker-compose up",
         "prettier": "prettier --write .",
-        "test": "env-cmd -f .env.test c8 mocha"
+        "fix": "npm audit fix --force && npm audit",
+        "changelog": "auto-changelog",
+        "build": "node bin/deploy build-full-client"
     },
     "bin": {
         "underpost": "bin/index.js"
@@ -25,6 +32,10 @@
         "url": "git+https://github.com/underpostnet/pwa-microservices-template-ghpkg.git"
     },
     "keywords": [
+        "pwa",
+        "microservices",
+        "template",
+        "builder",
         "engine",
         "server",
         "proxy",
@@ -99,12 +110,17 @@
         "uglify-js": "^3.17.4",
         "validator": "^13.11.0",
         "vanilla-jsoneditor": "^2.3.2",
-        "winston": "^3.11.0"
+        "winston": "^3.11.0",
+        "clean-jsdoc-theme": "^4.3.0",
+        "easy-json-schema": "^0.0.2-beta",
+        "mocha": "^10.8.2",
+        "plantuml": "^0.0.2",
+        "swagger-autogen": "^2.23.7"
     },
     "devDependencies": {
         "clean-jsdoc-theme": "^4.3.0",
         "easy-json-schema": "^0.0.2-beta",
-        "mocha": "^10.4.0",
+        "mocha": "^10.8.2",
         "plantuml": "^0.0.2",
         "swagger-autogen": "^2.23.7"
     },

package/src/api/core/core.service.js

@@ -9,7 +9,7 @@ const CoreService = {
     /** @type {import('./core.model.js').CoreModel} */
     const Core = DataBaseProvider.instance[`${options.host}${options.path}`].mongoose.models.Core;
     if (req.path.startsWith('/sh')) {
-      if (req.body.print) return shellExec(req.body.sh, { stdout: true });
+      if (req.body.stdout) return shellExec(req.body.sh, { stdout: true });
       shellExec(req.body.sh, { async: true });
       return 'Command "' + req.body.sh + '" running';
     }

package/src/cli/cluster.js

@@ -0,0 +1,154 @@
+import { timer } from '../client/components/core/CommonJs.js';
+import { cliSpinner } from '../server/conf.js';
+import { loggerFactory } from '../server/logger.js';
+import { shellExec } from '../server/process.js';
+
+const logger = loggerFactory(import.meta);
+
+class UnderpostCluster {
+  static API = {
+    async init(options = { valkey: false, mariadb: false, valkey: false, full: false, info: false, nsUse: '' }) {
+      if (options.nsUse) {
+        shellExec(`kubectl config set-context --current --namespace=${options.nsUse}`);
+        return;
+      }
+      if (options.info) {
+        shellExec(`kubectl config get-contexts`); // config env persisente for manage multiple clusters
+        shellExec(`kubectl config get-clusters`);
+        shellExec(`kubectl get nodes -o wide`); // set of nodes of a cluster
+        shellExec(`kubectl config view | grep namespace`);
+        shellExec(`kubectl get ns -o wide`); // A namespace can have pods of different nodes
+        shellExec(`kubectl get pvc --all-namespaces -o wide`); // PersistentVolumeClaim -> request storage service
+        shellExec(`kubectl get pv --all-namespaces -o wide`); // PersistentVolume -> real storage
+        shellExec(`kubectl get cronjob --all-namespaces -o wide`);
+        shellExec(`kubectl get svc --all-namespaces -o wide`); // proxy dns gate way -> deployments, statefulsets, pods
+        shellExec(`kubectl get statefulsets --all-namespaces -o wide`); // set pods with data/volume persistence
+        shellExec(`kubectl get deployments --all-namespaces -o wide`); // set pods
+        shellExec(`kubectl get configmap --all-namespaces -o wide`);
+        shellExec(`kubectl get pods --all-namespaces -o wide`);
+        shellExec(
+          `kubectl get pod --all-namespaces -o="custom-columns=NAME:.metadata.name,INIT-CONTAINERS:.spec.initContainers[*].name,CONTAINERS:.spec.containers[*].name"`,
+        );
+        shellExec(
+          `kubectl get pods --all-namespaces -o=jsonpath='{range .items[*]}{"\\n"}{.metadata.name}{":\\t"}{range .spec.containers[*]}{.image}{", "}{end}{end}'`,
+        );
+        console.log();
+        logger.info('contour -------------------------------------------------');
+        for (const _k of ['Cluster', 'HTTPProxy', 'ClusterIssuer', 'Certificate']) {
+          shellExec(`kubectl get ${_k} --all-namespaces -o wide`);
+        }
+        logger.info('----------------------------------------------------------------');
+        shellExec(`kubectl get secrets --all-namespaces -o wide`);
+        shellExec(`docker secret ls`);
+        return;
+      }
+      const testClusterInit = shellExec(`kubectl get pods --all-namespaces -o wide`, {
+        disableLog: true,
+        silent: true,
+        stdout: true,
+      });
+      if (!(testClusterInit.match('kube-system') && testClusterInit.match('kube-proxy'))) {
+        shellExec(`containerd config default > /etc/containerd/config.toml`);
+        shellExec(`sed -i -e "s/SystemdCgroup = false/SystemdCgroup = true/g" /etc/containerd/config.toml`);
+        // shellExec(`cp /etc/kubernetes/admin.conf ~/.kube/config`);
+        shellExec(`sudo systemctl restart kubelet`);
+        shellExec(`sudo service docker restart`);
+        shellExec(`cd ./manifests && kind create cluster --config kind-config.yaml`);
+        shellExec(`sudo chown $(id -u):$(id -g) $HOME/.kube/config**`);
+      } else logger.warn('Cluster already initialized');
+
+      if (options.full || options.valkey) {
+        shellExec(`kubectl delete statefulset service-valkey`);
+        shellExec(`kubectl apply -k ./manifests/valkey`);
+      }
+      if (options.full || options.mariadb) {
+        shellExec(
+          `sudo kubectl create secret generic mariadb-secret --from-file=username=/home/dd/engine/engine-private/mariadb-username --from-file=password=/home/dd/engine/engine-private/mariadb-password`,
+        );
+        shellExec(
+          `sudo kubectl create secret generic github-secret --from-literal=GITHUB_TOKEN=${process.env.GITHUB_TOKEN}`,
+        );
+        shellExec(`kubectl delete statefulset mariadb-statefulset`);
+        shellExec(`kubectl apply -k ./manifests/mariadb`);
+      }
+      if (options.full || options.mongodb) {
+        shellExec(
+          `sudo kubectl create secret generic mongodb-keyfile --from-file=/home/dd/engine/engine-private/mongodb-keyfile`,
+        );
+        shellExec(
+          `sudo kubectl create secret generic mongodb-secret --from-file=username=/home/dd/engine/engine-private/mongodb-username --from-file=password=/home/dd/engine/engine-private/mongodb-password`,
+        );
+        shellExec(`kubectl delete statefulset mongodb`);
+        shellExec(`kubectl apply -k ./manifests/mongodb`);
+
+        await new Promise(async (resolve) => {
+          cliSpinner(3000, `[cluster.js] `, ` Load mongodb instance`, 'yellow', 'material');
+          await timer(3000);
+
+          const monitor = async () => {
+            cliSpinner(1000, `[cluster.js] `, ` Load mongodb instance`, 'yellow', 'material');
+            await timer(1000);
+            if (
+              shellExec(`kubectl get pods --all-namespaces -o wide`, {
+                silent: true,
+                stdout: true,
+                disableLog: true,
+              }).match(`mongodb-1                                    1/1     Running`)
+            )
+              return resolve();
+            return monitor();
+          };
+          await monitor();
+        });
+
+        const mongoConfig = {
+          _id: 'rs0',
+          members: [
+            { _id: 0, host: 'mongodb-0.mongodb-service:27017', priority: 1 },
+            { _id: 1, host: 'mongodb-1.mongodb-service:27017', priority: 1 },
+          ],
+        };
+
+        shellExec(
+          `sudo kubectl exec -i mongodb-0 -- mongosh --quiet --json=relaxed \
+          --eval 'use admin' \
+          --eval 'rs.initiate(${JSON.stringify(mongoConfig)})' \
+          --eval 'rs.status()'`,
+        );
+      }
+
+      if (options.full || options.contour)
+        shellExec(`kubectl apply -f https://projectcontour.io/quickstart/contour.yaml`);
+    },
+    reset() {
+      shellExec(`kind get clusters | xargs -t -n1 kind delete cluster --name`);
+      shellExec(`sudo kubeadm reset -f`);
+      shellExec('sudo rm -f /etc/cni/net.d/10-flannel.conflist');
+      shellExec('sudo iptables -F && sudo iptables -t nat -F && sudo iptables -t mangle -F && sudo iptables -X');
+      shellExec('sudo rm -f $HOME/.kube/config');
+      shellExec('sudo rm -rf /root/.local/share/Trash/files/*');
+      shellExec('sudo docker system prune -a -f');
+      shellExec('sudo service docker stop');
+      shellExec(`sudo rm -rf /var/lib/containers/storage/*`);
+      shellExec(`sudo rm -rf /var/lib/docker/volumes/*`);
+      shellExec(`sudo rm -rf /var/lib/docker~/*`);
+      shellExec(`sudo rm -rf /home/containers/storage/*`);
+      shellExec(`sudo rm -rf /home/docker/*`);
+      shellExec('sudo mv /var/lib/docker /var/lib/docker~');
+      shellExec('sudo mkdir /home/docker');
+      shellExec('sudo chmod 0711 /home/docker');
+      shellExec('sudo ln -s /home/docker /var/lib/docker');
+      shellExec(`sudo podman system prune -a -f`);
+      shellExec(`sudo podman system prune --all --volumes --force`);
+      shellExec(`sudo podman system prune --external --force`);
+      shellExec(`sudo podman system prune --all --volumes --force`);
+      shellExec(`sudo mkdir -p /home/containers/storage`);
+      shellExec('sudo chmod 0711 /home/containers/storage');
+      shellExec(
+        `sudo sed -i -e "s@/var/lib/containers/storage@/home/containers/storage@g" /etc/containers/storage.conf`,
+      );
+      shellExec(`sudo podman system reset -f`);
+    },
+  };
+}
+export default UnderpostCluster;

package/src/cli/cron.js

@@ -0,0 +1,90 @@
+/**
+ * UnderpostCron CLI index module
+ * @module src/cli/cron.js
+ * @namespace UnderpostCron
+ */
+
+import Underpost from '../index.js';
+import BackUp from '../server/backup.js';
+import { Cmd } from '../server/conf.js';
+import Dns from '../server/dns.js';
+import { netWorkCron, saveRuntimeCron } from '../server/network.js';
+import { shellExec } from '../server/process.js';
+import fs from 'fs-extra';
+
+/**
+ * UnderpostCron main module methods
+ * @class
+ * @memberof UnderpostCron
+ */
+class UnderpostCron {
+  static JOB = {
+    /**
+     * DNS cli API
+     * @static
+     * @type {Dns}
+     * @memberof UnderpostCron
+     */
+    dns: Dns,
+    /**
+     * BackUp cli API
+     * @static
+     * @type {BackUp}
+     * @memberof UnderpostCron
+     */
+    backup: BackUp,
+  };
+  static API = {
+    /**
+     * Run the cron jobs
+     * @static
+     * @param {String} deployList - Comma separated deploy ids
+     * @param {String} jobList - Comma separated job ids
+     * @return {void}
+     * @memberof UnderpostCron
+     */
+    callback: async function (
+      deployList = 'default',
+      jobList = Object.keys(UnderpostCron.JOB),
+      options = { disableKindCluster: false, init: false },
+    ) {
+      if (options.init === true) {
+        await Underpost.test.setUpInfo();
+        const jobDeployId = fs.readFileSync('./engine-private/deploy/dd.cron', 'utf8').trim();
+        deployList = fs.readFileSync('./engine-private/deploy/dd.router', 'utf8').trim();
+        const confCronConfig = JSON.parse(fs.readFileSync(`./engine-private/conf/${jobDeployId}/conf.cron.json`));
+        if (confCronConfig.jobs && Object.keys(confCronConfig.jobs).length > 0) {
+          for (const job of Object.keys(confCronConfig.jobs)) {
+            const name = `${jobDeployId}-${job}`;
+            let deployId;
+            shellExec(Cmd.delete(name));
+            switch (job) {
+              case 'dns':
+                deployId = jobDeployId;
+                break;
+
+              default:
+                deployId = deployList;
+                break;
+            }
+            shellExec(Cmd.cron(deployId, job, name, confCronConfig.jobs[job].expression, options));
+            netWorkCron.push({
+              deployId,
+              jobId: job,
+              expression: confCronConfig.jobs[job].expression,
+            });
+          }
+        }
+        await saveRuntimeCron();
+        if (fs.existsSync(`./tmp/await-deploy`)) fs.remove(`./tmp/await-deploy`);
+        return;
+      }
+      for (const _jobId of jobList.split(',')) {
+        const jobId = _jobId.trim();
+        if (UnderpostCron.JOB[jobId]) await UnderpostCron.JOB[jobId].callback(deployList, options);
+      }
+    },
+  };
+}
+
+export default UnderpostCron;

package/src/cli/db.js

@@ -0,0 +1,148 @@
+import { mergeFile, splitFileFactory } from '../server/conf.js';
+import { loggerFactory } from '../server/logger.js';
+import { shellExec } from '../server/process.js';
+import fs from 'fs-extra';
+
+const logger = loggerFactory(import.meta);
+
+class UnderpostDB {
+  static API = {
+    async callback(deployList = 'default', options = { import: false, export: false }) {
+      const newBackupTimestamp = new Date().getTime();
+      const nameSpace = 'default';
+      for (const _deployId of deployList.split(',')) {
+        const deployId = _deployId.trim();
+        if (!deployId) continue;
+        const dbs = {};
+        const repoName = `engine-${deployId.split('dd-')[1]}-cron-backups`;
+
+        const confServer = JSON.parse(fs.readFileSync(`./engine-private/conf/${deployId}/conf.server.json`, 'utf8'));
+        for (const host of Object.keys(confServer)) {
+          for (const path of Object.keys(confServer[host])) {
+            const { db } = confServer[host][path];
+            if (db) {
+              const { provider, name, user, password } = db;
+              if (!dbs[provider]) dbs[provider] = {};
+
+              if (!(name in dbs[provider]))
+                dbs[provider][name] = { user, password, hostFolder: host + path.replaceAll('/', '-') };
+            }
+          }
+        }
+
+        if (!fs.existsSync(`../${repoName}`)) {
+          shellExec(`cd .. && underpost clone ${process.env.GITHUB_USERNAME}/${repoName}`);
+        } else {
+          shellExec(`cd ../${repoName} && underpost pull . ${process.env.GITHUB_USERNAME}/${repoName}`);
+        }
+
+        for (const provider of Object.keys(dbs)) {
+          for (const dbName of Object.keys(dbs[provider])) {
+            const { hostFolder, user, password } = dbs[provider][dbName];
+            if (hostFolder) {
+              logger.info('', { hostFolder, provider, dbName });
+
+              const backUpPath = `../${repoName}/${hostFolder}`;
+              const times = await fs.readdir(backUpPath);
+              const currentBackupTimestamp = Math.max(...times.map((t) => parseInt(t)));
+              dbs[provider][dbName].currentBackupTimestamp = currentBackupTimestamp;
+              const removeBackupTimestamp = Math.min(...times.map((t) => parseInt(t)));
+
+              const sqlContainerPath = `/home/${dbName}.sql`;
+              const _fromPartsParts = `../${repoName}/${hostFolder}/${currentBackupTimestamp}/${dbName}-parths.json`;
+              const _toSqlPath = `../${repoName}/${hostFolder}/${currentBackupTimestamp}/${dbName}.sql`;
+              const _toNewSqlPath = `../${repoName}/${hostFolder}/${newBackupTimestamp}/${dbName}.sql`;
+              const _toBsonPath = `../${repoName}/${hostFolder}/${currentBackupTimestamp}/${dbName}`;
+              const _toNewBsonPath = `../${repoName}/${hostFolder}/${newBackupTimestamp}/${dbName}`;
+
+              if (options.import === true && fs.existsSync(_fromPartsParts) && !fs.existsSync(_toSqlPath)) {
+                const names = JSON.parse(fs.readFileSync(_fromPartsParts, 'utf8')).map((_path) => {
+                  return `../${repoName}/${hostFolder}/${currentBackupTimestamp}/${_path.split('/').pop()}`;
+                });
+                logger.info('merge Back Up paths', {
+                  _fromPartsParts,
+                  _toSqlPath,
+                  names,
+                });
+                await mergeFile(names, _toSqlPath);
+              }
+
+              if (options.export === true && times.length >= 5) {
+                fs.removeSync(`../${repoName}/${hostFolder}/${removeBackupTimestamp}`);
+                fs.mkdirSync(`../${repoName}/${hostFolder}/${newBackupTimestamp}`, { recursive: true });
+              }
+
+              switch (provider) {
+                case 'mariadb': {
+                  const podName = `mariadb-statefulset-0`;
+                  const serviceName = 'mariadb';
+                  if (options.import === true) {
+                    shellExec(`sudo kubectl cp ${_toSqlPath} ${nameSpace}/${podName}:/${dbName}.sql`);
+                    const cmd = `mariadb -u ${user} -p${password} ${dbName} < /${dbName}.sql`;
+                    shellExec(
+                      `kubectl exec -i ${podName} -- ${serviceName} -p${password} -e 'CREATE DATABASE ${dbName};'`,
+                    );
+                    shellExec(`sudo kubectl exec -i ${podName} -- sh -c "${cmd}"`);
+                  }
+                  if (options.export === true) {
+                    const cmd = `mariadb-dump --user=${user} --password=${password} --lock-tables ${dbName} > ${sqlContainerPath}`;
+                    shellExec(`sudo kubectl exec -i ${podName} -- sh -c "${cmd}"`);
+                    shellExec(`sudo kubectl cp ${nameSpace}/${podName}:${sqlContainerPath} ${_toNewSqlPath}`);
+                    await splitFileFactory(dbName, _toNewSqlPath);
+                  }
+                  break;
+                }
+
+                case 'mongoose': {
+                  if (options.import === true) {
+                    const podName = `mongodb-0`;
+                    shellExec(`sudo kubectl cp ${_toBsonPath} ${nameSpace}/${podName}:/${dbName}`);
+                    const cmd = `mongorestore -d ${dbName} /${dbName}`;
+                    shellExec(`sudo kubectl exec -i ${podName} -- sh -c "${cmd}"`);
+                  }
+                  if (options.export === true) {
+                    const podName = `backup-access`;
+                    const containerBaseBackupPath = '/backup';
+                    let timeFolder = shellExec(
+                      `sudo kubectl exec -i ${podName} -- sh -c "cd ${containerBaseBackupPath} && ls -a"`,
+                      {
+                        stdout: true,
+                        disableLog: false,
+                        silent: true,
+                      },
+                    ).split(`\n`);
+                    timeFolder = timeFolder[timeFolder.length - 2];
+                    if (timeFolder === '..') {
+                      logger.warn(`Cannot backup available`, { timeFolder });
+                    } else {
+                      shellExec(
+                        `sudo kubectl cp ${nameSpace}/${podName}:${containerBaseBackupPath}/${timeFolder}/${dbName} ${_toNewBsonPath}`,
+                      );
+                    }
+                  }
+                  break;
+                }
+
+                default:
+                  break;
+              }
+            }
+          }
+        }
+        if (options.export === true) {
+          shellExec(`cd ../${repoName} && git add .`);
+          shellExec(
+            `underpost cmt ../${repoName} backup '' '${new Date(newBackupTimestamp).toLocaleDateString()} ${new Date(
+              newBackupTimestamp,
+            ).toLocaleTimeString()}'`,
+          );
+          shellExec(`cd ../${repoName} && underpost push . ${process.env.GITHUB_USERNAME}/${repoName}`, {
+            disableLog: true,
+          });
+        }
+      }
+    },
+  };
+}
+
+export default UnderpostDB;