@underpostnet/underpost 2.8.4 → 2.8.6

package/src/cli/db.js

@@ -0,0 +1,212 @@
+import { mergeFile, splitFileFactory } from '../server/conf.js';
+import { loggerFactory } from '../server/logger.js';
+import { shellExec } from '../server/process.js';
+import fs from 'fs-extra';
+import UnderpostDeploy from './deploy.js';
+
+const logger = loggerFactory(import.meta);
+
+class UnderpostDB {
+  static API = {
+    async callback(
+      deployList = 'default',
+      options = {
+        import: false,
+        export: false,
+        podName: false,
+        ns: false,
+        collection: '',
+        outPath: '',
+        drop: false,
+        preserveUUID: false,
+        git: false,
+      },
+    ) {
+      const newBackupTimestamp = new Date().getTime();
+      const nameSpace = options.ns && typeof options.ns === 'string' ? options.ns : 'default';
+      for (const _deployId of deployList.split(',')) {
+        const deployId = _deployId.trim();
+        if (!deployId) continue;
+        const dbs = {};
+        const repoName = `engine-${deployId.split('dd-')[1]}-cron-backups`;
+
+        const confServer = JSON.parse(fs.readFileSync(`./engine-private/conf/${deployId}/conf.server.json`, 'utf8'));
+        for (const host of Object.keys(confServer)) {
+          for (const path of Object.keys(confServer[host])) {
+            const { db } = confServer[host][path];
+            if (db) {
+              const { provider, name, user, password } = db;
+              if (!dbs[provider]) dbs[provider] = {};
+
+              if (!(name in dbs[provider]))
+                dbs[provider][name] = { user, password, hostFolder: host + path.replaceAll('/', '-') };
+            }
+          }
+        }
+
+        if (!fs.existsSync(`../${repoName}`)) {
+          shellExec(`cd .. && underpost clone ${process.env.GITHUB_USERNAME}/${repoName}`);
+        } else {
+          shellExec(`cd ../${repoName} && underpost pull . ${process.env.GITHUB_USERNAME}/${repoName}`);
+        }
+
+        for (const provider of Object.keys(dbs)) {
+          for (const dbName of Object.keys(dbs[provider])) {
+            const { hostFolder, user, password } = dbs[provider][dbName];
+            if (hostFolder) {
+              logger.info('', { hostFolder, provider, dbName });
+
+              const backUpPath = `../${repoName}/${hostFolder}`;
+              if (!fs.existsSync(backUpPath)) fs.mkdirSync(backUpPath, { recursive: true });
+              shellExec(`cd ${backUpPath} && find . -type d -empty -delete`); // delete empty folders
+              const times = await fs.readdir(backUpPath);
+              const currentBackupTimestamp = Math.max(...times.map((t) => parseInt(t)).filter((t) => !isNaN(t)));
+              dbs[provider][dbName].currentBackupTimestamp = currentBackupTimestamp;
+              const removeBackupTimestamp = Math.min(...times.map((t) => parseInt(t)).filter((t) => !isNaN(t)));
+
+              const sqlContainerPath = `/home/${dbName}.sql`;
+              const _fromPartsParts = `../${repoName}/${hostFolder}/${currentBackupTimestamp}/${dbName}-parths.json`;
+              const _toSqlPath = `../${repoName}/${hostFolder}/${currentBackupTimestamp}/${dbName}.sql`;
+              const _toNewSqlPath = `../${repoName}/${hostFolder}/${newBackupTimestamp}/${dbName}.sql`;
+              const _toBsonPath = `../${repoName}/${hostFolder}/${currentBackupTimestamp}/${dbName}`;
+              const _toNewBsonPath = `../${repoName}/${hostFolder}/${newBackupTimestamp}/${dbName}`;
+
+              if (options.import === true && fs.existsSync(_fromPartsParts) && !fs.existsSync(_toSqlPath)) {
+                const names = JSON.parse(fs.readFileSync(_fromPartsParts, 'utf8')).map((_path) => {
+                  return `../${repoName}/${hostFolder}/${currentBackupTimestamp}/${_path.split('/').pop()}`;
+                });
+                logger.info('merge Back Up paths', {
+                  _fromPartsParts,
+                  _toSqlPath,
+                  names,
+                });
+                await mergeFile(names, _toSqlPath);
+              }
+
+              if (options.export === true && times.length >= 5) {
+                logger.info('remove', `../${repoName}/${hostFolder}/${removeBackupTimestamp}`);
+                fs.removeSync(`../${repoName}/${hostFolder}/${removeBackupTimestamp}`);
+                logger.info('create', `../${repoName}/${hostFolder}/${newBackupTimestamp}`);
+                fs.mkdirSync(`../${repoName}/${hostFolder}/${newBackupTimestamp}`, { recursive: true });
+              }
+
+              switch (provider) {
+                case 'mariadb': {
+                  const podNames =
+                    options.podName && typeof options.podName === 'string'
+                      ? options.podName.split(',')
+                      : UnderpostDeploy.API.get('mariadb'); // `mariadb-statefulset-0`;
+                  const serviceName = 'mariadb';
+                  for (const podNameData of [podNames[0]]) {
+                    const podName = podNameData.NAME;
+                    if (options.import === true) {
+                      shellExec(`sudo kubectl exec -i ${podName} -- sh -c "rm -rf /${dbName}.sql"`);
+                      shellExec(`sudo kubectl cp ${_toSqlPath} ${nameSpace}/${podName}:/${dbName}.sql`);
+                      const cmd = `mariadb -u ${user} -p${password} ${dbName} < /${dbName}.sql`;
+                      shellExec(
+                        `kubectl exec -i ${podName} -- ${serviceName} -p${password} -e 'CREATE DATABASE ${dbName};'`,
+                      );
+                      shellExec(`sudo kubectl exec -i ${podName} -- sh -c "${cmd}"`);
+                    }
+                    if (options.export === true) {
+                      shellExec(`sudo kubectl exec -i ${podName} -- sh -c "rm -rf ${sqlContainerPath}"`);
+                      const cmd = `mariadb-dump --user=${user} --password=${password} --lock-tables ${dbName} > ${sqlContainerPath}`;
+                      shellExec(`sudo kubectl exec -i ${podName} -- sh -c "${cmd}"`);
+                      shellExec(
+                        `sudo kubectl cp ${nameSpace}/${podName}:${sqlContainerPath} ${
+                          options.outPath ? options.outPath : _toNewSqlPath
+                        }`,
+                      );
+                      await splitFileFactory(dbName, options.outPath ? options.outPath : _toNewSqlPath);
+                    }
+                  }
+                  break;
+                }
+
+                case 'mongoose': {
+                  if (options.import === true) {
+                    const podNames =
+                      options.podName && typeof options.podName === 'string'
+                        ? options.podName.split(',')
+                        : UnderpostDeploy.API.get('mongo');
+                    // `mongodb-0`;
+                    for (const podNameData of [podNames[0]]) {
+                      const podName = podNameData.NAME;
+                      shellExec(`sudo kubectl exec -i ${podName} -- sh -c "rm -rf /${dbName}"`);
+                      shellExec(
+                        `sudo kubectl cp ${
+                          options.outPath ? options.outPath : _toBsonPath
+                        } ${nameSpace}/${podName}:/${dbName}`,
+                      );
+                      const cmd = `mongorestore -d ${dbName} /${dbName}${options.drop ? ' --drop' : ''}${
+                        options.preserveUUID ? ' --preserveUUID' : ''
+                      }`;
+                      shellExec(`sudo kubectl exec -i ${podName} -- sh -c "${cmd}"`);
+                    }
+                  }
+                  if (options.export === true) {
+                    const podNames =
+                      options.podName && typeof options.podName === 'string'
+                        ? options.podName.split(',')
+                        : UnderpostDeploy.API.get('mongo'); // `backup-access`;
+                    for (const podNameData of [podNames[0]]) {
+                      const podName = podNameData.NAME;
+                      shellExec(`sudo kubectl exec -i ${podName} -- sh -c "rm -rf /${dbName}"`);
+                      if (options.collections)
+                        for (const collection of options.collections)
+                          shellExec(
+                            `sudo kubectl exec -i ${podName} -- sh -c "mongodump -d ${dbName} --collection ${collection} -o /${dbName}"`,
+                          );
+                      else shellExec(`sudo kubectl exec -i ${podName} -- sh -c "mongodump -d ${dbName} -o /${dbName}"`);
+                      shellExec(
+                        `sudo kubectl cp ${nameSpace}/${podName}:/${dbName} ${
+                          options.outPath ? options.outPath : _toNewBsonPath
+                        }`,
+                      );
+                    }
+                    if (false) {
+                      const containerBaseBackupPath = '/backup';
+                      let timeFolder = shellExec(
+                        `sudo kubectl exec -i ${podName} -- sh -c "cd ${containerBaseBackupPath} && ls -a"`,
+                        {
+                          stdout: true,
+                          disableLog: false,
+                          silent: true,
+                        },
+                      ).split(`\n`);
+                      timeFolder = timeFolder[timeFolder.length - 2];
+                      if (timeFolder === '..') {
+                        logger.warn(`Cannot backup available`, { timeFolder });
+                      } else {
+                        shellExec(
+                          `sudo kubectl cp ${nameSpace}/${podName}:${containerBaseBackupPath}/${timeFolder}/${dbName} ${_toNewBsonPath}`,
+                        );
+                      }
+                    }
+                  }
+                  break;
+                }
+
+                default:
+                  break;
+              }
+            }
+          }
+        }
+        if (options.export === true && options.git === true) {
+          shellExec(`cd ../${repoName} && git add .`);
+          shellExec(
+            `underpost cmt ../${repoName} backup '' '${new Date(newBackupTimestamp).toLocaleDateString()} ${new Date(
+              newBackupTimestamp,
+            ).toLocaleTimeString()}'`,
+          );
+          shellExec(`cd ../${repoName} && underpost push . ${process.env.GITHUB_USERNAME}/${repoName}`, {
+            disableLog: true,
+          });
+        }
+      }
+    },
+  };
+}
+
+export default UnderpostDB;

package/src/cli/deploy.js

@@ -0,0 +1,318 @@
+import {
+  buildKindPorts,
+  buildPortProxyRouter,
+  buildProxyRouter,
+  Config,
+  getDataDeploy,
+  loadReplicas,
+} from '../server/conf.js';
+import { loggerFactory } from '../server/logger.js';
+import { shellExec } from '../server/process.js';
+import fs from 'fs-extra';
+import dotenv from 'dotenv';
+import Underpost from '../index.js';
+
+const logger = loggerFactory(import.meta);
+
+class UnderpostDeploy {
+  static API = {
+    sync(deployList) {
+      const deployGroupId = 'dd.tmp';
+      fs.writeFileSync(`./engine-private/deploy/${deployGroupId}`, deployList, 'utf8');
+      return getDataDeploy({
+        buildSingleReplica: true,
+        deployGroupId,
+      });
+    },
+    async routerFactory(deployList, env) {
+      const initEnvPath = `./engine-private/conf/${deployList.split(',')[0]}/.env.${env}`;
+      const initEnvObj = dotenv.parse(fs.readFileSync(initEnvPath, 'utf8'));
+      process.env.PORT = initEnvObj.PORT;
+      process.env.NODE_ENV = env;
+      await Config.build(undefined, 'proxy', deployList);
+      return buildPortProxyRouter(env === 'development' ? 80 : 443, buildProxyRouter());
+    },
+    async buildManifest(deployList, env, version) {
+      for (const _deployId of deployList.split(',')) {
+        const deployId = _deployId.trim();
+        if (!deployId) continue;
+
+        const router = await UnderpostDeploy.API.routerFactory(deployId, env);
+        const ports = Object.values(router).map((p) => parseInt(p.split(':')[2]));
+        const fromPort = Math.min(...ports);
+        const toPort = Math.max(...ports);
+        const confServer = loadReplicas(
+          JSON.parse(fs.readFileSync(`./engine-private/conf/${deployId}/conf.server.json`, 'utf8')),
+          'proxy',
+        );
+
+        fs.mkdirSync(`./engine-private/conf/${deployId}/build/${env}`, { recursive: true });
+        if (env === 'development') fs.mkdirSync(`./manifests/deployment/${deployId}-${env}`, { recursive: true });
+
+        logger.info('port range', { deployId, fromPort, toPort });
+
+        const deploymentYamlParts = `apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ${deployId}-${env}
+  labels:
+    app: ${deployId}-${env}
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: ${deployId}-${env}
+  template:
+    metadata:
+      labels:
+        app: ${deployId}-${env}
+    spec:
+      containers:
+        - name: ${deployId}-${env}
+          image: localhost/underpost-engine:${version && typeof version === 'string' ? version : Underpost.version}
+          lifecycle:
+            postStart:
+              exec:
+                command:
+                  - /bin/sh
+                  - -c
+                  - >
+                    sleep 60 &&
+                    underpost config set deploy-id ${deployId} &&
+                    underpost config set deploy-env ${env}
+# image: localhost/${deployId}-${env}:${version && typeof version === 'string' ? version : Underpost.version}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: ${deployId}-${env}-service
+spec:
+  selector:
+    app: ${deployId}-${env}
+  ports:
+  type: LoadBalancer`.split('ports:');
+        deploymentYamlParts[1] =
+          buildKindPorts(fromPort, toPort) +
+          `  type: LoadBalancer
+`;
+
+        fs.writeFileSync(
+          `./engine-private/conf/${deployId}/build/${env}/deployment.yaml`,
+          deploymentYamlParts.join(`ports:
+`),
+        );
+
+        let proxyYaml = '';
+        let secretYaml = '';
+
+        for (const host of Object.keys(confServer)) {
+          if (env === 'production')
+            secretYaml += `
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: ${host}
+spec:
+  commonName: ${host}
+  dnsNames:
+    - ${host}
+  issuerRef:
+    name: letsencrypt-prod
+    kind: ClusterIssuer
+  secretName: ${host}`;
+
+          const pathPortConditions = [];
+          for (const path of Object.keys(confServer[host])) {
+            const { peer } = confServer[host][path];
+            if (!router[`${host}${path === '/' ? '' : path}`]) continue;
+            const port = parseInt(router[`${host}${path === '/' ? '' : path}`].split(':')[2]);
+            // logger.info('', { host, port, path });
+            pathPortConditions.push({
+              port,
+              path,
+            });
+
+            if (peer) {
+              //  logger.info('', { host, port: port + 1, path: '/peer' });
+              pathPortConditions.push({
+                port: port + 1,
+                path: '/peer',
+              });
+            }
+          }
+
+          // logger.info('', { host, pathPortConditions });
+          proxyYaml += `
+---
+apiVersion: projectcontour.io/v1
+kind: HTTPProxy
+metadata:
+  name: ${host}
+spec:
+  virtualhost:
+    fqdn: ${host}${
+            env === 'development'
+              ? ''
+              : `
+    tls:
+      secretName: ${host}`
+          }
+  routes:`;
+          for (const conditionObj of pathPortConditions) {
+            const { path, port } = conditionObj;
+            proxyYaml += `
+    - conditions:
+        - prefix: ${path}
+      enableWebsockets: true
+      services:
+        - name: ${deployId}-${env}-service
+          port: ${port}`;
+          }
+        }
+        const yamlPath = `./engine-private/conf/${deployId}/build/${env}/proxy.yaml`;
+        fs.writeFileSync(yamlPath, proxyYaml, 'utf8');
+        if (env === 'production') {
+          const yamlPath = `./engine-private/conf/${deployId}/build/${env}/secret.yaml`;
+          fs.writeFileSync(yamlPath, secretYaml, 'utf8');
+        } else {
+          const deploymentsFiles = ['Dockerfile', 'proxy.yaml', 'deployment.yaml'];
+          for (const file of deploymentsFiles) {
+            if (fs.existsSync(`./engine-private/conf/${deployId}/build/${env}/${file}`)) {
+              fs.copyFileSync(
+                `./engine-private/conf/${deployId}/build/${env}/${file}`,
+                `./manifests/deployment/${deployId}-${env}/${file}`,
+              );
+            }
+          }
+        }
+      }
+    },
+    async callback(
+      deployList = 'default',
+      env = 'development',
+      options = {
+        remove: false,
+        infoRouter: false,
+        sync: false,
+        buildManifest: false,
+        infoUtil: false,
+        expose: false,
+        cert: false,
+        version: '',
+      },
+    ) {
+      if (options.infoUtil === true)
+        return logger.info(`
+kubectl rollout restart deployment/deployment-name
+kubectl rollout undo deployment/deployment-name
+kubectl scale statefulsets <stateful-set-name> --replicas=<new-replicas>
+        `);
+      if (deployList === 'dd' && fs.existsSync(`./engine-private/deploy/dd.router`))
+        deployList = fs.readFileSync(`./engine-private/deploy/dd.router`, 'utf8');
+      if (options.sync) UnderpostDeploy.API.sync(deployList);
+      if (options.buildManifest === true) await UnderpostDeploy.API.buildManifest(deployList, env, options.version);
+      if (options.infoRouter === true)
+        return logger.info('router', await UnderpostDeploy.API.routerFactory(deployList, env));
+      const etcHost = (
+        concat,
+      ) => `127.0.0.1  ${concat} localhost localhost.localdomain localhost4 localhost4.localdomain4
+::1         localhost localhost.localdomain localhost6 localhost6.localdomain6`;
+      let concatHots = '';
+
+      for (const _deployId of deployList.split(',')) {
+        const deployId = _deployId.trim();
+        if (!deployId) continue;
+        if (options.expose === true) {
+          const svc = UnderpostDeploy.API.get(deployId, 'svc')[0];
+          const port = parseInt(svc[`PORT(S)`].split('/TCP')[0]);
+          logger.info(deployId, {
+            svc,
+            port,
+          });
+          shellExec(`sudo kubectl port-forward -n default svc/${svc.NAME} ${port}:${port}`, { async: true });
+          continue;
+        }
+        shellExec(`sudo kubectl delete svc ${deployId}-${env}-service`);
+        shellExec(`sudo kubectl delete deployment ${deployId}-${env}`);
+
+        const confServer = JSON.parse(fs.readFileSync(`./engine-private/conf/${deployId}/conf.server.json`, 'utf8'));
+        for (const host of Object.keys(confServer)) {
+          shellExec(`sudo kubectl delete HTTPProxy ${host}`);
+          if (env === 'production' && options.cert === true) shellExec(`sudo kubectl delete Certificate ${host}`);
+          if (!options.remove === true && env === 'development') concatHots += ` ${host}`;
+        }
+
+        const manifestsPath =
+          env === 'production'
+            ? `engine-private/conf/${deployId}/build/production`
+            : `manifests/deployment/${deployId}-${env}`;
+
+        if (!options.remove === true) {
+          shellExec(`sudo kubectl apply -f ./${manifestsPath}/deployment.yaml`);
+          shellExec(`sudo kubectl apply -f ./${manifestsPath}/proxy.yaml`);
+          if (env === 'production' && options.cert === true)
+            shellExec(`sudo kubectl apply -f ./${manifestsPath}/secret.yaml`);
+        }
+      }
+      let renderHosts;
+      switch (process.platform) {
+        case 'linux':
+          {
+            switch (env) {
+              case 'development':
+                renderHosts = etcHost(concatHots);
+                fs.writeFileSync(`/etc/hosts`, renderHosts, 'utf8');
+
+                break;
+
+              default:
+                break;
+            }
+          }
+          break;
+
+        default:
+          break;
+      }
+      if (renderHosts)
+        logger.info(
+          `
+` + renderHosts,
+        );
+    },
+    get(deployId, kindType = 'pods') {
+      const raw = shellExec(`sudo kubectl get ${kindType} --all-namespaces -o wide`, {
+        stdout: true,
+        disableLog: true,
+        silent: true,
+      });
+
+      const heads = raw
+        .split(`\n`)[0]
+        .split(' ')
+        .filter((_r) => _r.trim());
+
+      const pods = raw
+        .split(`\n`)
+        .filter((r) => (deployId ? r.match(deployId) : r.trim() && !r.match('NAME')))
+        .map((r) => r.split(' ').filter((_r) => _r.trim()));
+
+      const result = [];
+
+      for (const row of pods) {
+        const pod = {};
+        let index = -1;
+        for (const head of heads) {
+          index++;
+          pod[head] = row[index];
+        }
+        result.push(pod);
+      }
+
+      return result;
+    },
+  };
+}
+
+export default UnderpostDeploy;

package/src/cli/env.js

@@ -0,0 +1,52 @@
+import { getNpmRootPath, writeEnv } from '../server/conf.js';
+import fs from 'fs-extra';
+import { loggerFactory } from '../server/logger.js';
+import dotenv from 'dotenv';
+
+dotenv.config();
+
+const logger = loggerFactory(import.meta);
+
+class UnderpostRootEnv {
+  static API = {
+    set(key, value) {
+      const exeRootPath = `${getNpmRootPath()}/underpost`;
+      const envPath = `${exeRootPath}/.env`;
+      let env = {};
+      if (fs.existsSync(envPath)) env = dotenv.parse(fs.readFileSync(envPath, 'utf8'));
+      env[key] = value;
+      writeEnv(envPath, env);
+    },
+    delete(key) {
+      const exeRootPath = `${getNpmRootPath()}/underpost`;
+      const envPath = `${exeRootPath}/.env`;
+      let env = {};
+      if (fs.existsSync(envPath)) env = dotenv.parse(fs.readFileSync(envPath, 'utf8'));
+      delete env[key];
+      writeEnv(envPath, env);
+    },
+    get(key) {
+      const exeRootPath = `${getNpmRootPath()}/underpost`;
+      const envPath = `${exeRootPath}/.env`;
+      if (!fs.existsSync(envPath)) return logger.error(`Unable to find underpost root environment`);
+      const env = dotenv.parse(fs.readFileSync(envPath, 'utf8'));
+      logger.info('underpost root', { [key]: env[key] });
+      return env[key];
+    },
+    list() {
+      const exeRootPath = `${getNpmRootPath()}/underpost`;
+      const envPath = `${exeRootPath}/.env`;
+      if (!fs.existsSync(envPath)) return logger.error(`Unable to find underpost root environment`);
+      const env = dotenv.parse(fs.readFileSync(envPath, 'utf8'));
+      logger.info('underpost root', env);
+      return env;
+    },
+    clean() {
+      const exeRootPath = `${getNpmRootPath()}/underpost`;
+      const envPath = `${exeRootPath}/.env`;
+      fs.removeSync(envPath);
+    },
+  };
+}
+
+export default UnderpostRootEnv;