@drumee/setup-infra 1.0.13 → 1.0.15

package/index.js

@@ -1,37 +1,26 @@
 #!/usr/bin/env node
 
-// ======================================================
-//
-// ======================================================
 const Template = require("./templates");
-const { writeFileSync } = require(`jsonfile`);
+const { writeFileSync, readFileSync: readJson } = require(`jsonfile`);
 const { join, dirname } = require("path");
 const { isString } = require("lodash");
 const { exit } = process;
-const { sysEnv, uniqueId } = require("@drumee/server-essentials");
-const { totalmem } = require('os');
-const ARGV = require('minimist')(process.argv.slice(2));
+const { loadSysEnv, sysEnv, uniqueId } = require("@drumee/server-essentials");
+const { totalmem, userInfo } = require('os');
 const {
   existsSync, close, writeSync, openSync, readFileSync, mkdirSync
 } = require("fs");
+const { args, hasExistingSettings } = require('./templates/utils')
+
 
 const JSON_OPT = { spaces: 2, EOL: "\r\n" };
 
 const {
   ACME_DIR,
   ACME_EMAIL_ACCOUNT,
-  ADMIN_EMAIL,
-  DB_BACKUP,
-  DRUMEE_DB_DIR,
-  DRUMEE_DESCRIPTION,
-  DRUMEE_DOMAIN_NAME,
-  FORCE_INSTALL,
+  CERTS_DIR,
   MAIL_USER,
-  MAX_BODY_SIZE,
   NSUPDATE_KEY,
-  PUBLIC_IP4,
-  PUBLIC_IP6,
-  STORAGE_BACKUP,
 } = process.env;
 
 /**
@@ -69,8 +58,7 @@ function copyFields(data, keys) {
  */
 function factory(data) {
   let route = "main";
-  let mode = "dist";
-  let base = `${data.server_dir}/${mode}/${route}/`;
+  let base = `${data.server_dir}/${route}/`;
   return {
     name: "factory",
     script: `./index.js`,
@@ -109,11 +97,12 @@ function worker(data, instances = 1, exec_mode = 'fork_mode') {
     name,
     server_dir,
     runtime_dir,
-    mode,
   } = data;
+
   if (!server_dir) server_dir = join(runtime_dir, 'server');
-  let base = `${server_dir}/${mode}/${route}`;
-  return {
+  let base = `${server_dir}/${route}`;
+  let iname = name.replace('/', '-');
+  let opt = {
     name,
     script,
     cwd: base,
@@ -126,24 +115,61 @@ function worker(data, instances = 1, exec_mode = 'fork_mode') {
     },
     dependencies: [`pm2-logrotate`],
     exec_mode,
-    instances
+    instances,
+    out_file: join(data.log_dir, `log-${iname}.log`),
+    error_file: join(data.log_dir, `error-${iname}.log`),
+    pm2_log_routes: {
+      rotateInterval: '0 0 * * *', // Rotate daily at midnight
+      rotateModule: true,
+      max_size: '10M', // Rotate when log reaches 10MB
+      retain: 30 // Keep 30 rotated logs
+    }
   };
+  if (args.watch_dirs) {
+    let dirs = args.watch_dirs.split(/,+/);
+    if (dirs.length) {
+      opt.watch = dirs;
+      opt.watch_delay = args.watch_delay;
+      if (args.watch_symlinks) {
+        opt.watch_options = {
+          followSymlinks: true
+        }
+      } else {
+        opt.watch_options = {
+          followSymlinks: false
+        }
+      }
+      if (args.watch_ignore) {
+        let ignored = args.watch_ignore.split(/,+/);
+        if (ignored.length) {
+          opt.ignore_watch = ignored;
+        }
+      }
+    }
+  }
+  return opt;
+
 }
 
 /***
  * 
  */
 function writeTemplates(data, targets) {
-  if (ARGV.readonly || ARGV.noCheck) {
+  if (args.readonly || args.noCheck) {
     console.log("Readonly", targets, data);
     return
   }
   for (let target of targets) {
-    if (isString(target)) {
-      Template.write(data, target, target);
-    } else {
-      let { out, tpl } = target;
-      Template.write(data, out, tpl);
+    try {
+      if (isString(target)) {
+        Template.write(data, target, target);
+      } else {
+        let { out, tpl } = target;
+        Template.write(data, out, tpl);
+      }
+    } catch (e) {
+      console.error(e)
+      console.error("Failed to write configs for", target)
     }
   }
 }
@@ -155,10 +181,8 @@ function writeEcoSystem(data) {
   const ports = {
     pushPort: 23000,
     restPort: 24000,
-    mode: "dist",
     route: "main",
   };
-
   let main = worker({
     ...data,
     ...ports,
@@ -182,11 +206,14 @@ function writeEcoSystem(data) {
 
   let f = factory(data);
   let routes = [main, main_service, f];
-  let ecosystem = "/etc/drumee/infrastructure/ecosystem.json";
-  if (ARGV.readonly) {
+  //let ecosystem = "etc/drumee/infrastructure/ecosystem.json";
+  let ecosystem = Template.chroot("etc/drumee/infrastructure/ecosystem.json");
+  if (args.readonly) {
     console.log("Readonly", ecosystem, routes);
     return
   }
+  console.log("Writing ecosystem into ", ecosystem);
+  Template.makedir(dirname(ecosystem));
   writeFileSync(ecosystem, routes, JSON_OPT);
   let targets = [
     {
@@ -212,9 +239,17 @@ function getSocketPath() {
   return socketPath;
 }
 
+
+/**
+ * 
+ * @param {*} opt 
+ * @returns 
+ */
 function makeData(opt) {
   let data = sysEnv();
-
+  if (args.env_file && existsSync(args.env_file)) {
+    loadEnvFile(args.env_file, opt)
+  }
   data.chroot = Template.chroot();
   data.acme_store = join(data.certs_dir, `${data.domain_name}_ecc`);
   data.ca_server = data.ca_server || data.acme_ssl;
@@ -223,75 +258,147 @@ function makeData(opt) {
   }
   for (let row of opt) {
     let [key, value, fallback] = row;
-    if (!value) value = fallback;
-    if (!value) continue;
-    if (!data[key]) {
+    if (!value) value = data[key] || fallback;
+    if (value == null) continue;
+    if (isString(value)) {
       if (/.+\+$/.test(value)) {
         value = value.replace(/\+$/, data[key]);
       }
       data[key] = value.trim() || fallback;
+    } else {
+      data[key] = value
     }
   }
 
+  /** Named extra settings */
+  data.allow_recursion = 'localhost;';
   if (data.public_ip4) {
     data.reverse_ip4 = data.public_ip4.split('.').reverse().join('.');
+  } else {
+    data.reverse_ip4 = ""
+  }
+
+  if (!data.public_ip6) {
+    data.public_ip6 = "";
   }
 
+  if (!data.storage_backup) {
+    data.storage_backup = ""
+  }
   return data;
 }
+
+/**
+ * 
+ * @param {*} env 
+ * @param {*} opt 
+ */
+function loadEnvFile(file, opt) {
+  let src = readJson(file);
+  opt.map((r) => {
+    let [key] = r;
+    if (src[key] != null) r[1] = src[key];
+  })
+  console.log(opt)
+}
+
 /**
  *
  */
 function getSysConfigs() {
-  let { domain_name } = sysEnv();
-  if (existsSync('/etc/drumee/drumee.sh') && !FORCE_INSTALL) {
-    console.log(
-      `There is already a domain name configured on this server (${domain_name})\n`, `Use FORCE_INSTALL=yes to override`);
+  if (hasExistingSettings(Template.chroot('etc/drumee/drumee.json'))) {
     exit(0)
   }
-  domain_name = domain_name || ARGV.domain || DRUMEE_DOMAIN_NAME;
+
+  let use_email = 0;
+  if (args.public_domain) use_email = 1;
+  let domain_name = args.public_domain || args.private_domain;
   if (!domain_name) {
-    console.log("There no domain name defined for the installation");
-    exit(0)
+    if (!args.localhost) {
+      console.log("There is no domain name defined for the installation", args);
+      exit(0)
+    }
   }
+
+  const nsupdate_key = Template.chroot('etc/bind/keys/update.key')
   const opt = [
-    ["acme_dir", ACME_DIR, '/usr/share/acme'],
-    ["jitsi_domain", `jit.+`],
-    ["domain_name", domain_name],
-    ["nsupdate_key", NSUPDATE_KEY, '/etc/bind/keys/update.key'],
-    ["domain_desc", DRUMEE_DESCRIPTION, 'My Drumee Box'],
-    ["admin_email", ADMIN_EMAIL, ''],
-    ["acme_email_account", ACME_EMAIL_ACCOUNT, ADMIN_EMAIL],
-    ["public_ip4", PUBLIC_IP4, ''],
-    ["public_ip6", PUBLIC_IP6, ''],
-    ["storage_backup", STORAGE_BACKUP, ''],
-    ["db_backup", DB_BACKUP, ''],
-    ["db_dir", DRUMEE_DB_DIR, '/srv/db'],
-    ["max_body_size", MAX_BODY_SIZE, '10G'],
+    ["nsupdate_key", NSUPDATE_KEY, nsupdate_key],
+    ["admin_email", args.admin_email],
+    ["credential_dir", Template.chroot('etc/drumee/credential')],
+    ["domain_desc", args.description, 'My Drumee Box'],
+    ["max_body_size", args.max_body_size, '10G'],
+    ["drumee_root", args.drumee_root, "/var/lib/drumee"],
+    ["use_email", use_email, 0],
+    ["db_dir", args.db_dir, '/var/lib/mysql'],
+    ["log_dir", args.log_dir, '/var/log/drumee'],
+    ["system_user", args.system_user, 'www-data'],
+    ["system_group", args.system_group, 'www-data'],
+    ["backup_storage", args.backup_storage, ""],
+    ["data_dir", args.data_dir, '/var/lib/drumee/data'],
+    ["http_port", args.http_port, 80],
+    ["https_port", args.https_port, 443],
+    ["verbosity", args.verbosity, 2],
   ]
-  let data = makeData(opt)
+
+  if (!args.localhost) {
+    opt.push(
+      ["private_ip4", args.private_ip4],
+      ["public_domain", args.public_domain],
+      ["public_ip4", args.public_ip4],
+      ["public_ip6", args.public_ip6],
+      ["storage_backup", args.backup_storage], /** Legacy */
+      ["private_domain", args.private_domain],
+      ["acme_dir", ACME_DIR],
+      ["acme_email_account", ACME_EMAIL_ACCOUNT, args.admin_email],
+      ["certs_dir", CERTS_DIR],
+    )
+
+  }
+
+  let data = makeData(opt);
+
+  if (!data) {
+    exit(1);
+  }
   let d = new Date().toISOString();
   let [day, hour] = d.split('T')
   day = day.replace(/\-/g, '');
   hour = hour.split(':')[0];
   data.serial = `${day}${hour}`;
 
-  let args = { ...data };
+  let configs = { ...data };
   let keys = ["myConf", "chroot", "date"];
 
   for (let key of keys) {
-    delete args[key];
+    delete configs[key];
   }
 
-  if (ARGV.readonly) {
-    return args;
+  if (args.readonly) {
+    return configs;
   }
-  args.socketPath = getSocketPath();
-  console.log("Writing main conf into drumee.json");
-  writeFileSync(
-    Template.chroot("etc/drumee/drumee.json"), args, JSON_OPT
-  );
-  return args;
+
+  configs.socketPath = getSocketPath();
+  configs.runtime_dir = join(configs.drumee_root, 'runtime');
+  configs.server_dir = join(configs.runtime_dir, 'server');
+  configs.server_base = configs.server_dir;
+  configs.server_home = join(configs.server_base, 'main');
+  configs.server_location = configs.server_home;
+
+  //console.log(configs)
+  configs.ui_dir = join(configs.runtime_dir, 'ui');
+  configs.ui_base = join(configs.ui_dir, 'main');
+  configs.ui_home = configs.ui_base;
+  configs.ui_location = configs.ui_base;
+
+  configs.tmp_dir = join(configs.runtime_dir, 'tmp');
+  configs.static_dir = join(configs.runtime_dir, 'static');
+
+  let filename = Template.chroot("etc/drumee/drumee.json");
+  console.log("Writing main conf into drumee.json", filename);
+  Template.makedir(dirname(filename));
+  writeFileSync(filename, configs, JSON_OPT);
+  console.log(configs)
+  return configs;
 }
 
 /**
@@ -301,6 +408,7 @@ function getSysConfigs() {
 function writeCredentials(file, data) {
   let target = Template.chroot(`etc/drumee/credential/${file}.json`);
   console.log(`Writing credentials into ${target}`);
+  Template.makedir(dirname(target));
   writeFileSync(target, data, JSON_OPT);
 }
 
@@ -317,14 +425,13 @@ function errorHandler(err) {
  */
 function copyConfigs(items) {
   for (let item of items) {
-    let p = join(__dirname, 'configs', item);
-    console.log(`Copying ${item}`)
-    let dname = dirname(p);
-    mkdirSync(dname, { recursive: true });
-    let content = readFileSync(p);
+    let src = join(__dirname, 'configs', item);
+    let dest = Template.chroot(item);
+    console.log(`Copying ${src} to ${dest}`)
+    Template.makedir(dirname(dest))
+    let content = readFileSync(src);
     let str = String(content).toString();
     //Buffer.from(content, "utf8");
-    let dest = Template.chroot(item);
     let fd = openSync(dest, "w+");
     writeSync(fd, str);
     close(fd, errorHandler);
@@ -361,92 +468,127 @@ function getDkim(file) {
  *
  */
 function writeInfraConf(data) {
-  writeEcoSystem(data);
+
   const etc = 'etc';
   const nginx = join(etc, 'nginx');
   const drumee = join(etc, 'drumee');
-  const bind = join(etc, 'bind',);
-  const postfix = join(etc, 'postfix',);
+  const bind = join(etc, 'bind');
+  const libbind = join('var', 'lib', 'bind');
+  const postfix = join(etc, 'postfix');
   const mariadb = join(etc, 'mysql', 'mariadb.conf.d');
   const infra = join(drumee, 'infrastructure');
-  const { domain } = data;
-  const dkim = join(etc, 'opendkim', 'keys', domain, 'dkim.txt');
+  const { public_domain, private_domain } = data;
   let targets = [
-
-    // Nginx 
-    `${nginx}/sites-enabled/drumee.conf`,
-
-    // Drumee 
-    `${drumee}/ssl/main.conf`,
     `${drumee}/drumee.sh`,
-    `${drumee}/conf.d/conference.json`,
     `${drumee}/conf.d/drumee.json`,
     `${drumee}/conf.d/exchange.json`,
     `${drumee}/conf.d/myDrumee.json`,
-    `${drumee}/conf.d/conference.json`,
     `${drumee}/conf.d/drumee.json`,
     `${drumee}/conf.d/myDrumee.json`,
 
+    `${nginx}/nginx.conf`,
+
     `${infra}/mfs.conf`,
     `${infra}/routes/main.conf`,
     `${infra}/internals/accel.conf`,
-    {
-      tpl: `var/lib/bind/domain.tpl`,
-      out: `var/lib/bind/${domain}`
-    },
-    `${bind}/named.conf.local`,
-    `${bind}/named.conf.log`,
-    `${bind}/named.conf.options`,
-    `${postfix}/main.cf`,
-    `${postfix}/mysql-virtual-alias-maps.cf`,
-    `${postfix}/mysql-virtual-mailbox-domains.cf`,
-    `${postfix}/mysql-virtual-mailbox-maps.cf`,
-    `${etc}/dkimkeys/dkim.key`,
-    `${etc}/mail/dkim.key`,
-    `${etc}/mailname`,
-    `${etc}/opendkim/KeyTable`,
     `${mariadb}/50-server.cnf`,
+    `${mariadb}/50-client.cnf`,
   ];
 
-  if (data.reverse_ip4) {
-    targets.push({
-      tpl: `var/lib/bind/reverse.tpl`,
-      out: `var/lib/bind/${data.public_ip4}`
-    })
+  if (args.localhost) {
+    let { username } = userInfo();
+    let system_group = username;
+    if (username = 'root') {
+      username = data.system_user || 'www-data';
+      system_group = data.system_group || 'www-data';
+    }
+    data.system_user = username;
+    data.system_group = system_group;
+    targets.push(`${nginx}/sites-enabled/localhost.conf`)
+    let dir = join(args.drumee_root, 'cache', 'localhost')
+    mkdirSync(dir, { recursive: true });
+  } else {
+    targets.push(
+      `${bind}/named.conf.log`,
+      `${bind}/named.conf.options`,
+    )
+  }
+
+  writeEcoSystem(data);
+  if (data.public_ip4 && public_domain) {
+    let dir = join(args.drumee_root, 'cache', public_domain)
+    mkdirSync(dir, { recursive: true });
+    targets.push(
+      `${nginx}/sites-enabled/public.conf`,
+      `${drumee}/ssl/public.conf`,
+      `${bind}/named.conf.public`,
+      { tpl: `${libbind}/public.tpl`, out: `${libbind}/${public_domain}` },
+      { tpl: `${libbind}/public-reverse.tpl`, out: `${libbind}/${data.public_ip4}` }
+    );
+
+    const dkim = join(etc, 'opendkim', 'keys', public_domain, 'dkim.txt');
+    targets.push(
+      `${postfix}/main.cf`,
+      `${postfix}/mysql-virtual-alias-maps.cf`,
+      `${postfix}/mysql-virtual-mailbox-domains.cf`,
+      `${postfix}/mysql-virtual-mailbox-maps.cf`,
+      `${etc}/dkimkeys/dkim.key`,
+      `${etc}/mail/dkim.key`,
+      `${etc}/mailname`,
+      `${etc}/opendkim/KeyTable`,
+    )
+    data.dkim_key = getDkim(dkim);
+    data.mail_user = MAIL_USER || 'postfix';
+    data.mail_password = uniqueId();
+    data.smptd_cache_db = "btree:$";
   }
-  data.dkim_key = getDkim(dkim);
-  data.mail_user = MAIL_USER || 'postfix';
-  data.mail_password = uniqueId();
-  data.smptd_cache_db = "btree:$";
-  data.endpoint = 'main';
+
+  if (data.private_ip4 && private_domain) {
+    let dir = join(args.drumee_root, 'cache', private_domain)
+    mkdirSync(dir, { recursive: true });
+    targets.push(
+      `${nginx}/sites-enabled/private.conf`,
+      `${drumee}/ssl/private.conf`,
+      `${bind}/named.conf.private`,
+      { tpl: `${libbind}/private.tpl`, out: `${libbind}/${private_domain}` },
+      { tpl: `${libbind}/private-reverse.tpl`, out: `${libbind}/${data.private_ip4}` },
+    )
+  }
+
+
   writeTemplates(data, targets);
 
-  writeCredentials("postfix", {
-    host: 'localhost',
-    user: data.mail_user,
-    password: data.mail_password,
-  })
+  if (!args.localhost) {
+    writeCredentials("postfix", {
+      host: 'localhost',
+      user: data.mail_user,
+      password: data.mail_password,
+    })
 
-  writeCredentials("db", {
-    password: uniqueId(),
-    user: "drumee-app",
-    host: "localhost",
-  })
+    writeCredentials("db", {
+      password: uniqueId(),
+      user: "drumee-app",
+      host: "localhost",
+    })
 
-  writeCredentials("email", {
-    host: `localhost`,
-    port: 587,
-    secure: false,
-    auth: {
-      user: `butler@${domain}`,
-      pass: uniqueId()
-    },
-    tls: {
-      rejectUnauthorized: false
-    }
-  })
+    writeCredentials("email", {
+      host: `localhost`,
+      port: 587,
+      secure: false,
+      auth: {
+        user: `butler@${public_domain}`,
+        pass: uniqueId()
+      },
+      tls: {
+        rejectUnauthorized: false
+      }
+    })
 
-  copyConfigs(['etc/postfix/master.cf', 'etc/cron.d/drumee'])
+    copyConfigs([
+      'etc/postfix/master.cf',
+      'etc/cron.d/drumee',
+    ])
+  }
 }
 
 /**
@@ -479,7 +621,6 @@ function writeJitsiConf(data) {
     // Nginx 
     `${nginx}/sites-enabled/jitsi.conf`,
     `${nginx}/modules-enabled/90-turn-relay.conf`,
-    //`${nginx}/sites-enabled/turnrelay.conf`,
 
     // Prosody 
     `${prosody}/prosody.cfg.lua`,
@@ -514,15 +655,15 @@ function makeConfData(data) {
     jvb_password: randomString(),
     app_id: randomString(),
     app_password: randomString(),
-    ui_base: join(data.ui_base, 'dist', 'main'),
+    ui_base: join(data.ui_base, 'main'),
     location: '/-/',
     pushPort: 23000,
     restPort: 24000,
   };
   if (!data.export_dir) data.export_dir = null;
   if (!data.import_dir) data.import_dir = null;
-  if (!data.local_address) {
-    data.local_address = data.public_address || "127.0.0.1";
+  if (!data.private_address) {
+    data.private_address = data.public_address || "127.0.0.1";
   }
   return data
 }
@@ -536,38 +677,64 @@ function privateIp() {
   })
 }
 
+/**
+ * 
+ */
+async function getAddresses(data) {
+  const isPrivate = await privateIp();
+  let os = require("os");
+  let interfaces = os.networkInterfaces();
+  let private_ip4, public_ip4, private_ip6, public_ip6;
+  for (let name in interfaces) {
+    if (name == 'lo') continue;
+    for (let dev of interfaces[name]) {
+      switch (dev.family) {
+        case 'IPv4':
+          if (isPrivate(dev.address) && !private_ip4) {
+            private_ip4 = dev.address;
+          }
+          if (!isPrivate(dev.address) && !public_ip4) {
+            public_ip4 = dev.address;
+          }
+          break;
+        case 'IPv6':
+          if (isPrivate(dev.address) && !private_ip6) {
+            private_ip6 = dev.address;
+          }
+          if (!isPrivate(dev.address) && !public_ip6) {
+            public_ip6 = dev.address;
+          }
+          break;
+      }
+    }
+  }
+  data.private_ip4 = data.private_ip4 || private_ip4;
+  data.private_ip6 = data.private_ip6 || private_ip6;
+  data.local_address = data.private_ip4;
+
+  data.public_ip4 = data.public_ip4 || public_ip4;
+  data.public_ip6 = data.public_ip6 || public_ip6;
+
+  return data;
+}
+
 /**
  *
  * @returns
  */
-function configure() {
+function main() {
+  const env_root = args.outdir || args.chroot;
+  if (env_root) loadSysEnv(env_root);
   return new Promise(async (res, rej) => {
     let data = getSysConfigs();
     data.chroot = Template.chroot();
-    const isPrivate = await privateIp();
-    let os = require("os");
-    let interfaces = os.networkInterfaces();
-    for (let name in interfaces) {
-      for (let dev of interfaces[name]) {
-        if (dev.family == 'IPv4') {
-          if (isPrivate(dev.address)) {
-            data.local_address = dev.address;
-            break;
-          }
-          if (!dev.internal) {
-            public_address = dev.address;
-          }
-        }
-      }
-      if (data.local_address) break;
-    }
-    data = makeConfData(data);
+    data = { ...data, ...makeConfData(data) };
+    data = await getAddresses(data);
     let func = [];
-    if (!ARGV.infra && !ARGV.jitsi) {
-      func = [writeInfraConf, writeJitsiConf];
+    if (args.only_infra || args.no_jitsi || args.localhost || data.local_domain) {
+      func.push(writeInfraConf)
     } else {
-      if (ARGV.infra) func.push(writeInfraConf)
-      if (ARGV.jitsi) func.push(writeJitsiConf)
+      func = [writeInfraConf, writeJitsiConf];
     }
     func.map(function (f) {
       f(data);
@@ -577,7 +744,7 @@ function configure() {
   });
 }
 
-configure()
+main()
   .then(() => {
     exit(0);
   })