@drumee/setup-infra 1.0.13 → 1.0.14

package/index.js

@@ -1,37 +1,25 @@
 #!/usr/bin/env node
 
-// ======================================================
-//
-// ======================================================
 const Template = require("./templates");
-const { writeFileSync } = require(`jsonfile`);
+const { writeFileSync, readFileSync: readJson } = require(`jsonfile`);
 const { join, dirname } = require("path");
 const { isString } = require("lodash");
 const { exit } = process;
-const { sysEnv, uniqueId } = require("@drumee/server-essentials");
-const { totalmem } = require('os');
-const ARGV = require('minimist')(process.argv.slice(2));
+const { loadSysEnv, sysEnv, uniqueId } = require("@drumee/server-essentials");
+const { totalmem, userInfo } = require('os');
 const {
   existsSync, close, writeSync, openSync, readFileSync, mkdirSync
 } = require("fs");
+const { args, hasExistingSettings } = require('./templates/utils')
 
 const JSON_OPT = { spaces: 2, EOL: "\r\n" };
 
 const {
   ACME_DIR,
   ACME_EMAIL_ACCOUNT,
-  ADMIN_EMAIL,
-  DB_BACKUP,
-  DRUMEE_DB_DIR,
-  DRUMEE_DESCRIPTION,
-  DRUMEE_DOMAIN_NAME,
-  FORCE_INSTALL,
+  CERTS_DIR,
   MAIL_USER,
-  MAX_BODY_SIZE,
   NSUPDATE_KEY,
-  PUBLIC_IP4,
-  PUBLIC_IP6,
-  STORAGE_BACKUP,
 } = process.env;
 
 /**
@@ -69,8 +57,7 @@ function copyFields(data, keys) {
  */
 function factory(data) {
   let route = "main";
-  let mode = "dist";
-  let base = `${data.server_dir}/${mode}/${route}/`;
+  let base = `${data.server_dir}/${route}/`;
   return {
     name: "factory",
     script: `./index.js`,
@@ -109,10 +96,10 @@ function worker(data, instances = 1, exec_mode = 'fork_mode') {
     name,
     server_dir,
     runtime_dir,
-    mode,
   } = data;
+
   if (!server_dir) server_dir = join(runtime_dir, 'server');
-  let base = `${server_dir}/${mode}/${route}`;
+  let base = `${server_dir}/${route}`;
   return {
     name,
     script,
@@ -134,16 +121,21 @@ function worker(data, instances = 1, exec_mode = 'fork_mode') {
  * 
  */
 function writeTemplates(data, targets) {
-  if (ARGV.readonly || ARGV.noCheck) {
+  if (args.readonly || args.noCheck) {
     console.log("Readonly", targets, data);
     return
   }
   for (let target of targets) {
-    if (isString(target)) {
-      Template.write(data, target, target);
-    } else {
-      let { out, tpl } = target;
-      Template.write(data, out, tpl);
+    try {
+      if (isString(target)) {
+        Template.write(data, target, target);
+      } else {
+        let { out, tpl } = target;
+        Template.write(data, out, tpl);
+      }
+    } catch (e) {
+      console.error(e)
+      console.error("Failed to write configs for", target)
     }
   }
 }
@@ -155,10 +147,8 @@ function writeEcoSystem(data) {
   const ports = {
     pushPort: 23000,
     restPort: 24000,
-    mode: "dist",
     route: "main",
   };
-
   let main = worker({
     ...data,
     ...ports,
@@ -182,11 +172,14 @@ function writeEcoSystem(data) {
 
   let f = factory(data);
   let routes = [main, main_service, f];
-  let ecosystem = "/etc/drumee/infrastructure/ecosystem.json";
-  if (ARGV.readonly) {
+  //let ecosystem = "etc/drumee/infrastructure/ecosystem.json";
+  let ecosystem = Template.chroot("etc/drumee/infrastructure/ecosystem.json");
+  if (args.readonly) {
     console.log("Readonly", ecosystem, routes);
     return
   }
+  console.log("Writing ecosystem into ", ecosystem);
+  Template.makedir(dirname(ecosystem));
   writeFileSync(ecosystem, routes, JSON_OPT);
   let targets = [
     {
@@ -212,9 +205,17 @@ function getSocketPath() {
   return socketPath;
 }
 
+
+/**
+ * 
+ * @param {*} opt 
+ * @returns 
+ */
 function makeData(opt) {
   let data = sysEnv();
-
+  if (args.env_file && existsSync(args.env_file)) {
+    loadEnvFile(args.env_file, opt)
+  }
   data.chroot = Template.chroot();
   data.acme_store = join(data.certs_dir, `${data.domain_name}_ecc`);
   data.ca_server = data.ca_server || data.acme_ssl;
@@ -223,75 +224,143 @@ function makeData(opt) {
   }
   for (let row of opt) {
     let [key, value, fallback] = row;
-    if (!value) value = fallback;
-    if (!value) continue;
-    if (!data[key]) {
+    if (!value) value = data[key] || fallback;
+    if (value == null) continue;
+    if (isString(value)) {
       if (/.+\+$/.test(value)) {
         value = value.replace(/\+$/, data[key]);
       }
       data[key] = value.trim() || fallback;
+    } else {
+      data[key] = value
     }
   }
 
+  /** Named extra settings */
+  data.allow_recursion = 'localhost;';
   if (data.public_ip4) {
     data.reverse_ip4 = data.public_ip4.split('.').reverse().join('.');
+  } else {
+    data.reverse_ip4 = ""
   }
 
+  if (!data.public_ip6) {
+    data.public_ip6 = "";
+  }
+
+  if (!data.storage_backup) {
+    data.storage_backup = ""
+  }
   return data;
 }
+
+/**
+ * 
+ * @param {*} env 
+ * @param {*} opt 
+ */
+function loadEnvFile(file, opt) {
+  let src = readJson(file);
+  opt.map((r) => {
+    let [key] = r;
+    if (src[key] != null) r[1] = src[key];
+  })
+  console.log(opt)
+}
+
 /**
  *
  */
 function getSysConfigs() {
-  let { domain_name } = sysEnv();
-  if (existsSync('/etc/drumee/drumee.sh') && !FORCE_INSTALL) {
-    console.log(
-      `There is already a domain name configured on this server (${domain_name})\n`, `Use FORCE_INSTALL=yes to override`);
+  if (hasExistingSettings(Template.chroot('etc/drumee/drumee.json'))) {
     exit(0)
   }
-  domain_name = domain_name || ARGV.domain || DRUMEE_DOMAIN_NAME;
+
+  let use_email = 0;
+  if (args.public_domain) use_email = 1;
+  let domain_name = args.public_domain || args.private_domain;
   if (!domain_name) {
-    console.log("There no domain name defined for the installation");
-    exit(0)
+    if (!args.localhost) {
+      console.log("There is no domain name defined for the installation", args);
+      exit(0)
+    }
   }
+
+  const nsupdate_key = Template.chroot('etc/bind/keys/update.key')
   const opt = [
-    ["acme_dir", ACME_DIR, '/usr/share/acme'],
-    ["jitsi_domain", `jit.+`],
-    ["domain_name", domain_name],
-    ["nsupdate_key", NSUPDATE_KEY, '/etc/bind/keys/update.key'],
-    ["domain_desc", DRUMEE_DESCRIPTION, 'My Drumee Box'],
-    ["admin_email", ADMIN_EMAIL, ''],
-    ["acme_email_account", ACME_EMAIL_ACCOUNT, ADMIN_EMAIL],
-    ["public_ip4", PUBLIC_IP4, ''],
-    ["public_ip6", PUBLIC_IP6, ''],
-    ["storage_backup", STORAGE_BACKUP, ''],
-    ["db_backup", DB_BACKUP, ''],
-    ["db_dir", DRUMEE_DB_DIR, '/srv/db'],
-    ["max_body_size", MAX_BODY_SIZE, '10G'],
+    ["nsupdate_key", NSUPDATE_KEY, nsupdate_key],
+    ["admin_email", args.admin_email],
+    ["credential_dir", Template.chroot('etc/drumee/credential')],
+    ["domain_desc", args.description, 'My Drumee Box'],
+    ["max_body_size", args.max_body_size, '10G'],
+    ["drumee_root", args.drumee_root, "/var/lib/drumee"],
+    ["use_email", use_email, 0],
+    ["db_dir", args.db_dir, '/var/lib/mysql'],
+    ["backup_storage", args.backup_storage, ""],
+    ["data_dir", args.data_dir, '/var/lib/drumee/data'],
+    ["http_port", args.http_port, 80],
+    ["https_port", args.https_port, 443],
   ]
-  let data = makeData(opt)
+
+  if (!args.localhost) {
+    opt.push(
+      ["private_ip4", args.private_ip4],
+      ["public_domain", args.public_domain],
+      ["public_ip4", args.public_ip4],
+      ["public_ip6", args.public_ip6],
+      ["storage_backup", args.backup_storage], /** Legacy */
+      ["private_domain", args.private_domain],
+      ["acme_dir", ACME_DIR],
+      ["acme_email_account", ACME_EMAIL_ACCOUNT, args.admin_email],
+      ["certs_dir", CERTS_DIR],
+    )
+
+  }
+
+  let data = makeData(opt);
+
+  if (!data) {
+    exit(1);
+  }
   let d = new Date().toISOString();
   let [day, hour] = d.split('T')
   day = day.replace(/\-/g, '');
   hour = hour.split(':')[0];
   data.serial = `${day}${hour}`;
 
-  let args = { ...data };
+  let configs = { ...data };
   let keys = ["myConf", "chroot", "date"];
 
   for (let key of keys) {
-    delete args[key];
+    delete configs[key];
   }
 
-  if (ARGV.readonly) {
-    return args;
+  if (args.readonly) {
+    return configs;
   }
-  args.socketPath = getSocketPath();
-  console.log("Writing main conf into drumee.json");
-  writeFileSync(
-    Template.chroot("etc/drumee/drumee.json"), args, JSON_OPT
-  );
-  return args;
+
+  configs.socketPath = getSocketPath();
+  configs.runtime_dir = join(configs.drumee_root, 'runtime');
+  configs.server_dir = join(configs.runtime_dir, 'server');
+  configs.server_base = configs.server_dir;
+  configs.server_home = join(configs.server_base, 'main');
+  configs.server_location = configs.server_home;
+
+  //console.log(configs)
+  configs.ui_dir = join(configs.runtime_dir, 'ui');
+  configs.ui_base = join(configs.ui_dir, 'main');
+  configs.ui_home = configs.ui_base;
+  configs.ui_location = configs.ui_base;
+
+  configs.tmp_dir = join(configs.runtime_dir, 'tmp');
+  configs.static_dir = join(configs.runtime_dir, 'static');
+
+  let filename = Template.chroot("etc/drumee/drumee.json");
+  console.log("Writing main conf into drumee.json", filename);
+  Template.makedir(dirname(filename));
+  writeFileSync(filename, configs, JSON_OPT);
+  console.log(configs)
+  return configs;
 }
 
 /**
@@ -301,6 +370,7 @@ function getSysConfigs() {
 function writeCredentials(file, data) {
   let target = Template.chroot(`etc/drumee/credential/${file}.json`);
   console.log(`Writing credentials into ${target}`);
+  Template.makedir(dirname(target));
   writeFileSync(target, data, JSON_OPT);
 }
 
@@ -317,14 +387,13 @@ function errorHandler(err) {
  */
 function copyConfigs(items) {
   for (let item of items) {
-    let p = join(__dirname, 'configs', item);
-    console.log(`Copying ${item}`)
-    let dname = dirname(p);
-    mkdirSync(dname, { recursive: true });
-    let content = readFileSync(p);
+    let src = join(__dirname, 'configs', item);
+    let dest = Template.chroot(item);
+    console.log(`Copying ${src} to ${dest}`)
+    Template.makedir(dirname(dest))
+    let content = readFileSync(src);
     let str = String(content).toString();
     //Buffer.from(content, "utf8");
-    let dest = Template.chroot(item);
     let fd = openSync(dest, "w+");
     writeSync(fd, str);
     close(fd, errorHandler);
@@ -361,92 +430,131 @@ function getDkim(file) {
  *
  */
 function writeInfraConf(data) {
-  writeEcoSystem(data);
+
   const etc = 'etc';
   const nginx = join(etc, 'nginx');
   const drumee = join(etc, 'drumee');
-  const bind = join(etc, 'bind',);
-  const postfix = join(etc, 'postfix',);
+  const bind = join(etc, 'bind');
+  const libbind = join('var', 'lib', 'bind');
+  const postfix = join(etc, 'postfix');
   const mariadb = join(etc, 'mysql', 'mariadb.conf.d');
   const infra = join(drumee, 'infrastructure');
-  const { domain } = data;
-  const dkim = join(etc, 'opendkim', 'keys', domain, 'dkim.txt');
+  const { public_domain, private_domain } = data;
   let targets = [
-
-    // Nginx 
-    `${nginx}/sites-enabled/drumee.conf`,
-
-    // Drumee 
-    `${drumee}/ssl/main.conf`,
     `${drumee}/drumee.sh`,
-    `${drumee}/conf.d/conference.json`,
     `${drumee}/conf.d/drumee.json`,
     `${drumee}/conf.d/exchange.json`,
     `${drumee}/conf.d/myDrumee.json`,
-    `${drumee}/conf.d/conference.json`,
     `${drumee}/conf.d/drumee.json`,
     `${drumee}/conf.d/myDrumee.json`,
 
     `${infra}/mfs.conf`,
     `${infra}/routes/main.conf`,
     `${infra}/internals/accel.conf`,
-    {
-      tpl: `var/lib/bind/domain.tpl`,
-      out: `var/lib/bind/${domain}`
-    },
-    `${bind}/named.conf.local`,
-    `${bind}/named.conf.log`,
-    `${bind}/named.conf.options`,
-    `${postfix}/main.cf`,
-    `${postfix}/mysql-virtual-alias-maps.cf`,
-    `${postfix}/mysql-virtual-mailbox-domains.cf`,
-    `${postfix}/mysql-virtual-mailbox-maps.cf`,
-    `${etc}/dkimkeys/dkim.key`,
-    `${etc}/mail/dkim.key`,
-    `${etc}/mailname`,
-    `${etc}/opendkim/KeyTable`,
-    `${mariadb}/50-server.cnf`,
   ];
 
-  if (data.reverse_ip4) {
-    targets.push({
-      tpl: `var/lib/bind/reverse.tpl`,
-      out: `var/lib/bind/${data.public_ip4}`
-    })
+  if (args.localhost) {
+    let { username } = userInfo();
+    let system_group = username;
+    if (username = 'root') {
+      username = data.system_user || 'www-data';
+      system_group = data.system_group || 'www-data';
+    }
+    data.system_user = username;
+    data.system_group = system_group;
+    targets.push(`${nginx}/sites-enabled/localhost.conf`)
+    let dir = join(args.drumee_root, 'cache', 'localhost')
+    mkdirSync(dir, { recursive: true });
+    if (args.db_dir != '/var/lib/mysql') {
+      targets.push(
+        `${mariadb}/50-server.cnf`,
+        `${mariadb}/50-client.cnf`,
+      )
+    }
+  } else {
+    targets.push(
+      `${bind}/named.conf.log`,
+      `${bind}/named.conf.options`,
+      `${mariadb}/50-server.cnf`,
+      `${mariadb}/50-client.cnf`,
+    )
   }
-  data.dkim_key = getDkim(dkim);
-  data.mail_user = MAIL_USER || 'postfix';
-  data.mail_password = uniqueId();
-  data.smptd_cache_db = "btree:$";
-  data.endpoint = 'main';
+
+  writeEcoSystem(data);
+  if (data.public_ip4 && public_domain) {
+    let dir = join(args.drumee_root, 'cache', public_domain)
+    mkdirSync(dir, { recursive: true });
+    targets.push(
+      `${nginx}/sites-enabled/public.conf`,
+      `${drumee}/ssl/public.conf`,
+      `${bind}/named.conf.public`,
+      { tpl: `${libbind}/public.tpl`, out: `${libbind}/${public_domain}` },
+      { tpl: `${libbind}/public-reverse.tpl`, out: `${libbind}/${data.public_ip4}` }
+    );
+
+    const dkim = join(etc, 'opendkim', 'keys', public_domain, 'dkim.txt');
+    targets.push(
+      `${postfix}/main.cf`,
+      `${postfix}/mysql-virtual-alias-maps.cf`,
+      `${postfix}/mysql-virtual-mailbox-domains.cf`,
+      `${postfix}/mysql-virtual-mailbox-maps.cf`,
+      `${etc}/dkimkeys/dkim.key`,
+      `${etc}/mail/dkim.key`,
+      `${etc}/mailname`,
+      `${etc}/opendkim/KeyTable`,
+    )
+    data.dkim_key = getDkim(dkim);
+    data.mail_user = MAIL_USER || 'postfix';
+    data.mail_password = uniqueId();
+    data.smptd_cache_db = "btree:$";
+  }
+
+  if (data.private_ip4 && private_domain) {
+    let dir = join(args.drumee_root, 'cache', private_domain)
+    mkdirSync(dir, { recursive: true });
+    targets.push(
+      `${nginx}/sites-enabled/private.conf`,
+      `${drumee}/ssl/private.conf`,
+      `${bind}/named.conf.private`,
+      { tpl: `${libbind}/private.tpl`, out: `${libbind}/${private_domain}` },
+      { tpl: `${libbind}/private-reverse.tpl`, out: `${libbind}/${data.private_ip4}` },
+    )
+  }
+
+
   writeTemplates(data, targets);
 
-  writeCredentials("postfix", {
-    host: 'localhost',
-    user: data.mail_user,
-    password: data.mail_password,
-  })
+  if (!args.localhost) {
+    writeCredentials("postfix", {
+      host: 'localhost',
+      user: data.mail_user,
+      password: data.mail_password,
+    })
 
-  writeCredentials("db", {
-    password: uniqueId(),
-    user: "drumee-app",
-    host: "localhost",
-  })
+    writeCredentials("db", {
+      password: uniqueId(),
+      user: "drumee-app",
+      host: "localhost",
+    })
 
-  writeCredentials("email", {
-    host: `localhost`,
-    port: 587,
-    secure: false,
-    auth: {
-      user: `butler@${domain}`,
-      pass: uniqueId()
-    },
-    tls: {
-      rejectUnauthorized: false
-    }
-  })
+    writeCredentials("email", {
+      host: `localhost`,
+      port: 587,
+      secure: false,
+      auth: {
+        user: `butler@${public_domain}`,
+        pass: uniqueId()
+      },
+      tls: {
+        rejectUnauthorized: false
+      }
+    })
 
-  copyConfigs(['etc/postfix/master.cf', 'etc/cron.d/drumee'])
+    copyConfigs([
+      'etc/postfix/master.cf',
+      'etc/cron.d/drumee',
+    ])
+  }
 }
 
 /**
@@ -479,7 +587,6 @@ function writeJitsiConf(data) {
     // Nginx 
     `${nginx}/sites-enabled/jitsi.conf`,
     `${nginx}/modules-enabled/90-turn-relay.conf`,
-    //`${nginx}/sites-enabled/turnrelay.conf`,
 
     // Prosody 
     `${prosody}/prosody.cfg.lua`,
@@ -514,15 +621,15 @@ function makeConfData(data) {
     jvb_password: randomString(),
     app_id: randomString(),
     app_password: randomString(),
-    ui_base: join(data.ui_base, 'dist', 'main'),
+    ui_base: join(data.ui_base, 'main'),
     location: '/-/',
     pushPort: 23000,
     restPort: 24000,
   };
   if (!data.export_dir) data.export_dir = null;
   if (!data.import_dir) data.import_dir = null;
-  if (!data.local_address) {
-    data.local_address = data.public_address || "127.0.0.1";
+  if (!data.private_address) {
+    data.private_address = data.public_address || "127.0.0.1";
   }
   return data
 }
@@ -536,38 +643,64 @@ function privateIp() {
   })
 }
 
+/**
+ * 
+ */
+async function getAddresses(data) {
+  const isPrivate = await privateIp();
+  let os = require("os");
+  let interfaces = os.networkInterfaces();
+  let private_ip4, public_ip4, private_ip6, public_ip6;
+  for (let name in interfaces) {
+    if (name == 'lo') continue;
+    for (let dev of interfaces[name]) {
+      switch (dev.family) {
+        case 'IPv4':
+          if (isPrivate(dev.address) && !private_ip4) {
+            private_ip4 = dev.address;
+          }
+          if (!isPrivate(dev.address) && !public_ip4) {
+            public_ip4 = dev.address;
+          }
+          break;
+        case 'IPv6':
+          if (isPrivate(dev.address) && !private_ip6) {
+            private_ip6 = dev.address;
+          }
+          if (!isPrivate(dev.address) && !public_ip6) {
+            public_ip6 = dev.address;
+          }
+          break;
+      }
+    }
+  }
+  data.private_ip4 = data.private_ip4 || private_ip4;
+  data.private_ip6 = data.private_ip6 || private_ip6;
+  data.local_address = data.private_ip4;
+
+  data.public_ip4 = data.public_ip4 || public_ip4;
+  data.public_ip6 = data.public_ip6 || public_ip6;
+
+  return data;
+}
+
 /**
  *
  * @returns
  */
-function configure() {
+function main() {
+  const env_root = args.outdir || args.chroot;
+  if (env_root) loadSysEnv(env_root);
   return new Promise(async (res, rej) => {
     let data = getSysConfigs();
     data.chroot = Template.chroot();
-    const isPrivate = await privateIp();
-    let os = require("os");
-    let interfaces = os.networkInterfaces();
-    for (let name in interfaces) {
-      for (let dev of interfaces[name]) {
-        if (dev.family == 'IPv4') {
-          if (isPrivate(dev.address)) {
-            data.local_address = dev.address;
-            break;
-          }
-          if (!dev.internal) {
-            public_address = dev.address;
-          }
-        }
-      }
-      if (data.local_address) break;
-    }
-    data = makeConfData(data);
+    data = { ...data, ...makeConfData(data) };
+    data = await getAddresses(data);
     let func = [];
-    if (!ARGV.infra && !ARGV.jitsi) {
-      func = [writeInfraConf, writeJitsiConf];
+    if (args.only_infra || args.no_jitsi || args.localhost || data.local_domain) {
+      func.push(writeInfraConf)
     } else {
-      if (ARGV.infra) func.push(writeInfraConf)
-      if (ARGV.jitsi) func.push(writeJitsiConf)
+      func = [writeInfraConf, writeJitsiConf];
     }
     func.map(function (f) {
       f(data);
@@ -577,7 +710,7 @@ function configure() {
   });
 }
 
-configure()
+main()
   .then(() => {
     exit(0);
   })

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@drumee/setup-infra",
-  "version": "1.0.13",
+  "version": "1.0.14",
   "description": "Drumee Infrastructure Setup Utilities",
   "main": "index.js",
   "scripts": {
@@ -17,12 +17,11 @@
   "author": "Somanos Sar <somanos@drumee.com>",
   "license": "AGPL V3",
   "dependencies": {
-    "@drumee/setup-infra": "^1.0.0",
-    "@drumee/server-essentials": "^1.0.4",
+    "@drumee/server-essentials": "^1.1.31",
+    "argparse": "^2.0.1",
     "crypto": "^1.0.1",
     "jsonfile": "^5.0.0",
     "lodash": "^4.17.21",
-    "minimist": "^1.2.8",
     "ovh": "^2.0.3",
     "private-ip": "^3.0.1",
     "shelljs": "^0.8.5",

package/templates/etc/bind/named.conf.options

@@ -23,7 +23,7 @@ options {
 
     // allow recursion for trusted clients only.
     recursion yes;
-    allow-query { localhost; <%= public_ip4 %>; };
+    allow-query { <%= allow_recursion %> };
 
 	// enable the query log
 	querylog yes;