@pryv/boiler 1.0.8

package/src/config.js

@@ -0,0 +1,441 @@
+/**
+ * @license
+ * [BSD-3-Clause](https://github.com/pryv/pryv-boiler/blob/master/LICENSE)
+ */
+
+/**
+ * Load configuration in the following order (1st prevails)
+ *
+ * .0 'memory' -> empty, use when doing 'config.set()'
+ * .1 'override-file' -> Loaded at from override-config.yml (if present)
+ * .2 'test' -> empty, used by test to override any other config parameter
+ * .3 'argv' -> Loaded from arguments
+ * .4 'env' -> Loaded from environement variables
+ * .5 'base' -> Loaded from ${process.env.NODE_ENV}-config.yml (if present) or --config parameter
+ * .6 and next -> Loaded from extras
+ * .end
+ *  . 'default-file' -> Loaded from ${baseDir}/default-config.yml
+ *  . 'defaults' -> Hard coded defaults for logger
+ */
+
+const fs = require('fs');
+const path = require('path');
+
+const nconf = require('nconf');
+nconf.formats.yaml = require('./lib/nconf-yaml');
+
+const superagent = require('superagent');
+
+/**
+ * Default values for Logger
+ */
+const defaults = {
+  logs: {
+    console: {
+      active: true,
+      level: 'info',
+      format: {
+        color: true,
+        time: true,
+        aligned: true
+      }
+    },
+    file: {
+      active: true,
+      path: 'application.log',
+      rotation: {
+        isActive: false
+      }
+    },
+    airbrake: {
+      active: false
+    }
+  }
+};
+
+
+
+/**
+ * Config manager
+ */
+class Config {
+  store;
+  logger;
+  extraAsync;
+  baseConfigDir;
+  learnDirectoryAndFilename;
+
+  constructor() {
+    this.extraAsync = [];
+  }
+
+  /**
+   * @private
+   * Init Config with Files should be called just once when starting an APP
+   * @param {Object} options
+   * @param {string} appName
+   * @param {string} [learnDirectory] - (optional) if set, all .get() calls will be tracked in this files in this directory
+   * @param {string} [options.baseConfigDir] - (optional) directory to use to look for configs (default, env)
+   * @param {Array<ConfigFile|ConfigPlugin|ConfigData|ConfigRemoteURL|ConfigRemoteURLFromKey>} [options.extras] - (optional) and array of extra files or plugins to load (synchronously or async)
+   * @param {Object} logging
+   * @returns {Config} this
+   */
+  initSync(options, logging) {
+    this.appName = options.appName;
+    this.learnDirectoryAndFilename = getLearnFilename(options.appName, options.learnDirectory);
+
+    const logger = this.logger = logging.getLogger('config');
+    const store = this.store = new nconf.Provider();
+
+    const baseConfigDir = this.baseConfigDir = options.baseConfigDir || process.cwd();
+    logger.debug('Init with baseConfigDir: ' + baseConfigDir);
+
+    // 0. memory at top
+    store.use('memory');
+
+    // 1. eventual ovverride-config.yml
+    loadFile('override-file', path.resolve(baseConfigDir, 'override-config.yml'));
+
+    // 2. put a 'test' store up in the list that could be overwitten afterward and override other options
+    // override 'test' store with store.add('test', {type: 'literal', store: {....}});
+    store.use('test', { type: 'literal', store: {} });
+
+    // get config from arguments and env variables
+    // memory must come first for config.set() to work without loading config files
+    // 3. `process.env`
+    // 4. `process.argv`
+    store.argv({parseValues: true}).env({parseValues: true, separator: '__'});
+
+    // 5. Values in `${NODE_ENV}-config.yml` or from --config parameter
+    let configFile;
+    if (store.get('config')) {
+      configFile = store.get('config')
+    } else if (store.get('NODE_ENV')) {
+      configFile = path.resolve(baseConfigDir, store.get('NODE_ENV') + '-config.yml');
+    }
+    if (configFile) {
+      loadFile('base', configFile);
+    } else {
+      // book 'base' slot
+      store.use('base', { type: 'literal', store: {} });
+      logger.debug('Booked [base] empty as no --config or NODE_ENV was set');
+    }
+
+    // load extra config files & plugins
+    if (options.extras) {
+      for (let extra of options.extras) {
+        if (extra.file) {
+          loadFile(extra.scope, extra.file);
+          continue;
+        }
+        if (extra.plugin) {
+          const name = extra.plugin.load(this);
+          logger.debug('Loaded plugin: ' + name + ' ' + extra.plugin.load.then);
+          continue;
+        }
+        if (extra.data) {
+          const conf = extra.key ? {[extra.key]: extra.data} : extra.data;
+          store.use(extra.scope, { type: 'literal', store: conf });
+          logger.debug('Loaded [' + extra.scope + '] from DATA: ' + (extra.key ? ' under [' + extra.key + ']' : ''));
+          continue;
+        }
+        if (extra.url || extra.urlFromKey || extra.fileAsync) {
+          // register scope in the chain to keep order of configs
+          store.use(extra.scope, { type: 'literal', store: {} });
+          logger.debug('Booked [' + extra.scope +'] for async Loading ');
+          this.extraAsync.push(extra);
+          continue;
+        }
+        if (extra.pluginAsync) {
+          logger.debug('Added 1 plugin for async Loading ');
+          this.extraAsync.push(extra);
+          continue;
+        }
+        logger.warn('Unkown extra in config init', extra);
+      }
+    }
+
+
+    // .end-1 load default and custom config from configs/default-config.json
+    loadFile('default-file', path.resolve(baseConfigDir, 'default-config.yml'));
+
+    // .end load hard coded defaults
+    store.defaults(defaults);
+
+    // init Logger
+    logging.initLoggerWithConfig(this);
+    return this;
+
+    // --- helpers --/
+
+    function loadFile(scope, filePath) {
+
+      if (fs.existsSync(filePath)) {
+
+        if (filePath.endsWith('.js')) {  // JS file
+          const conf = require(filePath);
+          store.use(scope, { type: 'literal', store: conf });
+        } else {   // JSON or YAML
+          const options = { file: filePath }
+          if (filePath.endsWith('.yml') || filePath.endsWith('.yaml')) { options.format = nconf.formats.yaml }
+          store.file(scope, options);
+        }
+
+        logger.debug('Loaded [' + scope + '] from file: ' + filePath)
+      } else {
+        logger.debug('Cannot find file: ' + filePath + ' for scope [' + scope + ']');
+      }
+    }
+  }
+
+  async initASync() {
+    const store = this.store;
+    const logger = this.logger;
+    const baseConfigDir = this.baseConfigDir;
+
+    async function loadUrl(scope, key, url) {
+      if (typeof url === 'undefined' || url === null) {
+          logger.warn('Null or Undefined Url for [' + scope +']');
+          return;
+      }
+
+      let res = null;
+      if (isFileUrl(url)) {
+        res = loadFromFile(url);
+      } else {
+        res = await loadFromUrl(url);
+      }
+      const conf = key ? {[key]: res} : res;
+      store.add(scope, { type: 'literal', store: conf });
+      logger.debug('Loaded [' + scope + '] from URL: ' + url + (key ? ' under [' + key + ']' : ''));
+    }
+
+    // load remote config files
+    for (let extra of this.extraAsync) {
+      if (extra.url) {
+        await loadUrl(extra.scope, extra.key, extra.url);
+        continue;
+      }
+      if (extra.urlFromKey) {
+        const url = store.get(extra.urlFromKey);
+        await loadUrl(extra.scope, extra.key, url);
+        continue;
+      }
+
+      if (extra.pluginAsync) {
+        const name = await extra.pluginAsync.load(this);
+        logger.debug('Loaded async plugin: ' + name);
+        continue;
+      }
+
+      if (extra.fileAsync) {
+        const filePath = path.resolve(baseConfigDir, extra.fileAsync);
+
+        if (! fs.existsSync(filePath)) {
+          logger.warn('Cannot find file: ' + filePath + ' for scope [' + extra.scope + ']');
+          continue;
+        }
+        if (! filePath.endsWith('.js')) {
+          logger.warn('Cannot only load .js file: ' + filePath + ' for scope [' + extra.scope + ']');
+          continue;
+        }
+
+        const conf = await require(filePath)();
+        store.add(extra.scope, { type: 'literal', store: conf });
+
+        logger.debug('Loaded in scope [' + extra.scope + ']async .js file: ' + filePath);
+      }
+    }
+
+    logger.debug('Config fully Loaded');
+    saveConfig(this.learnDirectoryAndFilename, this.store);
+    return this;
+  }
+
+  /**
+   * Return true if key as value
+   * @param {string} key
+   * @returns {boolean}
+   */
+  has(key) {
+    if (! this.store) { throw(new Error('Config not yet initialized'))}
+    const value = this.store.get(key);
+    return (typeof value !== 'undefined');
+  }
+
+  /**
+   * Retreive value
+   * @param {string} key
+   */
+  get(key) {
+    if (! this.store) { throw(new Error('Config not yet initialized'))}
+    learn(this.learnDirectoryAndFilename, key);
+    const value = this.store.get(key);
+    if (typeof value === 'undefined') this.logger.debug('get: [' + key +'] is undefined');
+    return value;
+  }
+
+  /**
+   * Retreive value and store info that applies
+   * @param {string} key
+   */
+  getScopeAndValue(key) {
+    if (! this.store) { throw(new Error('Config not yet initialized'))};
+    for (let scopeName of Object.keys(this.store.stores)) {
+      const store = this.store.stores[scopeName];
+      const value = store.get(key);
+      if (typeof value !== 'undefined') {
+        const res = {
+          value: value,
+          scope: scopeName
+        }
+        if (store.type === 'file') {
+          res.info = 'From file: ' + store.file
+        } else {
+          info = 'Type: ' + store.type
+        }
+        return res;
+      }
+    }
+    return null;
+  }
+
+  /**
+   * Set value
+   * @param {string} key
+   * @param {Object} value
+   */
+  set(key, value) {
+    if (! this.store) { throw(new Error('Config not yet initialized'))}
+    this.store.set(key, value);
+  }
+
+  /**
+   * Inject Test Config and override any other option
+   * @param {Object} configObject;
+   */
+  injectTestConfig(configObject) {
+    this.replaceScopeConfig('test', configObject);
+  }
+
+  /**
+   * Replace a scope config set
+   * @param {string} scope;
+   * @param {Object} configObject;
+   */
+  replaceScopeConfig(scope, configObject) {
+    if (! this.store) { throw(new Error('Config not yet initialized'))}
+    this.logger.debug('Replace ['+ scope + '] with: ', configObject);
+    this.store.add(scope, {type: 'literal', store: configObject});
+  }
+
+}
+
+module.exports = Config;
+
+// --- remote and local json ressource loader ---- //
+
+const FILE_PROTOCOL = 'file://';
+const FILE_PROTOCOL_LENGTH = FILE_PROTOCOL.length;
+
+async function loadFromUrl(serviceInfoUrl ) {
+  const res = await superagent.get(serviceInfoUrl);
+  return res.body;
+}
+
+function loadFromFile(fileUrl ) {
+  const filePath = stripFileProtocol(fileUrl);
+
+  if (isRelativePath(filePath)) {
+    const serviceCorePath = path.resolve(__dirname, '../../../../../'); // assuming /node_modules/@pryv/boiler/src/
+    fileUrl = path.resolve(serviceCorePath, filePath);
+    fileUrl = 'file://' + fileUrl;
+  } else {
+    // absolute path, do nothing.
+  }
+  const res = JSON.parse(
+    fs.readFileSync(stripFileProtocol(fileUrl), 'utf8')
+  );
+  return res;
+}
+
+
+function isFileUrl(filePath) {
+  return filePath.startsWith(FILE_PROTOCOL);
+}
+
+function isRelativePath(filePath) {
+  return !path.isAbsolute(filePath);
+}
+
+function stripFileProtocol(filePath) {
+  return filePath.substring(FILE_PROTOCOL_LENGTH);
+}
+
+
+// -------- learning mode ------- //
+
+function getLearnFilename(appName, learnDirectory) {
+  if (! learnDirectory) return;
+  let i = 0;
+  let res;
+  do {
+    res = path.join(learnDirectory, appName + i );
+    i++;
+  } while(fs.existsSync(res + '-config.json'));
+  return res;
+}
+
+function learn(learnDirectoryAndFilename, key) {
+  if (learnDirectoryAndFilename) {
+    const caller_line = (new Error()).stack.split('\n')[3]; // get callee name and line
+    const index = caller_line.indexOf("at ");
+    str = key + ';' + caller_line.slice(index+3, caller_line.length) + '\n';
+    fs.appendFileSync(learnDirectoryAndFilename + '-calls.csv', str);
+  }
+}
+
+function saveConfig(learnDirectoryAndFilename, store) {
+  if (learnDirectoryAndFilename) {
+    let i = 0;
+    let filename;
+    do {
+      filename =learnDirectoryAndFilename + '-config.json';
+      i++;
+    } while(fs.existsSync(filename));
+    fs.writeFileSync(filename, JSON.stringify({stores: store.stores, config: store.get()}, null, 2));
+  }
+}
+
+
+/**
+ * @typedef ConfigFile
+ * @property {string} scope - scope for nconf hierachical load
+ * @property {string} file - the config file (.yml, .json, .js)
+ */
+
+ /**
+ * @typedef ConfigPlugin
+ * @property {Object} plugin
+ * @property {Function} plugin.load - a function that takes the "nconf store" as argument and returns the "name" of the plugin
+ */
+
+ /**
+ * @typedef ConfigData
+ * @property {string} scope - scope for nconf hierachical load
+ * @property {string} [key] - (optional) key to load result of url. If null loaded at root of the config
+ * @property {object} data - the data to load
+
+
+/**
+ * @typedef ConfigRemoteURL
+ * @property {string} scope - scope for nconf hierachical load
+ * @property {string} [key] - (optional) key to load result of url. If null loaded at root of the config
+ * @property {string} url - the url to the config definition
+ */
+/**
+ * @typedef ConfigRemoteURLFromKey
+ * @property {string} scope - scope for nconf hierachical load
+ * @property {string} [key] - (optional) key to load result of url. If null override
+ * @property {string} urlFromKey - retrieve url from config matching this key
+ */

package/src/index.js

@@ -0,0 +1,115 @@
+/**
+ * @license
+ * [BSD-3-Clause](https://github.com/pryv/pryv-boiler/blob/master/LICENSE)
+ */
+
+/**
+  * Pryv Boiler module.
+  * @module boiler
+  */
+
+const Config = require('./config');
+const logging = require('./logging');
+const airbrake = require('./airbrake');
+
+const config = new Config();
+
+const boiler = {
+  /**
+   * notify Airbrake.
+   * If initalize, arguments will be passed to airbrake.notify()
+   */
+  notifyAirbrake: airbrake.notifyAirbrake,
+
+  /**
+   * get a Logger
+   * @param {string} name
+   * @returns {Logger}
+   */
+  getLogger: logging.getLogger,
+  /**
+   * Prefered way to get the configuration
+   * @returns {Promise}
+   */
+  getConfig: getConfig,
+  /**
+   * get the configuration.
+   * If the configuration is not fully initialized throw an error
+   * @param {boolean} warnOnly - Only warns about potential misuse of config
+   * @returns {Config}
+   */
+  getConfigUnsafe: getConfigUnsafe,
+
+  /**
+   * Init Boiler, should be called just once when starting an APP
+   * @param {Object} options
+   * @param {string} options.appName - the name of the Application used by Logger and debug
+   * @param {string} [options.baseConfigDir] - (optional) directory to use to look for configs
+   * @param {Array<ConfigFile|ConfigRemoteURL|ConfigRemoteURLFromKey|ConfigPlugin>} [options.extraConfigs] - (optional) and array of extra files to load
+   * @param {Function} [fullyLoadedCallback] - (optional) called when the config is fully loaded
+   */
+  init: init
+};
+
+let logger;
+let configInitialized = false;
+let configInitCalledWithName = null;
+
+function init (options, fullyLoadedCallback) {
+  if (configInitCalledWithName) {
+    logger.warn('Skipping initalization! boiler is already initialized with appName: ' + configInitCalledWithName);
+    return boiler;
+  }
+
+  // append the value of process.env.PRYV_BOILER_SUFFIX if present
+  options.appNameWithoutPostfix = options.appName;
+  if (process.env.PRYV_BOILER_SUFFIX) options.appName += process.env.PRYV_BOILER_SUFFIX;
+
+  logging.setGlobalName(options.appName);
+  configInitCalledWithName = options.appName;
+  config.initSync({
+    baseConfigDir: options.baseConfigDir,
+    extras: options.extraConfigs,
+    appName: options.appNameWithoutPostfix,
+    learnDirectory: process.env.CONFIG_LEARN_DIR
+  }, logging);
+
+  logger = logging.getLogger('boiler');
+  airbrake.setUpAirbrakeIfNeeded(config, logger);
+
+  config.initASync().then((config) => {
+    configInitialized = true;
+    // airbrake config might come from async settings, so we try twice.
+    airbrake.setUpAirbrakeIfNeeded(config, logger);
+    if (fullyLoadedCallback) fullyLoadedCallback(config);
+  });
+
+  return boiler;
+}
+
+async function getConfig () {
+  if (!configInitCalledWithName) {
+    throw (new Error('boiler must be initialized with init() before using getConfig()'));
+  }
+  /* eslint-disable-next-line no-unmodified-loop-condition */
+  while (!configInitialized) {
+    await new Promise(resolve => setTimeout(resolve, 100)); // wait 100ms
+  }
+  return config;
+}
+
+function getConfigUnsafe (warnOnly) {
+  if (!configInitCalledWithName) {
+    throw (new Error('boiler must be initialized with init() before using getConfigUnsafe()'));
+  }
+  if (!configInitialized) {
+    if (warnOnly) {
+      logger.warn('Warning! config loaded before being fully initialized');
+    } else {
+      throw (new Error('Config loaded before being fully initialized'));
+    }
+  }
+  return config;
+}
+
+module.exports = boiler;

package/src/lib/nconf-yaml.js

@@ -0,0 +1,13 @@
+/**
+ * @license
+ * [BSD-3-Clause](https://github.com/pryv/pryv-boiler/blob/master/LICENSE)
+ */
+const yaml = require('js-yaml');
+
+exports.stringify = function (obj, options) {
+  return yaml.dump(obj, options);
+};
+
+exports.parse = function (obj, options) {
+  return yaml.load(obj, options);
+};