@pryv/boiler 1.0.8

package/.licenser.yml

@@ -0,0 +1,72 @@
+files:
+  "**/*.js":
+    header:
+      startBlock: |
+        /**
+         * @license
+      linePrefix: " * "
+      endBlock: " */"
+      license: |
+        [{SPDX}]({LICENSE_URL})
+
+  package.json:
+    json:
+      force:
+        author: "{AUTHOR_NAME} <{AUTHOR_EMAIL}> ({AUTHOR_WEB})"
+        license: "{SPDX}"
+      sortPackage: true
+
+    siblingLicenseFile:
+      name: "LICENSE"
+
+  README.md:
+    footer:
+      startBlock: "\n\n## License\n\n"
+      linePrefix: ""
+      endBlock: ""
+      license: "[{SPDX}]({LICENSE_URL})"
+
+ignore:
+- .git
+- .vscode
+- node_modules
+
+license: |
+  Copyright (c) {YEARS} {AUTHOR_NAME} {AUTHOR_WEB}
+
+  Redistribution and use in source and binary forms, with or without
+  modification, are permitted provided that the following conditions are met:
+
+  1. Redistributions of source code must retain the above copyright notice,
+    this list of conditions and the following disclaimer.
+
+  2. Redistributions in binary form must reproduce the above copyright notice,
+    this list of conditions and the following disclaimer in the documentation
+    and/or other materials provided with the distribution.
+
+  3. Neither the name of the copyright holder nor the names of its contributors
+    may be used to endorse or promote products derived from this software
+    without specific prior written permission.
+
+  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+  DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+  FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+  DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+  SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+  CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+  OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+  SPDX-License-Identifier: {SPDX}
+
+substitutions:
+  YEARS:
+    start: 2020
+    end: CURRENT_YEAR
+  AUTHOR_NAME: "Pryv S.A"
+  AUTHOR_EMAIL: info@pryv.com
+  AUTHOR_WEB: https://pryv.com
+  LICENSE_URL: https://github.com/pryv/pryv-boiler/blob/master/LICENSE
+  SPDX: BSD-3-Clause

package/LICENSE

@@ -0,0 +1,28 @@
+Copyright (c) 2020–2022 Pryv S.A https://pryv.com
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice,
+  this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors
+  may be used to endorse or promote products derived from this software
+  without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+SPDX-License-Identifier: BSD-3-Clause

package/README.md

@@ -0,0 +1,168 @@
+# Pryv config and logging boilerplate for Node.js
+
+
+## Usage
+
+
+### Initialization
+
+The "boiler" must be initialized with the application name and configuration files settings, before invoking `getConfig()` or `getLogger()`:
+
+```js
+require('@pryv/boiler').init({
+  appName: 'my-app', // This will will be prefixed to any log messages
+  baseConfigDir: path.resolve(__dirname, '../config'),
+  extraConfigs: [{
+    scope: 'extra-config',
+    file: path.resolve(__dirname, '../config/extras.js')
+  }]
+});
+```
+
+
+### Configuration
+
+We use [nconf](https://www.npmjs.com/package/nconf).
+
+#### Base config directory structure
+
+During initialization `default-config.yml` and `${NODE_ENV}-config.yml` will be loaded if present in `baseConfigDir`.
+
+#### Loading order and precedence
+
+The order configuration sources are loaded is important. Each source is loaded with a "scope" name, which can be used to replace the source's contents at any point of time.
+
+The configuration sources are loaded in the following order, the first taking precedence:
+
+1. **'test'**: Empty slot reserved for tests to override any other config parameter
+2. **'argv'**: Command line arguments
+3. **'env'**: Environment variables
+4. **'base'**: File `${NODE_ENV}-config.yml` (if present) or specified by the `--config` command line argument
+5. **'extra-config'**: Additional source(s) as specified by the `extraConfigs` option (see below)
+6. Defaults:
+   - **'default-file'**: `${baseConfigDir}/default-config.yml`
+   - **'defaults'**: Hard-coded defaults for logger
+
+#### Extra configurations
+
+Additional configuration sources can be set or reserved at initialization with the `extraConfigs` option, which expects an array of objects with one of the following structures:
+
+- File: `{scope: <name>, file: <path to file> }`. Accepts `.yml`, `.json` and `.js` files. Note `.js` content is loaded with `require(<path to file>)`.
+- Data: `{scope: <name>, key: <optional key>, data: <object> }`. If `key` is provided, the content of `data` will be accessible by this key, otherwise it is loaded at the root of the configuration.
+- Remote URL: `{scope: <name>, key: <optional key>, url: <URL to json content> }`. The JSON contents of this URL will be loaded asynchronously.
+- URL from key: `{scope: <name>, key: <optional key>, urlFromKey: <key> }`. Similar to remove URL, with the URL obtained from an existing configuration key.
+
+#### Working with the configuration
+
+Retrieving the configuration object:
+
+```javascript
+// synchronous loading
+const { getConfigUnsafe } = require('@pryv/boiler'); // Until all asynchronous sources such as URL are loaded, items might not be available
+const config = await getConfigUnsafe();
+
+// asynchronous loading
+const { getConfig } = require('@pryv/boiler');
+const config = await getConfig(); // Here we can be sure all items are fully loaded
+```
+
+Retrieving settings:
+
+```javascript
+// configuration content is {foo: { bar: 'hello'}};
+const foo = config.get('foo'); // {bar: 'hello'}
+const bar = config.get('foo:bar'); // 'hello'
+
+const barExists = config.has('bar'); // true
+```
+
+Assigning settings:
+
+```javascript
+// configuration content is {foo: { bar: 'hello'}};
+config.set('foo', 'bye bye'); // {bar: 'hello'}
+const foo = config.get('foo'); // 'bye bye'
+```
+
+Changing a scope's contents:
+
+```javascript
+// configuration content is {foo: { bar: 'hello'}};
+config.get('foo'); // {bar: 'hello'}
+// replace 'test' existing content (test is always present as the topmost configuration source)
+config.replaceScopeConfig('test', {foo: 'test'});
+config.get('foo'); // 'test'
+// reset content of 'test' scope
+config.replaceScopeConfig('test', {});
+config.get('foo'); // {bar: 'hello'}
+
+// Note: for 'test' scope there is a "sugar" function with config.injectTestConfig(object)
+```
+
+#### "Learn" mode
+
+To help detect unused configuration settings, a "learn" mode can be activated to track all calls to `config.get()` in files.
+
+Example when running tests:
+```
+export CONFIG_LEARN_DIR="{absolute path}/service-core/learn-config"
+yarn test
+```
+
+
+### Logging
+
+All messages are prefixed by the `appName` value provided at initialization (see above)). `appName` can be postfixed with a string by setting the environment variable `PRYV_BOILER_SUFFIX`, which is useful when spawning several concurrent processes of the same application.
+
+#### Using the logger
+
+```javascript
+const {getLogger} = require('@pryv/boiler');
+
+logger.info('Message', item); // standard log
+logger.warn('Message', item); // warning
+logger.error('Message', item); // warning
+logger.debug('Message', item); // debug
+
+logger.getLogger('sub'); // new logger name spaced with parent, here '{appName}:sub'
+```
+
+`logger.info()`, `logger.warn()` and `logger.error()` use [Winston](https://www.npmjs.com/package/winston) `logger.debug()` is based on [debug](https://www.npmjs.com/package/debug).
+
+#### Outputting debug messages
+
+Set the `DEBUG` environment variable. For example: `DEBUG="*" node app.js` will output all debug messages.
+
+As "debug" is a widely used package, you might get way more debug lines than expected, so you can use the `appName` property to only output messages from your application code: `DEBUG="<appName>*" node app.js`
+
+#### Log configuration sample
+
+```javascript
+logs: {
+    console: {
+      active: true,
+      level: 'info',
+      format: {
+        color: true,
+        time: true,
+        aligned: true
+      }
+    },
+    file: {
+      active: true,
+      path: 'application.log'
+    }
+  }
+```
+
+
+## Contributing
+
+`npm run lint` lints the code with [Semi-Standard](https://github.com/standard/semistandard).
+
+`npm run license` updates license information.
+
+
+## License
+
+[BSD-3-Clause](https://github.com/pryv/pryv-boiler/blob/master/LICENSE)

package/examples/configs/default-config.yml

@@ -0,0 +1,7 @@
+---
+foo:
+  bar: 'Hello Bar'
+  foo: 'Hello Foo'
+service:
+  name: 'To be overriden by Async Service load'
+default-yaml: 'default-yaml loaded'

package/examples/configs/extra-config.js

@@ -0,0 +1,8 @@
+/**
+ * @license
+ * [BSD-3-Clause](https://github.com/pryv/pryv-boiler/blob/master/LICENSE)
+ */
+
+module.exports = {
+  'extra-js': 'extra-js loaded'
+};

package/examples/configs/extra-config.json

@@ -0,0 +1,3 @@
+{
+  "extra-json": "extra-config json loaded"
+}

package/examples/configs/extra-config.yml

@@ -0,0 +1,2 @@
+---
+extra-yaml: 'Extra YAML loaded'

package/examples/configs/extra-js-async.js

@@ -0,0 +1,11 @@
+/**
+ * @license
+ * [BSD-3-Clause](https://github.com/pryv/pryv-boiler/blob/master/LICENSE)
+ */
+
+module.exports = async function () {
+  await new Promise(resolve => setTimeout(resolve, 100));
+  return {
+    'extra-js-async': 'extra-js-async loaded'
+  };
+};

package/examples/configs/test-config.yml

@@ -0,0 +1,16 @@
+---
+logs:
+  console:
+    active: true
+    level: debug
+    format:
+      color: true
+      time: false
+      aligned: true
+
+  file:
+    active: true
+    path: test-application.log
+
+foo:
+  bar: 'Hello Test'

package/examples/doubleinit.js

@@ -0,0 +1,9 @@
+/**
+ * @license
+ * [BSD-3-Clause](https://github.com/pryv/pryv-boiler/blob/master/LICENSE)
+ */
+/* eslint-disable */
+const boiler = require('../src');
+const config = boiler.init({ appName: 'sample-double-init' });
+
+boiler.init({ appName: 'sample-double-init2' });

package/examples/index.js

@@ -0,0 +1,100 @@
+/**
+ * @license
+ * [BSD-3-Clause](https://github.com/pryv/pryv-boiler/blob/master/LICENSE)
+ */
+const path = require('path');
+const boiler = require('../src');
+const { getConfigUnsafe, getLogger, getConfig } = require('../src').init({
+  appName: 'sample',
+  baseConfigDir: path.resolve(__dirname, './configs'),
+  extraConfigs: [{
+    scope: 'airbrake',
+    key: 'logs',
+    data: {
+      airbrake: {
+        active: false,
+        projectId: 319858,
+        key: '44ca9a107f4546505c7e24c8c598b0c7'
+      }
+    }
+  }, {
+    scope: 'extra1',
+    file: path.resolve(__dirname, './configs/extra-config.yml')
+  }, {
+    scope: 'extra2',
+    file: path.resolve(__dirname, './configs/extra-config.json')
+  }, {
+    scope: 'extra3',
+    file: path.resolve(__dirname, './configs/extra-config.js')
+  }, {
+    scope: 'extra4',
+    data: {
+      'extra-4-data': 'extra 4 object loaded'
+    }
+  }, {
+    scope: 'extra5',
+    key: 'extra-5-data',
+    data: 'extra 5 object loaded'
+  },
+  {
+    scope: 'extra-js-async',
+    fileAsync: path.resolve(__dirname, './configs/extra-js-async.js')
+  }, {
+    scope: 'pryv.li',
+    url: 'https://reg.pryv.li/service/info'
+  }, {
+    scope: 'pryv.me',
+    key: 'service',
+    url: 'https://reg.pryv.me/service/info'
+  }, {
+    scope: 'pryv.me-def',
+    key: 'definitions',
+    urlFromKey: 'service:assets:definitions'
+  }, {
+    scope: 'ondisk-scope',
+    key: 'ondisk',
+    url: 'file://' + path.resolve(__dirname, './remotes/ondisk.json')
+  }, {
+    plugin: require('./plugins/plugin-sync')
+  }, {
+    pluginAsync: require('./plugins/plugin-async')
+  }]
+}, function () {
+  console.log('Ready');
+});
+
+const config = getConfigUnsafe(true);
+
+const rootLogger = getLogger();
+rootLogger.debug('hello root');
+
+const indexLogger = getLogger('index');
+indexLogger.debug('hello index');
+indexLogger.info('extra Yaml', config.get('extra-yaml'));
+indexLogger.info('extra Json', config.get('extra-json'));
+indexLogger.info('extra Js', config.get('extra-js'));
+indexLogger.info('extra 4 data', config.get('extra-4-data'));
+indexLogger.info('extra 5 data', config.get('extra-5-data'));
+indexLogger.info('default yaml', config.get('default-yaml'));
+indexLogger.info('Default Service Name', config.get('service:name'));
+
+config.replaceScopeConfig('extra5', { 'extra-5-data': 'new Extra 5 data' });
+indexLogger.info('extra 5 data', config.get('extra-5-data'));
+
+const subLogger = indexLogger.getLogger('sub');
+subLogger.debug('hello sub');
+indexLogger.info('plugin sync', config.get('plugin-sync'));
+indexLogger.info('hide stuff auth=c08r0xs95xlb1xgssmp6tr7c0000gp', { password: 'toto' });
+
+(async () => {
+  await getConfig();
+  await boiler.notifyAirbrake('Hello');
+  indexLogger.info('pryv.li serial: ', config.get('serial'));
+  indexLogger.info('pryv.me name: ', config.get('service:name'));
+  indexLogger.info('Favicon: ', config.get('definitions:favicon:default:url'));
+  indexLogger.info('OnDisk: ', config.get('ondisk'));
+  indexLogger.info('Plugin async: ', config.get('plugin-async'));
+  indexLogger.info('Service Name', config.get('service:name'));
+
+  indexLogger.info('Scope of foo', config.getScopeAndValue('foo'));
+})();

package/examples/plugins/plugin-async.js

@@ -0,0 +1,10 @@
+/**
+ * @license
+ * [BSD-3-Clause](https://github.com/pryv/pryv-boiler/blob/master/LICENSE)
+ */
+module.exports = {
+  load: async function (store) {
+    store.set('plugin-async', 'plugin async loaded');
+    return 'plugin-async'; // my name
+  }
+};

package/examples/plugins/plugin-sync.js

@@ -0,0 +1,10 @@
+/**
+ * @license
+ * [BSD-3-Clause](https://github.com/pryv/pryv-boiler/blob/master/LICENSE)
+ */
+module.exports = {
+  load: function (store) {
+    store.set('plugin-sync', 'plugin sync loaded');
+    return 'plugin-sync'; // my name
+  }
+};

package/examples/remotes/ondisk.json

@@ -0,0 +1,3 @@
+{
+  "tom": "bob"
+}

package/package.json

@@ -0,0 +1,37 @@
+{
+  "name": "@pryv/boiler",
+  "version": "1.0.8",
+  "private": false,
+  "description": "Logging and config boilerplate library for Node.js apps and services at Pryv",
+  "keywords": [
+    "Pryv",
+    "Config",
+    "Logging"
+  ],
+  "homepage": "https://github.com/pryv/pryv-boiler#readme",
+  "bugs": {
+    "url": "https://github.com/pryv/pryv-boiler/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/pryv/pryv-boiler.git"
+  },
+  "license": "BSD-3-Clause",
+  "author": "Pryv S.A <info@pryv.com> (https://pryv.com)",
+  "main": "src/index",
+  "scripts": {
+    "license": "source-licenser -c .licenser.yml .",
+    "lint": "semistandard"
+  },
+  "dependencies": {
+    "@airbrake/node": "^1.4.2",
+    "debug": "^4.3.4",
+    "js-yaml": "^4.0.0",
+    "nconf": "^0.12.0",
+    "semistandard": "^16.0.1",
+    "source-licenser": "^2.0.4",
+    "superagent": "^7.1.2",
+    "winston": "^3.7.2",
+    "winston-daily-rotate-file": "^4.6.1"
+  }
+}

package/src/airbrake.js

@@ -0,0 +1,52 @@
+/**
+ * @license
+ * [BSD-3-Clause](https://github.com/pryv/pryv-boiler/blob/master/LICENSE)
+ */
+
+const { Notifier } = require('@airbrake/node');
+
+let airbrake;
+let logger;
+
+function setUpAirbrakeIfNeeded (config, rootLogger) {
+  if (airbrake) {
+    rootLogger.debug('Skipping airBrake setup (already done)');
+    return;
+  }
+  logger = rootLogger;
+
+  const airBrakeSettings = config.get('logs:airbrake');
+  if (airBrakeSettings.active) {
+    airbrake = new Notifier({
+      projectId: airBrakeSettings.projectId,
+      projectKey: airBrakeSettings.key,
+      environment: 'production'
+    });
+    logger.debug('Airbrake active with projectId: ', airBrakeSettings);
+  }
+
+  // Catch uncaught Promise rejections
+  process.on('unhandledRejection', (reason) => {
+    throw reason;
+  });
+
+  // Catch uncaught Exceptions
+  process.on('uncaughtException', async (error) => {
+    logger.error('uncaughtException', error);
+    await notifyAirbrake(error);
+    if (process.env.NODE_ENV !== 'test') process.exit(1);
+  });
+}
+
+async function notifyAirbrake () {
+  if (airbrake != null && typeof airbrake.notify === 'function') {
+    await airbrake.notify(...arguments);
+  } else {
+    logger.debug('Skipping notifyAirbake', ...arguments);
+  }
+}
+
+module.exports = {
+  setUpAirbrakeIfNeeded: setUpAirbrakeIfNeeded,
+  notifyAirbrake: notifyAirbrake
+};

package/src/compute-learn.js

@@ -0,0 +1,109 @@
+/**
+ * @license
+ * [BSD-3-Clause](https://github.com/pryv/pryv-boiler/blob/master/LICENSE)
+ */
+/**
+ * read all files in CONFIG_LEARN_DIR and output a readable result
+ */
+
+const path = require('path');
+const learnDir = process.env.CONFIG_LEARN_DIR || path.resolve(__dirname, '../../../learn-config');
+console.log('Looking for learning files in: ' + learnDir);
+const fs = require('fs');
+
+const apps = {};
+
+// get all files
+const files = fs.readdirSync(learnDir);
+
+for (const file of files) {
+  if (file.endsWith('-calls.csv')) {
+    handleCSV(path.join(learnDir, file));
+  }
+}
+
+for (const file of files) {
+  if (file.endsWith('-config.json')) {
+    handleConfig(path.join(learnDir, file));
+  }
+}
+
+function handleConfig (file) {
+  /* eslint-disable-next-line no-useless-escape */
+  const appNameSearch = /.*\/([a-zA-Z\-]*)[0-9]{1,2}-config.json/;
+  const appName = file.match(appNameSearch)[1];
+  const config = require(file).config;
+  const calls = apps[appName].calls;
+
+  checkExistsAndFlag(config, calls);
+
+  function checkExistsAndFlag (configItem, callsItem) {
+    if (typeof configItem !== 'object' || Array.isArray(configItem)) return;
+    for (const key of Object.keys(configItem)) {
+      if (key !== 'calls') {
+        if (typeof callsItem[key] === 'undefined') {
+          callsItem[key] = 'UNUSED';
+          // console.log(callsItem)
+        } else {
+          checkExistsAndFlag(configItem[key], callsItem[key]);
+        }
+      }
+    }
+  }
+}
+
+function handleCSV (file) {
+  /* eslint-disable-next-line no-useless-escape */
+  const appNameSearch = /.*\/([a-zA-Z\-]*)[0-9]{1,2}-calls.csv/;
+  const appName = file.match(appNameSearch)[1];
+
+  // initialize apps.appname if needed
+  if (!apps[appName]) {
+    apps[appName] = {
+      calls: {},
+      rank: {}
+    };
+  }
+
+  const filelines = fs.readFileSync(file, 'utf-8').split('\n');
+  for (const line of filelines) {
+    // -- calls count
+    const [path, call] = line.split(';');
+    const key = deepFind(apps[appName].calls, path + ':calls');
+    if (!key[call]) key[call] = 0;
+    key[call]++;
+    // -- ranking
+    apps[appName].rank[line] = key[call];
+  }
+}
+
+function deepFind (obj, path) {
+  const paths = path.split(':');
+  let current = obj;
+  let i;
+
+  for (i = 0; i < paths.length; ++i) {
+    if (current[paths[i]] === undefined) {
+      current[paths[i]] = {}; // initialize path while searching
+    }
+    current = current[paths[i]];
+  }
+  return current;
+}
+
+// sort and filter ranking
+const KEEP_HIGHER_N = 10;
+
+for (const appName of Object.keys(apps)) {
+  const app = apps[appName];
+  const arrayOfCalls = [];
+  for (const callLine of Object.keys(app.rank)) {
+    arrayOfCalls.push({ count: app.rank[callLine], line: callLine });
+  }
+  const arrayOfCallsSorted = arrayOfCalls.sort((a, b) => { return b.count - a.count; });
+  // replace rank info
+  app.rank = arrayOfCallsSorted.slice(0, KEEP_HIGHER_N);
+}
+
+fs.writeFileSync(path.join(learnDir, 'compute.json'), JSON.stringify(apps, null, 2));
+// console.log(apps);