@hyperwatch/hyperwatch 3.8.1 → 3.9.0

package/config/default.js

@@ -1,61 +1,88 @@
-const hyperWatch = require('../hyperwatch');
+const defaultConfig = function (hyperwatch) {
+  const { pipeline, input, format, logger } = hyperwatch;
 
-const { pipeline, input, format } = hyperWatch;
+  /* Init Hyperwatch (will load modules) */
 
-/* Input configuration
-====================== */
+  hyperwatch.init();
 
-/* Syslog inputs
----------------- */
+  /* Input configuration */
+  /* =================== */
 
-/* Syslog input in Nginx 'combined' format */
+  /* Syslog inputs */
+  /* ------------- */
 
-const syslogNginxCombinedInput = input.syslog.create({
-  name: 'Syslog (nginx combined format)',
-  port: 1514,
-  parse: format.nginx.parser({ format: format.nginx.formats.combined }),
-});
+  /* Syslog input in Nginx 'combined' format */
 
-pipeline.registerInput(syslogNginxCombinedInput);
+  const syslogNginxCombinedInput = input.syslog.create({
+    name: 'Syslog (nginx combined format)',
+    port: 1514,
+    parse: format.nginx.parser({ format: format.nginx.formats.combined }),
+  });
 
-/* Syslog input in Nginx 'access_watch' format */
+  pipeline.registerInput(syslogNginxCombinedInput);
 
-const syslogNginxAccessWatchInput = input.syslog.create({
-  name: 'Syslog (nginx access_watch format)',
-  port: 1515,
-  parse: format.nginx.parser({ format: format.nginx.formats.accessWatch }),
-});
+  /* Syslog input in Nginx 'hyperwatch_combined' format */
 
-pipeline.registerInput(syslogNginxAccessWatchInput);
+  const syslogNginxHyperwatchCombinedInput = input.syslog.create({
+    name: 'Syslog (nginx hyperwatch_combined format)',
+    port: 1515,
+    parse: format.nginx.parser({
+      format: format.nginx.formats.hyperwatchCombined,
+    }),
+  });
 
-/* Syslog input in Hyperwatch JSON format */
+  pipeline.registerInput(syslogNginxHyperwatchCombinedInput);
 
-const syslogInput = input.syslog.create({
-  name: 'Syslog (JSON standard format)',
-  port: 1516,
-});
+  /* Syslog input in Hyperwatch JSON format */
 
-pipeline.registerInput(syslogInput);
+  const syslogInput = input.syslog.create({
+    name: 'Syslog (JSON standard format)',
+    port: 1516,
+  });
 
-/* HTTP inputs
--------------- */
+  pipeline.registerInput(syslogInput);
 
-/* HTTP input in Hyperwatch JSON format */
+  /* HTTP inputs */
+  /* ----------- */
 
-const httpInput = input.http.create({
-  name: 'HTTP server (JSON standard format)',
-  path: '/input/log',
-});
+  /* HTTP input in Hyperwatch JSON format */
 
-pipeline.registerInput(httpInput);
+  const httpInput = input.http.create({
+    name: 'HTTP server (JSON standard format)',
+    path: '/input/log',
+  });
 
-const webSocketServerInput = input.websocket.create({
-  name: 'WebSocket server (JSON standard format)',
-  type: 'server',
-  path: '/input/log',
-});
+  pipeline.registerInput(httpInput);
 
-pipeline.registerInput(webSocketServerInput);
+  const webSocketServerInput = input.websocket.create({
+    name: 'WebSocket server (JSON standard format)',
+    type: 'server',
+    path: '/input/log',
+  });
 
-// Output to the console as JS object
-pipeline.map((log) => console.log(log.toJS()));
+  pipeline.registerInput(webSocketServerInput);
+
+  /* Pipeline configuration */
+  /* ====================== */
+
+  pipeline
+    .getNode('main')
+    .map((log) => console.log(logger.defaultFormatter.format(log, 'console')));
+};
+
+if (require.main === module) {
+  let hyperwatch;
+
+  try {
+    hyperwatch = require('../hyperwatch');
+  } catch (e) {
+    // eslint-disable-next-line node/no-missing-require
+    hyperwatch = require('@hyperwatch/hyperwatch');
+  }
+
+  defaultConfig(hyperwatch);
+
+  hyperwatch.start();
+}
+
+module.exports = defaultConfig;

package/config/example.js

@@ -1,151 +1,177 @@
-const hyperWatch = require('../hyperwatch');
+const exampleConfig = function (hyperwatch) {
+  const { pipeline, input, format, logger } = hyperwatch;
 
-const { pipeline, input, format, plugins } = hyperWatch;
+  /* Init Hyperwatch (will load modules) */
 
-/* Input configuration
-====================== */
+  hyperwatch.init();
 
-/* Syslog inputs
----------------- */
+  /* Input configuration */
+  /* =================== */
 
-/* Syslog input in Nginx 'combined' format */
+  /* Syslog inputs */
+  /* ------------- */
 
-const syslogNginxCombinedInput = input.syslog.create({
-  name: 'Syslog (nginx combined format)',
-  port: 1514,
-  parse: format.nginx.parser({ format: format.nginx.formats.combined }),
-});
+  /* Syslog input in Nginx 'combined' format */
 
-pipeline.registerInput(syslogNginxCombinedInput);
+  const syslogNginxCombinedInput = input.syslog.create({
+    name: 'Syslog (nginx combined format)',
+    port: 1514,
+    parse: format.nginx.parser({ format: format.nginx.formats.combined }),
+  });
 
-/* Syslog input in Nginx 'access_watch' format */
+  pipeline.registerInput(syslogNginxCombinedInput);
 
-const syslogNginxAccessWatchInput = input.syslog.create({
-  name: 'Syslog (nginx access_watch format)',
-  port: 1515,
-  parse: format.nginx.parser({ format: format.nginx.formats.accessWatch }),
-});
+  /* Syslog input in Nginx 'hyperwatch_combined' format */
 
-pipeline.registerInput(syslogNginxAccessWatchInput);
+  const syslogNginxHyperwatchCombinedInput = input.syslog.create({
+    name: 'Syslog (nginx hyperwatch_combined format)',
+    port: 1515,
+    parse: format.nginx.parser({
+      format: format.nginx.formats.hyperwatchCombined,
+    }),
+  });
 
-/* Syslog input in Hyperwatch JSON format */
+  pipeline.registerInput(syslogNginxHyperwatchCombinedInput);
 
-const syslogInput = input.syslog.create({
-  name: 'Syslog (JSON standard format)',
-  port: 1516,
-});
+  /* Syslog input in Hyperwatch JSON format */
 
-pipeline.registerInput(syslogInput);
+  const syslogInput = input.syslog.create({
+    name: 'Syslog (JSON standard format)',
+    port: 1516,
+  });
 
-/* Syslog input in Apache 'combined' format */
+  pipeline.registerInput(syslogInput);
 
-// const syslogApacheCombinedInput = input.syslog.create({
-//   port: 1517,
-//   parse: format.apache.parser({
-//     format: format.apache.formats.combined
-//   })
-// })
+  /* Syslog input in Apache 'combined' format */
 
-// pipeline.registerInput(syslogApacheCombinedInput)
+  const syslogApacheCombinedInput = input.syslog.create({
+    name: 'Syslog (apache combined format)',
+    port: 1517,
+    parse: format.apache.parser({
+      format: format.apache.formats.combined,
+    }),
+  });
 
-/* Syslog input in Apache 'access_watch_combined' format */
+  pipeline.registerInput(syslogApacheCombinedInput);
 
-// const syslogApacheAccessWatchCombinedInput = input.syslog.create({
-//   port: 1518,
-//   parse: format.apache.parser({
-//     format: format.apache.formats.accessWatchCombined
-//   })
-// })
+  /* Syslog input in Apache 'hyperwatch_combined' format */
 
-// pipeline.registerInput(syslogApacheAccessWatchCombinedInput)
+  const syslogApacheHyperwatchCombinedInput = input.syslog.create({
+    name: 'Syslog (apache hyperwatch_combined format)',
+    port: 1518,
+    parse: format.apache.parser({
+      format: format.apache.formats.hyperwatchCombined,
+    }),
+  });
 
-/* HTTP inputs
--------------- */
+  pipeline.registerInput(syslogApacheHyperwatchCombinedInput);
 
-/* HTTP input in Hyperwatch JSON format */
+  /* HTTP inputs */
+  /* ----------- */
 
-const httpInput = input.http.create({
-  name: 'HTTP server (JSON standard format)',
-  path: '/input/log',
-});
+  /* HTTP input in Hyperwatch JSON format */
 
-pipeline.registerInput(httpInput);
+  const httpInput = input.http.create({
+    name: 'HTTP server (JSON standard format)',
+    path: '/input/log',
+  });
 
-/* WebSocket inputs
-------------------- */
+  pipeline.registerInput(httpInput);
 
-/* WebSocket server input in Hyperwatch JSON format (listening for logs) */
+  /* WebSocket inputs */
+  /* ---------------- */
 
-// const webSocketServerInput = input.websocket.create({
-//   name: 'WebSocket server (JSON standard format)',
-//   type: 'server',
-//   path: '/input/log'
-// })
+  /* WebSocket server input in Hyperwatch JSON format (listening for logs) */
 
-// pipeline.registerInput(webSocketServerInput)
+  const webSocketServerInput = input.websocket.create({
+    name: 'WebSocket server (JSON standard format)',
+    type: 'server',
+    path: '/input/log',
+  });
 
-/* WebSocket client input in Hyperwatch JSON format (subscribing to logs) */
+  pipeline.registerInput(webSocketServerInput);
 
-// const websocketClientInput = input.websocket.create({
-//   address: 'ws://HOST:PORT/logs'
-// })
+  /* WebSocket client input in Hyperwatch JSON format (subscribing to logs) */
 
-// pipeline.registerInput(websocketClientInput)
+  // const websocketClientInput = input.websocket.create({
+  //   name: 'WebSocket client (JSON standard format)',
+  //   address: 'ws://HOST:PORT/logs',
+  // });
 
-/* File inputs
--------------- */
+  // pipeline.registerInput(websocketClientInput);
 
-/* File input in Nginx 'combined' format */
+  /* File inputs */
+  /* ----------- */
 
-// const fileNginxCombinedInput = input.file.create({
-//   path: '/var/log/nginx/access.log',
-//   parse: format.nginx.parser({
-//     format: format.nginx.formats.combined
-//   })
-// })
+  /* File input in Nginx 'combined' format */
 
-// pipeline.registerInput(fileNginxCombinedInput)
+  // const fileNginxCombinedInput = input.file.create({
+  //   name: 'File input (nginx combined format)',
+  //   path: '/var/log/nginx/access.log',
+  //   parse: format.nginx.parser({
+  //     format: format.nginx.formats.combined,
+  //   }),
+  // });
 
-/* File input in Nginx 'access_watch' format */
+  // pipeline.registerInput(fileNginxCombinedInput);
 
-// const fileNginxAccessWatchInput = input.file.create({
-//   path: '/var/log/nginx/access_watch.log',
-//   parse: format.nginx.parser({
-//     format: format.nginx.formats.accessWatch
-//   })
-// })
+  /* File input in Nginx 'hyperwatch' format */
 
-// pipeline.registerInput(fileNginxAccessWatchInput)
+  // const fileNginxHyperwatchCombinedInput = input.file.create({
+  //   name: 'File input (nginx hyperwatch_combined format)',
+  //   path: '/var/log/nginx/hyperwatch.log',
+  //   parse: format.nginx.parser({
+  //     format: format.nginx.formats.hyperwatchCombined,
+  //   }),
+  // });
 
-/*
-Pipeline configuration
-======================
-*/
+  // pipeline.registerInput(fileNginxHyperwatchCombinedInput);
 
-const { proxy } = plugins;
+  /* File input in Apache 'combined' format */
 
-const stream = pipeline
+  // const fileApacheCombinedInput = input.file.create({
+  //   name: 'File input (nginx combined format)',
+  //   path: '/var/log/apache2/access.log',
+  //   parse: format.nginx.parser({
+  //     format: format.nginx.formats.combined,
+  //   }),
+  // });
 
-  /* Filter requests */
+  // pipeline.registerInput(fileApacheCombinedInput);
 
-  // .filter(log => log.getIn(['request', 'host']) === 'example.com')
+  /* File input in Apache 'hyperwatch' format */
 
-  /* Detect the public IP address if it's behind a proxy */
+  // const fileApacheHyperwatchCombinedInput = input.file.create({
+  //   name: 'File input (nginx hyperwatch_combined format)',
+  //   path: '/var/log/apache2/hyperwatch.log',
+  //   parse: format.nginx.parser({
+  //     format: format.apache.formats.hyperwatchCombined,
+  //   }),
+  // });
 
-  .map((log) =>
-    log.set(
-      'address',
-      proxy.detectAddress(
-        log.getIn(['request', 'address']),
-        log.getIn(['request', 'headers'])
-      )
-    )
-  )
+  // pipeline.registerInput(fileApacheHyperwatchCombinedInput);
 
-  /* Output to the console as JS object */
+  /* Pipeline configuration */
+  /* ====================== */
 
-  .map((log) => console.log(log.toJS()));
-
-module.exports = {
-  stream,
+  pipeline
+    .getNode('main')
+    .map((log) => console.log(logger.defaultFormatter.format(log, 'console')));
 };
+
+if (require.main === module) {
+  let hyperwatch;
+
+  try {
+    hyperwatch = require('../hyperwatch');
+  } catch (e) {
+    // eslint-disable-next-line node/no-missing-require
+    hyperwatch = require('@hyperwatch/hyperwatch');
+  }
+
+  exampleConfig(hyperwatch);
+
+  hyperwatch.start();
+}
+
+module.exports = exampleConfig;

package/docs/configuration.md

@@ -14,7 +14,7 @@ Here is an example of how this file can look like :
 }
 ```
 
-This example would make the app be served on the 4000 port and ask the metrics to be garbage collected if they are more than one hour old.
+This example would make the app be served on the 4000 port.
 
 You can find below the list of all configurable constants:
 

package/docs/tutorials/apache_input.md

@@ -16,7 +16,15 @@ As a prerequirement, you'll need Node.js >= 7. Use nvm if you're in trouble.
 nvm install node
 ```
 
-During the beta phase, let's use Git and clone the public repository:
+#### Install from npm
+
+```bash
+npm install -g @hyperwatch/hyperwatch
+```
+
+#### Install from Git
+
+Alternatively, for developemnt purpose, you can use Git and clone the public repository:
 
 ```bash
 git clone https://github.com/hyperwatch/hyperwatch.git
@@ -30,31 +38,33 @@ In our suggested configuration, Hyperwatch will be listening for access logs in
 
 We always recommand using the `access_watch_combined` format, which is logging more detailed information and allows for a much better analysis than the regular `combined` format.
 
-To get more familiar, you can inspect default and example configurations in `./config/default.js` and `./config/example.js` file.
+To get more familiar, you can inspect default and example configurations in [`config/default.js`](<(../../config/default.js)>) and [`config/example.js`](../../config/example.js) file.
 
-Now, you can create your own configuration in `./config/apache.js`:
+Now, you can create your own configuration in `apache_syslog_example.js`:
 
 ```javascript
-const hyperWatch = require('../hyperwatch');
+module.exports = function (hyperwatch) {
+  const { pipeline, input, format } = hyperwatch;
 
-const { pipeline, input, format } = hyperWatch;
+  hyperwatch.init();
 
-const syslogApacheAccessWatchCombinedInput = input.syslog.create({
-  port: 1518,
-  parse: format.apache.parser({
-    format: format.apache.formats.accessWatchCombined,
-  }),
-});
+  const syslogApacheHyperwatchCombinedInput = input.syslog.create({
+    port: 1518,
+    parse: format.apache.parser({
+      format: format.apache.formats.hyperwatchCombined,
+    }),
+  });
 
-pipeline.registerInput(syslogApacheAccessWatchCombinedInput);
+  pipeline.registerInput(syslogApacheHyperwatchCombinedInput);
+};
 ```
 
 ### Configure Apache
 
-First, if you're following our recommendation and opted for the `access_watch_combined` format, you need to define it in the Apache configuration. This will not replace the standard log format, just create an additional one.
+First, if you're following our recommendation and opted for the `hyperwatch_combined` format, you need to define it in the Apache configuration. This will not replace the standard log format, just create an additional one.
 
 ```
-LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\" \"%{Accept}i\" \"%{Accept-Charset}i\" \"%{Accept-Encoding}i\" \"%{Accept-Language}i\" \"%{Connection}i\" \"%{Dnt}i\" \"%{From}i\" \"%{Host}i\"" access_watch_combined
+LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\" \"%{Accept}i\" \"%{Accept-Charset}i\" \"%{Accept-Encoding}i\" \"%{Accept-Language}i\" \"%{Connection}i\" \"%{Dnt}i\" \"%{From}i\" \"%{Host}i\"" hyperwatch_combined
 ```
 
 Note that you're free to use whatever `LogFormat`, you just need to properly report it in the Hyperwatch configuration.
@@ -62,18 +72,18 @@ Note that you're free to use whatever `LogFormat`, you just need to properly rep
 Second, you need to instruct Apache where to send the access logs. If it's not the same, you need to make sure that Apache can reach the server where Hyperwatch is running.
 
 ```
-CustomLog "|/usr/bin/logger -n localhost -P 1518 --rfc3164" access_watch_combined
+CustomLog "|/usr/bin/logger -n localhost -P 1518 --rfc3164" hyperwatch_combined
 ```
 
-Note: This is known to be working on _Ubuntu 16.04_ with _logger 2.27.1_, let us know if you're in trouble and using something else.
+Note: This is known to be working on _Ubuntu 16.04, 18.04_ with _logger 2.27.1, 2.31.1_, let us know if you're in trouble and using something else.
 
 In this example, there are 3 important things:
 
 1. If Hyperwatch is running on the same server, we can use `localhost` as IP address.
    If it's on a different server, replace `localhost` by the proper private or public IP address.
-2. We configured Hyperwatch to listen for syslog messages in the `access_watch_combined` format on port `1518`.
+2. We configured Hyperwatch to listen for syslog messages in the `hyperwatch_combined` format on port `1518`.
    We're properly passing that port in the configuration
-3. Finally, we're asking Apache to use the `access_watch_combined` log format we previously configured.
+3. Finally, we're asking Apache to use the `hyperwatch_combined` log format we previously configured.
 
 Don't forget to reload Aapche with the updated configuration. On Ubuntu, it would be:
 
@@ -83,10 +93,10 @@ service apache2 reload
 
 ### Start Hyperwatch
 
-Ok, now go back to where Hyperwatch is installed and start it.
+Ok, now go back to where you wrote the config.
 
 ```bash
-npm start config/apache.js
+hyperwatch apache_syslog_example.js
 ```
 
 ### Browse the interface

package/docs/tutorials/express_input.md

@@ -16,7 +16,15 @@ As a prerequirement, you'll need Node.js >= 7. Use nvm if you're in trouble.
 nvm install node
 ```
 
-During the beta phase, let's use Git and clone the public repository:
+#### Install from npm
+
+```bash
+npm install -g @hyperwatch/hyperwatch
+```
+
+#### Install from Git
+
+Alternatively, for developement purpose, you can use Git and clone the public repository:
 
 ```bash
 git clone https://github.com/hyperwatch/hyperwatch.git
@@ -28,22 +36,24 @@ npm install
 
 In our suggested configuration, Hyperwatch will be listening for access logs using the Websocket protocol.
 
-All communications between your Node/Express application and Hyperwatch will be happening in clear, so please only use that setup in your internal network. If on the public internet, we're advising to use the Websocket Secure protocol (wss) which is straightforward but out of the scope of this tutorial.
+All communications between your Node/Express application and Hyperwatch will be happening in clear, please only use that setup on your internal network. If on the public internet, we're advising to use the Websocket Secure protocol (wss) which is straightforward but out of the scope of this tutorial.
 
-Now, you can create your own configuration in `./config/config.js`:
+Now, you can create your own configuration in `express_websocket_example.js`:
 
 ```javascript
-const hyperWatch = require('../hyperwatch');
+module.exports = function (hyperwatch) {
+  const { pipeline, input } = hyperwatch;
 
-const { pipeline, input } = hyperWatch;
+  hyperwatch.init();
 
-const webSocketServerInput = input.websocket.create({
-  name: 'WebSocket server (JSON standard format)',
-  type: 'server',
-  path: '/input/log',
-});
+  const webSocketServerInput = input.websocket.create({
+    name: 'WebSocket server (JSON standard format)',
+    type: 'server',
+    path: '/input/log',
+  });
 
-pipeline.registerInput(webSocketServerInput);
+  pipeline.registerInput(webSocketServerInput);
+};
 ```
 
 ### Configure Node/Express
@@ -51,18 +61,18 @@ pipeline.registerInput(webSocketServerInput);
 Now, install the Hyperwatch Express Logger middleware in your Node application:
 
 ```bash
-npm install --save access-watch-express-logger
+npm install --save @hyperwatch/express-logger
 ```
 
 Then simply configure it like any other middlewares:
 
 ```javascript
 const express = require('express');
-const accessWatchExpressLogger = require('access-watch-express-logger');
+const hyperwatchExpressLogger = require('@hyperwatch/express-logger');
 
 const app = express();
 
-app.use(accessWatchExpressLogger('websocket', 'ws://localhost:3000/input/log'));
+app.use(hyperwatchExpressLogger('websocket', 'ws://localhost:3000/input/log'));
 ```
 
 In this example, there are 3 important things:
@@ -78,10 +88,10 @@ Note: The Hyperwatch middleware is also capable of logging using the HTTP(s) or
 
 ### Start Hyperwatch
 
-Ok, now go back to where Hyperwatch is installed and start it.
+Ok, now go back to where you wrote the config.
 
 ```bash
-npm start config/config.js
+hyperwatch express_websocket_example.js
 ```
 
 ### Browse the interface

package/package.json

@@ -1,5 +1,6 @@
 {
   "name": "@hyperwatch/hyperwatch",
+  "version": "3.9.0",
   "description": "Open Source HTTP Traffic Manager",
   "license": "Apache-2.0",
   "author": "François Hodierne <francois@hodierne.net>",
@@ -11,7 +12,6 @@
     "type": "git",
     "url": "https://github.com/hyperwatch/hyperwatch.git"
   },
-  "version": "3.8.1",
   "main": "hyperwatch.js",
   "bin": {
     "hyperwatch": "./bin/hyperwatch"
@@ -27,38 +27,39 @@
     "prettier:write": "npm run prettier -- --write"
   },
   "dependencies": {
-    "@hyperwatch/useragent": "^3.7.4",
+    "@hyperwatch/useragent": "^3.9.1",
     "accept-language-parser": "^1.5.0",
-    "ajv": "^6.12.6",
+    "ajv": "^8.11.0",
+    "ajv-formats": "^2.1.1",
     "chalk": "^4.1.0",
-    "country-code-emoji": "^1.0.0",
-    "debug": "^4.3.1",
+    "country-code-emoji": "^2.3.0",
+    "debug": "^4.3.4",
     "dnsbl": "^3.2.0",
-    "express": "^4.17.1",
-    "express-ws": "^4.0.0",
-    "geoip-lite": "^1.4.2",
-    "immutable": "^4.0.0-rc.12",
-    "ip-cidr": "^2.1.1",
-    "lodash": "^4.17.20",
-    "lru-cache": "^6.0.0",
+    "express": "^4.18.2",
+    "express-ws": "^5.0.2",
+    "geoip-lite": "^1.4.6",
+    "immutable": "^4.1.0",
+    "ip-cidr": "^3.0.10",
+    "lodash": "^4.17.21",
+    "lru-cache": "^7.14.0",
     "micro-strptime": "^0.2.3",
-    "proxy-addr": "^2.0.6",
+    "proxy-addr": "^2.0.7",
     "rc": "^1.2.8",
     "syslog-parse": "^1.3.1",
-    "tail": "^2.0.4",
-    "uuid": "^8.3.2",
-    "ws": "^7.4.1"
+    "tail": "^2.2.4",
+    "uuid": "^9.0.0",
+    "ws": "^8.9.0"
   },
   "devDependencies": {
-    "depcheck": "^1.3.1",
-    "eslint": "^7.15.0",
-    "eslint-plugin-import": "^2.22.1",
+    "depcheck": "^1.4.3",
+    "eslint": "^8.25.0",
+    "eslint-plugin-import": "^2.26.0",
     "eslint-plugin-node": "^11.1.0",
     "husky": "^4.3.5",
     "lint-staged": "^10.5.3",
-    "mocha": "^8.2.1",
-    "prettier": "^2.2.1",
-    "prettier-package-json": "^2.1.3"
+    "mocha": "^10.0.0",
+    "prettier": "^2.7.1",
+    "prettier-package-json": "^2.7.0"
   },
   "engines": {
     "node": ">=12.0"

package/src/format/apache.js

@@ -94,7 +94,7 @@ function parser({ format }) {
 
 const formats = {
   combined: '%h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-agent}i"',
-  accessWatchCombined:
+  hyperwatchCombined:
     '%h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-agent}i" "%{Accept}i" "%{Accept-Charset}i" "%{Accept-Encoding}i" "%{Accept-Language}i" "%{Connection}i" "%{Dnt}i" "%{From}i" "%{Host}i"',
 };
 

package/src/format/nginx.js

@@ -97,7 +97,7 @@ const formats = {
     '$remote_addr - $remote_user [$time_local] "$request" $status $bytes_sent "$http_referer" "$http_user_agent"',
   accessWatch:
     '"$time_iso8601" "$remote_addr" "$http_host" "$request" $status "$http_user_agent" "$http_accept" "$http_accept_language" "$http_accept_charset" "$http_accept_encoding" "$http_from" "$http_dnt" "$http_connection" "$http_referer"',
-  accessWatchCombined:
+  hyperwatchCombined:
     '$remote_addr - $remote_user [$time_local] "$request" $status $bytes_sent "$http_referer" "$http_user_agent" "$http_accept" "$http_accept_charset" "$http_accept_encoding" "$http_accept_language" "$http_connection" "$http_dnt" "$http_from" "$http_host"',
 };
 

package/src/index.js

@@ -8,17 +8,17 @@ const lib = require('./lib');
 const modules = require('./modules');
 const plugins = require('./plugins');
 
-const { cache, pipeline, util } = lib;
+const { cache, logger, pipeline, util } = lib;
 
 let initialized = false;
 
 function init(config = {}) {
   if (initialized) {
-    console.warn(`Can't init, Hyperwatch was already initialized`);
+    console.warn(`Can't init, Hyperwatch was already initialized.`);
     return;
   }
   merge(constants, config);
-  modules.load();
+  modules.init();
   initialized = true;
 }
 
@@ -27,7 +27,7 @@ function start() {
     console.warn(`Can't start, Hyperwatch was not initialized.`);
     return;
   }
-  modules.beforeStart();
+  modules.start();
   return Promise.all([app.start(), pipeline.start()]);
 }
 
@@ -42,6 +42,7 @@ module.exports = {
   format,
   input,
   lib,
+  logger,
   modules,
   pipeline,
   plugins,

package/src/lib/formatter.js

@@ -28,6 +28,9 @@ const request = (log) => {
 };
 
 const executionTime = (log, output) => {
+  if (!log.get('executionTime')) {
+    return;
+  }
   return log.get('executionTime') <= 100
     ? colorize('green', `${log.get('executionTime')}ms`, output)
     : log.get('executionTime') >= 1000

package/src/lib/pipeline.js

@@ -9,7 +9,8 @@
  *
  * This modules provides functions to build a pipeline, a tree of stream processors.
  */
-const Ajv = require('ajv');
+const Ajv = require('ajv').default;
+const addFormats = require('ajv-formats').default;
 const { Map } = require('immutable');
 
 const schema = require('../format/log-schema.json');
@@ -18,6 +19,8 @@ const monitoring = require('./monitoring');
 const { complement } = require('./util');
 
 const validator = new Ajv();
+addFormats(validator);
+
 const validate = validator.compile(schema);
 
 function errorLog(log, severity = 'warn') {

package/src/modules/address.js

@@ -11,11 +11,11 @@ function fill(log) {
   return log;
 }
 
-function load() {
+function init() {
   pipeline.getNode('main').map(fill).registerNode('main');
 }
 
-function beforeStart() {
+function start() {
   const aggregator = new Aggregator();
 
   aggregator.setIdentifier(identifier);
@@ -26,6 +26,6 @@ function beforeStart() {
 }
 
 module.exports = {
-  load,
-  beforeStart,
+  init,
+  start,
 };

package/src/modules/agent.js

@@ -66,7 +66,7 @@ const osFormat = (log) => {
   return agentOs.get('family');
 };
 
-function load() {
+function init() {
   pipeline.getNode('main').map(augment).registerNode('main');
 
   aggregator.defaultFormatter.insertFormat('agent', agentFormat, {
@@ -84,7 +84,7 @@ function load() {
 module.exports = {
   lookup,
   augment,
-  load,
+  init,
   agentFormat,
   osFormat,
 };

package/src/modules/cloudflare.js

@@ -35,11 +35,11 @@ function augment(log) {
   return log;
 }
 
-function load() {
+function init() {
   pipeline.getNode('main').map(augment).registerNode('main');
 }
 
 module.exports = {
   augment,
-  load,
+  init,
 };

package/src/modules/dnsbl.js

@@ -33,7 +33,7 @@ const xblFormat = (log, output) => {
   }
 };
 
-function load() {
+function init() {
   pipeline.getNode('main').map(augment).registerNode('main');
 
   aggregator.defaultFormatter.insertFormat('xbl', xblFormat, {
@@ -44,5 +44,5 @@ function load() {
 
 module.exports = {
   augment,
-  load,
+  init,
 };

package/src/modules/geoip.js

@@ -1,4 +1,4 @@
-const countryCodeEmoji = require('country-code-emoji');
+const { default: flag } = require('country-code-emoji');
 const geoip = require('geoip-lite');
 const { fromJS } = require('immutable');
 
@@ -30,11 +30,11 @@ const city = (log) => {
 const country = (log, output) => {
   const cc = log.getIn(['geoip', 'country']);
   if (cc) {
-    return output === 'html' ? `${countryCodeEmoji(cc)} ${cc}` : cc;
+    return output === 'html' ? `${flag(cc)} ${cc}` : cc;
   }
 };
 
-function load() {
+function init() {
   pipeline.getNode('main').map(augment).registerNode('main');
 
   aggregator.defaultFormatter.insertFormat('country', country, {
@@ -53,6 +53,6 @@ function load() {
 
 module.exports = {
   lookup,
-  load,
+  init,
   augment,
 };

package/src/modules/hostname.js

@@ -78,12 +78,12 @@ async function augment(log, { fast = false } = {}) {
   return log;
 }
 
-function load() {
+function init() {
   pipeline.getNode('main').map(augment).registerNode('main');
 }
 
 module.exports = {
   lookup,
   augment,
-  load,
+  init,
 };

package/src/modules/identity.js

@@ -74,7 +74,7 @@ function augment(log) {
         ? log.set('identity', 'WebMeUp')
         : log;
     case 'PetalBot':
-      return hostname && hostname.endsWith('.aspiegel.com')
+      return hostname && hostname.endsWith('.petalsearch.com')
         ? log.set('identity', family)
         : log;
     case 'AhrefsBot':
@@ -174,6 +174,10 @@ function augment(log) {
       return hostname && hostname.endsWith('.arquivo.pt')
         ? log.set('identity', 'Arquivo')
         : log;
+    case 'SeekportBot':
+      return hostname && hostname.endsWith('.007ac9.net')
+        ? log.set('identity', 'Seekport')
+        : log;
 
     // Per hostname + CIDR
     case 'Twitterbot':
@@ -236,6 +240,15 @@ function augment(log) {
       return hostname && hostname.endsWith('.compute-1.amazonaws.com')
         ? log.set('identity', 'Common Crawl')
         : log;
+    case 'TransferWise-Webhook':
+      return hostname &&
+        hostname.endsWith('.eu-central-1.compute.amazonaws.com')
+        ? log.set('identity', 'Wise')
+        : log;
+    case 'Amazonbot':
+      return hostname && hostname.endsWith('.compute-1.amazonaws.com')
+        ? log.set('identity', 'Amazonbot')
+        : log;
 
     // GCE
     case 'VelenPublicWebCrawler':
@@ -301,11 +314,11 @@ const identifier = (log) =>
   log.getIn(['address', 'value']) ||
   log.getIn(['request', 'address']);
 
-function load() {
+function init() {
   pipeline.getNode('main').map(augment).registerNode('main');
 }
 
-function beforeStart() {
+function start() {
   const aggregator = new Aggregator();
 
   aggregator.setIdentifier(identifier);
@@ -317,6 +330,6 @@ function beforeStart() {
 
 module.exports = {
   augment,
-  load,
-  beforeStart,
+  init,
+  start,
 };

package/src/modules/index.js

@@ -45,25 +45,25 @@ function activeModules() {
     .filter((m) => m);
 }
 
-function load() {
+function init() {
   for (const module of activeModules()) {
-    if (module && module.load) {
-      module.load();
+    if (module && module.init) {
+      module.init();
     }
   }
 }
 
-function beforeStart() {
+function start() {
   for (const module of activeModules()) {
-    if (module && module.beforeStart) {
-      module.beforeStart();
+    if (module && module.start) {
+      module.start();
     }
   }
 }
 
 module.exports = {
   get,
-  load,
-  beforeStart,
+  init,
+  start,
   activeModules,
 };

package/src/modules/language.js

@@ -29,7 +29,7 @@ const language = (log) => {
   }
 };
 
-function load() {
+function init() {
   pipeline.getNode('main').map(augment).registerNode('main');
 
   aggregator.defaultFormatter.insertFormat('language', language, {
@@ -40,5 +40,5 @@ function load() {
 
 module.exports = {
   augment,
-  load,
+  init,
 };

package/src/modules/logs.js

@@ -2,7 +2,7 @@ const { api, websocket } = require('../app');
 const { defaultFormatter } = require('../lib/logger');
 const pipeline = require('../lib/pipeline');
 
-function beforeStart() {
+function start() {
   for (const [name, stream] of Object.entries(pipeline.nodes)) {
     websocket.streamToWebsocket(`/logs/${name}`, stream, {
       name: `WebSocket (${name} logs)`,
@@ -16,4 +16,4 @@ function beforeStart() {
   }
 }
 
-module.exports = { beforeStart };
+module.exports = { start };

package/src/modules/signature.js

@@ -56,11 +56,11 @@ function augment(log) {
   return log;
 }
 
-function load() {
+function init() {
   pipeline.getNode('main').map(augment).registerNode('main');
 }
 
-function beforeStart() {
+function start() {
   const aggregator = new Aggregator();
 
   aggregator.setIdentifier((log) => log.getIn(['signature', 'id']));
@@ -127,6 +127,6 @@ function beforeStart() {
 }
 
 module.exports = {
-  load,
-  beforeStart,
+  init,
+  start,
 };

package/src/modules/sparkline.js

@@ -21,12 +21,12 @@ sparkline ('${id}', ${JSON.stringify(points)}, '#797979', 14, 5);
 </script>`;
 };
 
-function load() {
+function init() {
   aggregator.defaultFormatter.insertFormat('activity', (entry) =>
     sparkline(entry, 'per_minute')
   );
 }
 
 module.exports = {
-  load,
+  init,
 };

package/src/modules/status.js

@@ -22,7 +22,7 @@ function mapper(entry, format) {
   };
 }
 
-function beforeStart() {
+function start() {
   api.get('/status(.:format(txt|json))?', (req, res) => {
     const raw = req.query.raw ? true : false;
     const format = req.params.format || (raw ? 'json' : null);
@@ -48,4 +48,4 @@ function beforeStart() {
   });
 }
 
-module.exports = { beforeStart };
+module.exports = { start };

package/start.js

@@ -1,13 +1,19 @@
 const path = require('path');
 
+const { isFunction } = require('lodash');
+
 const hyperwatch = require('./hyperwatch');
 
 // Load configuration
 
+let config;
 if (process.argv[2]) {
-  require(path.resolve(process.cwd(), process.argv[2]));
+  config = require(path.resolve(process.cwd(), process.argv[2]));
 } else {
-  require(path.resolve(__dirname, './config/default'));
+  config = require(path.resolve(__dirname, './config/default'));
+}
+if (isFunction(config)) {
+  config(hyperwatch);
 }
 
 // Start