@emartech/aggregate-metric-logger 1.0.0

package/.eslintrc.js

@@ -0,0 +1,24 @@
+'use strict';
+
+module.exports = {
+  env: {
+    es6: true,
+    browser: true,
+    node: true
+  },
+  parserOptions: {
+    ecmaVersion: 2018
+  },
+  extends: ['eslint:recommended', 'plugin:jest/recommended', 'emarsys'],
+  rules: {
+    'curly': ['error', 'multi-line']
+  },
+  overrides: [
+    {
+      files: ['*.spec.js'],
+      env: {
+        jest: true
+      }
+    }
+  ]
+};

package/.nvmrc

@@ -0,0 +1 @@
+v16.14.0

package/LICENSE.txt

@@ -0,0 +1,7 @@
+Copyright 2022 Emarsys
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package/README.md

@@ -0,0 +1,80 @@
+# aggregate-metric-logger
+aggregate metrics in memory and log them every minute
+
+## Setup
+
+Set `METRIC_LOGGER_ENABLED` env variable to `true` for aggregate-metric-logger to
+start collecting and logging measuremnts.
+
+To change the default `aggregate-metric-logger` namespace used for logs, set
+`METRIC_LOGGER_NAMESPACE` to a custom value.
+
+## Usage
+
+The package exposes similar logging methods as the `@emartech/json-logger` package:
+`trace`, `debug`, `info`, `warn`, `error` and `fatal`.
+
+When these are called, it will aggregate logs with the same parameters withing a minute, add
+an extra `count` property to the logs and log them at the end of the minute with
+`@emartech/json-logger`.
+
+```js
+const metrictLogger = require('aggregate-metric-logger')
+
+metricLogger.warn('etwas-went-wrong', { customer_id: 11 });
+metricLogger.warn('etwas-went-wrong', { customer_id: 11 });
+
+// will aggrete these to a single warning log
+```
+
+### Measure values
+
+If you want aggregate measuremts about something you need to simply call the `measure` method
+for each value:
+
+```js
+const metrictLogger = require('aggregate-metric-logger')
+
+metricLogger.measure('thing-to-measure', 14)
+metricLogger.measure('thing-to-measure', 20)
+```
+
+Every minute metric logger will log `min`, `max`, `sum`, `count` and `average` aggregated metrics
+for each measured tags. (Uses `@emartech/json-logger` behind the scenes for logging).
+
+### Measuring durations
+
+For duration measuremts, there are two convinience methods: `start`, and `stop`;
+
+You start the measurement with `start`, the counted value for aggregation will be the duration
+between this `start` and the matcing `stop` call;
+
+```js
+function exampleIOHeavyFunction() {
+  const measurementId = metricLogger.start('tag-for-the-measuement');
+  const result = db.findItems();
+  metricLogger.stop(measurementId);
+  return result;
+}
+```
+
+### Group by extra parameters
+
+If you want to group by additional parameters inside a tag, you can pass down these extra
+params in an object to `count` and `start` as the last optional argument:
+
+```js
+metricLogger.count('thing-to-count', 14, { customer_id: 12 });
+
+const measurementId = metricLogger.start(
+  'tag-for-the-measuement',
+  { event_type: 'nyul', account: 3 }
+);
+```
+
+A separete log entry will be created for each seen value combinations seen in the extra params
+and these extra params will appended to the log entry.
+
+Be very carefull when using extra params, and only include information you really need.
+If you include too many parameters, you could loose the advantage you gain by aggregating your logs
+with this package.

package/package.json

@@ -0,0 +1,44 @@
+{
+  "name": "@emartech/aggregate-metric-logger",
+  "version": "1.0.0",
+  "description": "aggregate metrics in memory for logging",
+  "main": "src/index.js",
+  "scripts": {
+    "test": "npm run lint && npm run test:once",
+    "lint": "eslint src/",
+    "test:once": "jest",
+    "test:watch": "jest --watch --bail",
+    "semantic-release": "CI=true semantic-release"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/emartech/aggregate-metric-logger.git"
+  },
+  "keywords": [
+    "metric",
+    "logger",
+    "aggregate"
+  ],
+  "author": "RTI team",
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/emartech/aggregate-metric-logger/issues"
+  },
+  "homepage": "https://github.com/emartech/aggregate-metric-logger#readme",
+  "dependencies": {
+    "@emartech/json-logger": "^3.1.0",
+    "date-fns": "^2.3.0",
+    "json-stable-stringify": "^1.0.1",
+    "lodash": "^4.17.15",
+    "uuid": "8.3.2"
+  },
+  "devDependencies": {
+    "@sinonjs/fake-timers": "9.1.1",
+    "@types/jest": "27.4.1",
+    "eslint": "^8.12.0",
+    "eslint-config-emarsys": "^5.1.0",
+    "eslint-plugin-jest": "^26.1.3",
+    "jest": "27.5.1",
+    "semantic-release": "^19.0.2"
+  }
+}

package/repo-info.json

@@ -0,0 +1,7 @@
+{
+  "is_in_production": true,
+  "is_scannable": true,
+  "is_critical": true,
+  "contact": "g-gsuite-rti@emarsys.com",
+  "hosted": null
+}

package/src/index.js

@@ -0,0 +1,29 @@
+'use strict';
+
+const metricLoggerFactory = require('./metric-logger');
+
+const enabled = process.env.METRIC_LOGGER_ENABLED === 'true';
+const inProgressMeasurementWarningLimit =
+  parseInt(process.env.METRIC_LOGGER_IN_PROGRESS_MEASUREMENT_WARNING_LIMIT, 10) || 10000;
+
+const exposedInstance = metricLoggerFactory({
+  enabled,
+  namespace: process.env.METRIC_LOGGER_NAMESPACE || 'aggregate-metric-logger',
+  inProgressMeasurementWarningLimit
+});
+const exposedFactoryInstanceHybrid = namespace => metricLoggerFactory({ enabled, namespace });
+
+exposedFactoryInstanceHybrid.setThresholds = exposedInstance.setThresholds;
+exposedFactoryInstanceHybrid.measure = exposedInstance.measure;
+exposedFactoryInstanceHybrid.count = exposedInstance.count;
+exposedFactoryInstanceHybrid.trace = exposedInstance.trace;
+exposedFactoryInstanceHybrid.debug = exposedInstance.debug;
+exposedFactoryInstanceHybrid.info = exposedInstance.info;
+exposedFactoryInstanceHybrid.warn = exposedInstance.warn;
+exposedFactoryInstanceHybrid.error = exposedInstance.error;
+exposedFactoryInstanceHybrid.fatal = exposedInstance.fatal;
+exposedFactoryInstanceHybrid.start = exposedInstance.start;
+exposedFactoryInstanceHybrid.stop = exposedInstance.stop;
+exposedFactoryInstanceHybrid.cancel = exposedInstance.cancel;
+
+module.exports = exposedFactoryInstanceHybrid;

package/src/metric-logger.js

@@ -0,0 +1,145 @@
+'use strict';
+
+const { v4: uuid } = require('uuid');
+const loggerFactory = require('@emartech/json-logger');
+const { getSeconds, startOfMinute, addSeconds } = require('date-fns');
+const { omit, pick, fromPairs, toPairs } = require('lodash');
+const stringify = require('json-stable-stringify');
+const util = require('util');
+
+module.exports = ({
+  enabled = true,
+  namespace = 'aggregate-metric-logger',
+  inProgressMeasurementWarningLimit = 10000
+} = {}) => {
+  const logger = loggerFactory(namespace);
+
+  const thresholdsByTag = {};
+  let measurements = {};
+  let metrics = {};
+  let initialized = false;
+
+  const flush = () => {
+    logMetrics();
+    const inProgressMeasurementCount = Object.keys(measurements).length;
+    if (inProgressMeasurementCount > inProgressMeasurementWarningLimit) {
+      logger.warn('too-many-in-progress-metric-log-measurements', { count: inProgressMeasurementCount });
+    }
+    metrics = {};
+    setupNextFlush();
+  };
+
+  const setupNextFlush = () => {
+    const secondsInCurrentMinute = getSeconds(new Date());
+    const flushOffset = secondsInCurrentMinute >= 30 ? 90 : 30;
+    const targetFlushtime = addSeconds(startOfMinute(new Date()), flushOffset);
+    const timeout = targetFlushtime - Date.now();
+    setTimeout(flush, timeout);
+  };
+
+  const incrementThresholdCount = (tag, value, countsSoFar) => {
+    const thresholds = thresholdsByTag[tag];
+    if (!thresholds) return countsSoFar;
+
+    return fromPairs(thresholds.map(threshold => {
+      const previousCount = countsSoFar[threshold] || 0;
+      const newCount = value > threshold ? previousCount + 1 : previousCount;
+      return [threshold, newCount];
+    }));
+  };
+
+  const addMeasurement = ({ tag, value, params, method = 'info', countOnly = false }) => {
+    if (!enabled) return;
+
+    if (!initialized) {
+      setupNextFlush();
+      initialized = true;
+    }
+
+    const key = `${tag}_${stringify(params)}`;
+    if (!metrics[key]) {
+      const metricForKey = { method, tag, ...params, count: 1 };
+      if (!countOnly) {
+        metricForKey.min = metricForKey.max = metricForKey.sum = metricForKey.average = value;
+        metricForKey.thresholdCounts = incrementThresholdCount(tag, value, {});
+      }
+      metrics[key] = metricForKey;
+    } else {
+      const metricsSoFar = metrics[key];
+      metrics[key].count = metricsSoFar.count + 1;
+      if (!countOnly) {
+        metrics[key].min = Math.min(metricsSoFar.min, value);
+        metrics[key].max = Math.max(metricsSoFar.max, value);
+        metrics[key].sum = metricsSoFar.sum + value;
+        metrics[key].average = metrics[key].sum / metrics[key].count;
+        metrics[key].thresholdCounts = incrementThresholdCount(
+          tag,
+          value,
+          metricsSoFar.thresholdCounts
+        );
+      }
+    }
+  };
+
+  const logMetrics = () => {
+    Object.values(metrics).forEach(metrics => {
+      const params = {
+        is_metric: true,
+        ...omit(metrics, ['tag', 'method', 'thresholdCounts']),
+        ...fromPairs(
+          toPairs(metrics.thresholdCounts).map(([limit, count]) => [`above_${limit}`, count])
+        )
+      };
+      logger[metrics.method](metrics.tag, params);
+    });
+  };
+
+  return {
+    setThresholds(tag, thresholds) {
+      thresholdsByTag[tag] = thresholds;
+    },
+    measure(tag, value, params = {}) {
+      addMeasurement({ tag, value, params });
+    },
+    count: util.deprecate(
+      (tag, value, params = {}) => addMeasurement({ tag, value, params }),
+      'count method is deprecated, use measure instead'
+    ),
+    trace(tag, params = {}) {
+      addMeasurement({ method: 'trace', tag, params, value: 1, countOnly: true });
+    },
+    debug(tag, params = {}) {
+      addMeasurement({ method: 'debug', tag, params, value: 1, countOnly: true });
+    },
+    info(tag, params = {}) {
+      addMeasurement({ method: 'info', tag, params, value: 1, countOnly: true });
+    },
+    warn(tag, params = {}) {
+      addMeasurement({ method: 'warn', tag, params, value: 1, countOnly: true });
+    },
+    error(tag, params = {}) {
+      addMeasurement({ method: 'error', tag, params, value: 1, countOnly: true });
+    },
+    fatal(tag, params = {}) {
+      addMeasurement({ method: 'fatal', tag, params, value: 1, countOnly: true });
+    },
+    start(tag, params = {}) {
+      const id = uuid();
+      measurements[id] = { tag, params, start: Date.now() };
+      return id;
+    },
+    stop(measurementId) {
+      const measurement = measurements[measurementId];
+      if (!measurement) return;
+
+      addMeasurement({
+        ...pick(measurement, ['tag', 'params']),
+        value: Date.now() - measurement.start
+      });
+      delete measurements[measurementId];
+    },
+    cancel(measurementId) {
+      delete measurements[measurementId];
+    }
+  };
+};

package/src/metric-logger.spec.js

@@ -0,0 +1,288 @@
+'use strict';
+
+const metricLoggerFactory = require('./metric-logger');
+const Logger = require('@emartech/json-logger').Logger;
+const fakeTimers = require('@sinonjs/fake-timers');
+
+jest.spyOn(Logger.prototype, 'info');
+let metricLogger;
+let clock;
+beforeEach(() => {
+  jest.resetAllMocks();
+  metricLogger = metricLoggerFactory();
+  clock = fakeTimers.install();
+});
+afterEach(() => clock.uninstall());
+
+describe('metricLogger', () => {
+  describe('measure', () => {
+    it('should log a single measurement at the next flush period', () => {
+      metricLogger.measure('elfogyasztot-tap', 4);
+
+      clock.tick(60 * 1000 + 1);
+      expect(Logger.prototype.info).toBeCalledWith('elfogyasztot-tap', {
+        min: 4,
+        max: 4,
+        count: 1,
+        sum: 4,
+        average: 4,
+        is_metric: true
+      });
+    });
+
+    it('should aggregate measurements withing a flush period', () => {
+      metricLogger.measure('elfogyasztot-tap', 4);
+      metricLogger.measure('elfogyasztot-tap', 8);
+      metricLogger.measure('kacsa-cucc', 18);
+
+      clock.tick(60 * 1000 + 1);
+      expect(Logger.prototype.info).toBeCalledWith('elfogyasztot-tap', {
+        min: 4,
+        max: 8,
+        count: 2,
+        sum: 12,
+        average: 6,
+        is_metric: true
+      });
+      expect(Logger.prototype.info).toBeCalledWith('kacsa-cucc', {
+        min: 18,
+        max: 18,
+        count: 1,
+        sum: 18,
+        average: 18,
+        is_metric: true
+      });
+    });
+
+    it('should log counts for each combination of extra params provedid within a tag', () => {
+      metricLogger.measure('elfogyasztot-tap', 4, { emberke: 'bela' });
+      metricLogger.measure('elfogyasztot-tap', 8, { emberke: 'bela' });
+      metricLogger.measure('elfogyasztot-tap', 18, { emberke: 'jano' });
+
+      clock.tick(60 * 1000 + 1);
+      expect(Logger.prototype.info).toBeCalledWith('elfogyasztot-tap', {
+        min: 4,
+        max: 8,
+        count: 2,
+        sum: 12,
+        average: 6,
+        emberke: 'bela',
+        is_metric: true
+      });
+      expect(Logger.prototype.info).toBeCalledWith('elfogyasztot-tap', {
+        min: 18,
+        max: 18,
+        count: 1,
+        sum: 18,
+        average: 18,
+        emberke: 'jano',
+        is_metric: true
+      });
+    });
+
+    it('should group logs with same tag and deeply equal params together', () => {
+      metricLogger.measure('elfogyasztot-tap', 4, { a: 1, b: 2 });
+      metricLogger.measure('elfogyasztot-tap', 8, { b: 2, a: 1 });
+
+      clock.tick(60 * 1000 + 1);
+      expect(Logger.prototype.info).toBeCalledWith('elfogyasztot-tap', {
+        min: 4,
+        max: 8,
+        count: 2,
+        sum: 12,
+        average: 6,
+        a: 1,
+        b: 2,
+        is_metric: true
+      });
+    });
+
+    it('should count event distribution within specified thresholds', () => {
+      metricLogger.setThresholds('etwas', [5, 10, 20, 1000]);
+
+      metricLogger.measure('etwas', 2);
+      metricLogger.measure('etwas', 3);
+      metricLogger.measure('etwas', 4);
+      metricLogger.measure('etwas', 6);
+      metricLogger.measure('etwas', 11);
+      metricLogger.measure('etwas', 12);
+      metricLogger.measure('etwas', 816);
+
+      clock.tick(60 * 1000 + 1);
+      expect(Logger.prototype.info).toBeCalledWith('etwas', {
+        min: 2,
+        max: 816,
+        count: 7,
+        sum: 854,
+        average: 122,
+        above_5: 4,
+        above_10: 3,
+        above_20: 1,
+        above_1000: 0,
+        is_metric: true
+      });
+    });
+  });
+
+  describe('start/stop', () => {
+    it('should log min, max, count and sum when log was called only once', () => {
+      const measurementId = metricLogger.start('utlegeles');
+      clock.tick(50);
+      metricLogger.stop(measurementId);
+
+      clock.tick(60 * 1000);
+      expect(Logger.prototype.info).toBeCalledWith('utlegeles', {
+        min: 50,
+        max: 50,
+        count: 1,
+        sum: 50,
+        average: 50,
+        is_metric: true
+      });
+    });
+
+    it('should aggregate metrics for two measurements and log after a timeout', () => {
+      const measurement1Id = metricLogger.start('utlegeles');
+      const measurement2Id = metricLogger.start('utlegeles');
+      clock.tick(50);
+      metricLogger.stop(measurement1Id);
+      clock.tick(20);
+      metricLogger.stop(measurement2Id);
+
+      clock.tick(60 * 1000);
+      expect(Logger.prototype.info).toBeCalledWith('utlegeles', {
+        min: 50,
+        max: 70,
+        sum: 120,
+        count: 2,
+        average: 60,
+        is_metric: true
+      });
+    });
+
+    it('should not count a measurment if it is cancelled with cancel', () => {
+      const measurement1Id = metricLogger.start('utlegeles');
+      const measurement2Id = metricLogger.start('utlegeles');
+      clock.tick(50);
+      metricLogger.stop(measurement1Id);
+      clock.tick(20);
+      metricLogger.cancel(measurement2Id);
+      metricLogger.stop(measurement2Id);
+
+      clock.tick(60 * 1000);
+      expect(Logger.prototype.info).toBeCalledWith('utlegeles', expect.objectContaining({
+        count: 1
+      }));
+    });
+
+    it('should flush results at the closes minute:30', () => {
+      const measurementId = metricLogger.start('utlegeles');
+      metricLogger.stop(measurementId);
+
+      clock.tick(29 * 1000 + 999);
+      expect(Logger.prototype.info).not.toBeCalled();
+      clock.tick(1 * 1000);
+      expect(Logger.prototype.info).toBeCalledWith('utlegeles', expect.anything());
+    });
+
+    it('should flush at next minute:30 if we are after 30 in current minute', () => {
+      clock.tick(45 * 1000);
+      const measurementId = metricLogger.start('utlegeles');
+      metricLogger.stop(measurementId);
+
+      clock.tick(44 * 1000);
+      expect(Logger.prototype.info).not.toBeCalled();
+      clock.tick(1 * 1000);
+      expect(Logger.prototype.info).toBeCalledWith('utlegeles', expect.anything());
+    });
+
+    it('should periodically log measuerments', () => {
+      const measurement1 = metricLogger.start('utlegeles');
+      const measurement2 = metricLogger.start('utlegeles');
+      clock.tick(10 * 1000);
+      const measurement3 = metricLogger.start('utlegeles');
+      metricLogger.stop(measurement1);
+
+      clock.tick(21 * 1000);
+      expect(Logger.prototype.info).toBeCalledWith('utlegeles', {
+        count: 1,
+        max: 10000,
+        min: 10000,
+        sum: 10000,
+        average: 10000,
+        is_metric: true
+      });
+
+      metricLogger.stop(measurement2);
+      metricLogger.stop(measurement3);
+      clock.tick(60 * 1000);
+      expect(Logger.prototype.info).toBeCalledWith('utlegeles', {
+        count: 2,
+        max: 31000,
+        min: 21000,
+        sum: 52000,
+        average: 26000,
+        is_metric: true
+      });
+    });
+
+    it('should log measurements for multiple tags and include extra parameters', () => {
+      const measurement1 = metricLogger.start('utlegeles', { a: 21 });
+      const measurement2 = metricLogger.start('nyavogas');
+      clock.tick(10 * 1000);
+      metricLogger.stop(measurement1);
+      clock.tick(5 * 1000);
+      metricLogger.stop(measurement2);
+      clock.tick(15 * 1000);
+
+      expect(Logger.prototype.info).toBeCalledWith('utlegeles', {
+        count: 1,
+        max: 10000,
+        min: 10000,
+        sum: 10000,
+        average: 10000,
+        a: 21,
+        is_metric: true
+      });
+      expect(Logger.prototype.info).toBeCalledWith('nyavogas', {
+        count: 1,
+        max: 15000,
+        min: 15000,
+        sum: 15000,
+        average: 15000,
+        is_metric: true
+      });
+    });
+
+    it('should log warning if there are too many measurements in progress', () => {
+      jest.spyOn(Logger.prototype, 'warn');
+      const metricLogger = metricLoggerFactory({ inProgressMeasurementWarningLimit: 1 });
+
+      metricLogger.measure('kacsa');
+      metricLogger.start('nyavogas');
+      metricLogger.start('nyavogas');
+      clock.tick(60 * 1000);
+
+      expect(Logger.prototype.warn).toBeCalledWith('too-many-in-progress-metric-log-measurements', { count: 2 });
+    });
+  });
+
+  describe('simple count one log methods', () => {
+    it.each(['trace', 'debug', 'info', 'warn', 'error', 'fatal'])(
+      'should log a only the count when %s method is called',
+      method => {
+        jest.spyOn(Logger.prototype, method);
+
+        metricLogger[method]('elfogyasztot-tap', { alma: 2 });
+        metricLogger[method]('elfogyasztot-tap', { alma: 2 });
+
+        clock.tick(60 * 1000 + 11500);
+        expect(Logger.prototype[method]).toBeCalledWith('elfogyasztot-tap', {
+          count: 2,
+          alma: 2,
+          is_metric: true
+        });
+      }
+    );
+  });
+});