@emartech/aggregate-metric-logger 5.2.1 → 6.0.0

package/README.md

@@ -1,42 +1,28 @@
 # aggregate-metric-logger
-Aggregate metrics in memory and log them every minute
+
+Aggregate metrics in memory and send them to GCP them every minute.
 
 ## Setup
 
 Set `METRIC_LOGGER_ENABLED` env variable to `true` for aggregate-metric-logger to
 start collecting and logging measurements.
 
-To change the default `aggregate-metric-logger` namespace used for logs, set
-`METRIC_LOGGER_NAMESPACE` to a custom value.
-
-## Setup to send metrics to GCP monitoring
-
-(currently only the `metricLogger.count` and `metricLogger.measureGcp` methods send metrics to GCP)
-
-Set `METRIC_LOGGER_ENABLED` just us for LaaS based metrics.
-
 Set `MONITORING_PROJECT_ID` and `GCP_CREDENTIALS` env variables. `HOSTNAME` env variable also has to be
 set, but for a pod running on kubernetes this should already be set.
 
 Add `Monitoring Metrics Writer` Role to the service account in the GCP project.
 
-
 ## Usage
 
-The package exposes similar logging methods as the `@emartech/json-logger` package:
-`trace`, `debug`, `info`, `warn`, `error` and `fatal`.
-
-When these are called, it will aggregate logs with the same parameters withing a minute, add
-an extra `count` property to the logs and log them at the end of the minute with
-`@emartech/json-logger`.
+### Count occurrences
 
 ```js
 const metricLogger = require('@emartech/aggregate-metric-logger')
 
-metricLogger.warn('etwas-went-wrong', { customer_id: 11 });
-metricLogger.warn('etwas-went-wrong', { customer_id: 11 });
+metricLogger.count('etwas-went-wrong', { customer_id: 11 });
+metricLogger.count('etwas-went-wrong', { customer_id: 11 });
 
-// will aggregate these to a single warning log
+// will aggregate these to a single metric
 ```
 
 ### Measure values
@@ -47,13 +33,10 @@ for each value:
 ```js
 const metricLogger = require('@emartech/aggregate-metric-logger')
 
-metricLogger.measure('thing-to-measure', 14)
-metricLogger.measure('thing-to-measure', 20)
+metricLogger.measure('thing-to-measure', 14, { customer_id: 12 })
+metricLogger.measure('thing-to-measure', 20, { customer_id: 44 })
 ```
 
-Every minute metric logger will log `min`, `max`, `sum`, `count` and `average` aggregated metrics
-for each measured tags. (Uses `@emartech/json-logger` behind the scenes for logging).
-
 ### Measuring durations
 
 For duration measurements, there are two convenient methods: `start`, and `stop`;
@@ -63,30 +46,9 @@ between this `start` and the matching `stop` call;
 
 ```js
 function exampleIOHeavyFunction() {
-  const measurement = metricLogger.start('tag-for-the-measuement');
+  const measurement = metricLogger.start('tag-for-the-measuement', { customer_id: 23 });
   const result = db.findItems();
   metricLogger.stop(measurement);
   return result;
 }
 ```
-
-### Group by extra parameters
-
-If you want to group by additional parameters inside a tag, you can pass down these extra
-params in an object to `measure` and `start` as the last optional argument:
-
-```js
-metricLogger.measure('thing-to-count', 14, { customer_id: 12 });
-
-const measurement = metricLogger.start(
-  'tag-for-the-measuement',
-  { event_type: 'nyul', account: 3 }
-);
-```
-
-A separate log entry will be created for each seen value combinations seen in the extra params
-and these extra params will be appended to the log entry.
-
-Be very careful when using extra params, and only include information you really need.
-If you include too many parameters, you could lose the advantage you gain by aggregating your logs
-with this package.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@emartech/aggregate-metric-logger",
-  "version": "5.2.1",
+  "version": "6.0.0",
   "description": "aggregate metrics in memory for logging",
   "main": "src/index.js",
   "scripts": {
@@ -31,18 +31,16 @@
   "dependencies": {
     "@emartech/json-logger": "^8.0.3",
     "@google-cloud/monitoring": "^4.1.0",
-    "date-fns": "^3.6.0",
     "json-stable-stringify": "^1.1.1",
     "lodash": "^4.17.15",
     "ms": "^2.1.3"
   },
   "devDependencies": {
-    "@emartech/rti-server-tools": "^6.17.0",
-    "@sinonjs/fake-timers": "^11.2.2",
+    "@emartech/rti-server-tools": "^7.9.5",
     "@types/jest": "^29.5.12",
     "dotenv-haphap": "^3.1.2",
     "eslint": "^9.7.0",
-    "globals": "15.8.0",
+    "globals": "16.0.0",
     "jest": "^29.2.2",
     "semantic-release": "^24.0.0"
   }

package/renovate.json

@@ -0,0 +1,6 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": [
+    "config:recommended"
+  ]
+}

package/src/gcp-metric-collector/gcp-metric-collector.js

@@ -79,13 +79,9 @@ class GcpMetricCollector {
     await createGcpTimeSeriesPublisher(this.projectId).publishTimeSeries(timeSeries);
     this.setupNextFlush();
   }
-
-  createGcpMetricsCollector(projectId) {
-    return new GcpMetricCollector(projectId);
-  }
 }
 
-const collector = new GcpMetricCollector()
+const collector = new GcpMetricCollector();
 
 exports.count = (metricName, labels = {}, value = 1) =>
   collector.count(metricName, labels, value);
@@ -99,4 +95,4 @@ exports.stop = measurement => collector.stop(measurement);
 
 exports.reset = () => collector.reset();
 
-exports.createGcpMetricsCollector = projectId => new GcpMetricCollector(projectId)
+exports.createGcpMetricsCollector = projectId => new GcpMetricCollector(projectId);

package/src/gcp-metric-collector/gcp-metric-collector.spec.js

@@ -124,7 +124,7 @@ describe('gcpMetricCollector', () => {
           }
         ]
       ]);
-    })
+    });
 
     it('should sum up counts that happened and publish the sum, send a metric for each label combo', () => {
       stubCreateTimeSeries();

package/src/gcp-time-series-builder/gcp-time-series-builder.js

@@ -36,9 +36,9 @@ class GcpTimeSeriesBuilder {
 
 exports.createGcpTimeSeriesBuilder = projectId => {
   return new GcpTimeSeriesBuilder(projectId);
-}
+};
 
 exports.buildTimeSeries = metrics => {
   const builder = new GcpTimeSeriesBuilder();
   return builder.buildTimeSeries(metrics);
-}
+};

package/src/gcp-time-series-publisher/gcp-time-series-publisher.js

@@ -47,9 +47,9 @@ class GcpTimeSeriesPublisher {
   };
 }
 
-exports.createGcpTimeSeriesPublisher = projectId => new GcpTimeSeriesPublisher(projectId)
+exports.createGcpTimeSeriesPublisher = projectId => new GcpTimeSeriesPublisher(projectId);
 
 exports.publishTimeSeries = async timeSeries => {
-  const publisher = new GcpTimeSeriesPublisher()
+  const publisher = new GcpTimeSeriesPublisher();
   await publisher.publishTimeSeries(timeSeries);
 };

package/src/index.js

@@ -1,31 +1,11 @@
 'use strict';
 
-const metricLoggerFactory = require('./laas-metric-logger/laas-metric-logger');
 const gcpMetricCollector = require('./gcp-metric-collector/gcp-metric-collector');
 
-const enabled = process.env.METRIC_LOGGER_ENABLED === 'true';
-
-const exposedInstance = metricLoggerFactory({
-  enabled,
-  namespace: process.env.METRIC_LOGGER_NAMESPACE || 'aggregate-metric-logger'
-});
-const exposedFactoryInstanceHybrid = namespace => metricLoggerFactory({ enabled, namespace });
-
-exposedFactoryInstanceHybrid.setThresholds = exposedInstance.setThresholds;
-exposedFactoryInstanceHybrid.measure = exposedInstance.measure;
-exposedFactoryInstanceHybrid.trace = exposedInstance.trace;
-exposedFactoryInstanceHybrid.debug = exposedInstance.debug;
-exposedFactoryInstanceHybrid.info = exposedInstance.info;
-exposedFactoryInstanceHybrid.warn = exposedInstance.warn;
-exposedFactoryInstanceHybrid.error = exposedInstance.error;
-exposedFactoryInstanceHybrid.fatal = exposedInstance.fatal;
-exposedFactoryInstanceHybrid.start = exposedInstance.start;
-exposedFactoryInstanceHybrid.stop = exposedInstance.stop;
-
-exposedFactoryInstanceHybrid.count = gcpMetricCollector.count;
-exposedFactoryInstanceHybrid.measureGcp = gcpMetricCollector.measure;
-exposedFactoryInstanceHybrid.startGcp = gcpMetricCollector.start;
-exposedFactoryInstanceHybrid.stopGcp = gcpMetricCollector.stop;
-exposedFactoryInstanceHybrid.createGcpMetricsCollector = gcpMetricCollector.createGcpMetricsCollector;
-
-module.exports = exposedFactoryInstanceHybrid;
+module.exports = {
+  count: gcpMetricCollector.count,
+  measure: gcpMetricCollector.measure,
+  start: gcpMetricCollector.start,
+  stop: gcpMetricCollector.stop,
+  createGcpMetricsCollector: gcpMetricCollector.createGcpMetricsCollector
+};

package/src/laas-metric-logger/laas-metric-logger.js

@@ -1,124 +0,0 @@
-'use strict';
-
-const { createLogger } = require('@emartech/json-logger');
-const { getSeconds, startOfMinute, addSeconds } = require('date-fns');
-const { omit, pick, fromPairs, toPairs } = require('lodash');
-const stringify = require('json-stable-stringify');
-
-module.exports = ({
-  enabled = true,
-  namespace = 'aggregate-metric-logger'
-} = {}) => {
-  const logger = createLogger(namespace);
-
-  const thresholdsByTag = {};
-  let metrics = {};
-  let initialized = false;
-
-  const flush = () => {
-    logMetrics();
-    metrics = {};
-    setupNextFlush();
-  };
-
-  const setupNextFlush = () => {
-    const secondsInCurrentMinute = getSeconds(new Date());
-    const flushOffset = secondsInCurrentMinute >= 30 ? 90 : 30;
-    const targetFlushtime = addSeconds(startOfMinute(new Date()), flushOffset);
-    const timeout = targetFlushtime - Date.now();
-    setTimeout(flush, timeout);
-  };
-
-  const incrementThresholdCount = (tag, value, countsSoFar) => {
-    const thresholds = thresholdsByTag[tag];
-    if (!thresholds) return countsSoFar;
-
-    return fromPairs(thresholds.map(threshold => {
-      const previousCount = countsSoFar[threshold] || 0;
-      const newCount = value > threshold ? previousCount + 1 : previousCount;
-      return [threshold, newCount];
-    }));
-  };
-
-  const addMeasurement = ({ tag, value, params, method = 'info', countOnly = false }) => {
-    if (!enabled) return;
-
-    if (!initialized) {
-      setupNextFlush();
-      initialized = true;
-    }
-
-    const key = `${tag}_${stringify(params)}`;
-    if (!metrics[key]) {
-      const metricForKey = { method, tag, ...params, count: 1 };
-      if (!countOnly) {
-        metricForKey.min = metricForKey.max = metricForKey.sum = metricForKey.average = value;
-        metricForKey.thresholdCounts = incrementThresholdCount(tag, value, {});
-      }
-      metrics[key] = metricForKey;
-    } else {
-      const metricsSoFar = metrics[key];
-      metrics[key].count = metricsSoFar.count + 1;
-      if (!countOnly) {
-        metrics[key].min = Math.min(metricsSoFar.min, value);
-        metrics[key].max = Math.max(metricsSoFar.max, value);
-        metrics[key].sum = metricsSoFar.sum + value;
-        metrics[key].average = metrics[key].sum / metrics[key].count;
-        metrics[key].thresholdCounts = incrementThresholdCount(
-          tag,
-          value,
-          metricsSoFar.thresholdCounts
-        );
-      }
-    }
-  };
-
-  const logMetrics = () => {
-    Object.values(metrics).forEach(metrics => {
-      const params = {
-        is_metric: true,
-        ...omit(metrics, ['tag', 'method', 'thresholdCounts']),
-        ...fromPairs(
-          toPairs(metrics.thresholdCounts).map(([limit, count]) => [`above_${limit}`, count])
-        )
-      };
-      logger[metrics.method](metrics.tag, params);
-    });
-  };
-
-  return {
-    setThresholds(tag, thresholds) {
-      thresholdsByTag[tag] = thresholds;
-    },
-    measure(tag, value, params = {}) {
-      addMeasurement({ tag, value, params });
-    },
-    trace(tag, params = {}) {
-      addMeasurement({ method: 'trace', tag, params, value: 1, countOnly: true });
-    },
-    debug(tag, params = {}) {
-      addMeasurement({ method: 'debug', tag, params, value: 1, countOnly: true });
-    },
-    info(tag, params = {}) {
-      addMeasurement({ method: 'info', tag, params, value: 1, countOnly: true });
-    },
-    warn(tag, params = {}) {
-      addMeasurement({ method: 'warn', tag, params, value: 1, countOnly: true });
-    },
-    error(tag, params = {}) {
-      addMeasurement({ method: 'error', tag, params, value: 1, countOnly: true });
-    },
-    fatal(tag, params = {}) {
-      addMeasurement({ method: 'fatal', tag, params, value: 1, countOnly: true });
-    },
-    start(tag, params = {}) {
-      return { tag, params, start: Date.now() };
-    },
-    stop(measurement) {
-      addMeasurement({
-        ...pick(measurement, ['tag', 'params']),
-        value: Date.now() - measurement.start
-      });
-    }
-  };
-};

package/src/laas-metric-logger/laas-metric-logger.spec.js

@@ -1,271 +0,0 @@
-'use strict';
-
-const { Logger } = require('@emartech/json-logger');
-const fakeTimers = require('@sinonjs/fake-timers');
-const metricLoggerFactory = require('./laas-metric-logger');
-
-let metricLogger;
-let clock;
-beforeEach(() => {
-  jest.spyOn(Logger.prototype, 'info');
-  metricLogger = metricLoggerFactory();
-  clock = fakeTimers.install();
-});
-afterEach(() => clock.uninstall());
-
-describe('metricLogger', () => {
-  describe('#measure', () => {
-    it('should log a single measurement at the next flush period', () => {
-      metricLogger.measure('elfogyasztot-tap', 4);
-
-      clock.tick(60 * 1000 + 1);
-      expect(Logger.prototype.info).toHaveBeenCalledWith('elfogyasztot-tap', {
-        min: 4,
-        max: 4,
-        count: 1,
-        sum: 4,
-        average: 4,
-        is_metric: true
-      });
-    });
-
-    it('should aggregate measurements withing a flush period', () => {
-      metricLogger.measure('elfogyasztot-tap', 4);
-      metricLogger.measure('elfogyasztot-tap', 8);
-      metricLogger.measure('kacsa-cucc', 18);
-
-      clock.tick(60 * 1000 + 1);
-      expect(Logger.prototype.info).toHaveBeenCalledWith('elfogyasztot-tap', {
-        min: 4,
-        max: 8,
-        count: 2,
-        sum: 12,
-        average: 6,
-        is_metric: true
-      });
-      expect(Logger.prototype.info).toHaveBeenCalledWith('kacsa-cucc', {
-        min: 18,
-        max: 18,
-        count: 1,
-        sum: 18,
-        average: 18,
-        is_metric: true
-      });
-    });
-
-    it('should log counts for each combination of extra params provedid within a tag', () => {
-      metricLogger.measure('elfogyasztot-tap', 4, { emberke: 'bela' });
-      metricLogger.measure('elfogyasztot-tap', 8, { emberke: 'bela' });
-      metricLogger.measure('elfogyasztot-tap', 18, { emberke: 'jano' });
-
-      clock.tick(60 * 1000 + 1);
-      expect(Logger.prototype.info).toHaveBeenCalledWith('elfogyasztot-tap', {
-        min: 4,
-        max: 8,
-        count: 2,
-        sum: 12,
-        average: 6,
-        emberke: 'bela',
-        is_metric: true
-      });
-      expect(Logger.prototype.info).toHaveBeenCalledWith('elfogyasztot-tap', {
-        min: 18,
-        max: 18,
-        count: 1,
-        sum: 18,
-        average: 18,
-        emberke: 'jano',
-        is_metric: true
-      });
-    });
-
-    it('should group logs with same tag and deeply equal params together', () => {
-      metricLogger.measure('elfogyasztot-tap', 4, { a: 1, b: 2 });
-      metricLogger.measure('elfogyasztot-tap', 8, { b: 2, a: 1 });
-
-      clock.tick(60 * 1000 + 1);
-      expect(Logger.prototype.info).toHaveBeenCalledWith('elfogyasztot-tap', {
-        min: 4,
-        max: 8,
-        count: 2,
-        sum: 12,
-        average: 6,
-        a: 1,
-        b: 2,
-        is_metric: true
-      });
-    });
-
-    it('should count event distribution within specified thresholds', () => {
-      metricLogger.setThresholds('etwas', [5, 10, 20, 1000]);
-
-      metricLogger.measure('etwas', 2);
-      metricLogger.measure('etwas', 3);
-      metricLogger.measure('etwas', 4);
-      metricLogger.measure('etwas', 6);
-      metricLogger.measure('etwas', 11);
-      metricLogger.measure('etwas', 12);
-      metricLogger.measure('etwas', 816);
-
-      clock.tick(60 * 1000 + 1);
-      expect(Logger.prototype.info).toHaveBeenCalledWith('etwas', {
-        min: 2,
-        max: 816,
-        count: 7,
-        sum: 854,
-        average: 122,
-        above_5: 4,
-        above_10: 3,
-        above_20: 1,
-        above_1000: 0,
-        is_metric: true
-      });
-    });
-  });
-
-  describe('#start', () => {
-    it('should return a measurement object with the given tag, params and start time', () => {
-      const fakeCurrentTimestamp = 2323434543;
-      clock.setSystemTime(fakeCurrentTimestamp);
-
-      const measurement = metricLogger.start('test-tag', { foo: 'bar' });
-
-      expect(measurement).toEqual({ tag: 'test-tag', params: { foo: 'bar' }, start: fakeCurrentTimestamp });
-    });
-  });
-
-  describe('start/stop', () => {
-    it('should log min, max, count and sum when log was called only once', () => {
-      const measurement = metricLogger.start('utlegeles');
-      clock.tick(50);
-      metricLogger.stop(measurement);
-
-      clock.tick(60 * 1000);
-      expect(Logger.prototype.info).toHaveBeenCalledWith('utlegeles', {
-        min: 50,
-        max: 50,
-        count: 1,
-        sum: 50,
-        average: 50,
-        is_metric: true
-      });
-    });
-
-    it('should aggregate metrics for two measurements and log after a timeout', () => {
-      const measurement1 = metricLogger.start('utlegeles');
-      const measurement2 = metricLogger.start('utlegeles');
-      clock.tick(50);
-      metricLogger.stop(measurement1);
-      clock.tick(20);
-      metricLogger.stop(measurement2);
-
-      clock.tick(60 * 1000);
-      expect(Logger.prototype.info).toHaveBeenCalledWith('utlegeles', {
-        min: 50,
-        max: 70,
-        sum: 120,
-        count: 2,
-        average: 60,
-        is_metric: true
-      });
-    });
-
-    it('should flush results at the closes minute:30', () => {
-      const measurement = metricLogger.start('utlegeles');
-      metricLogger.stop(measurement);
-
-      clock.tick(29 * 1000 + 999);
-      expect(Logger.prototype.info).not.toHaveBeenCalled();
-      clock.tick(1 * 1000);
-      expect(Logger.prototype.info).toHaveBeenCalledWith('utlegeles', expect.anything());
-    });
-
-    it('should flush at next minute:30 if we are after 30 in current minute', () => {
-      clock.tick(45 * 1000);
-      const measurement = metricLogger.start('utlegeles');
-      metricLogger.stop(measurement);
-
-      clock.tick(44 * 1000);
-      expect(Logger.prototype.info).not.toHaveBeenCalled();
-      clock.tick(1 * 1000);
-      expect(Logger.prototype.info).toHaveBeenCalledWith('utlegeles', expect.anything());
-    });
-
-    it('should periodically log measuerments', () => {
-      const measurement1 = metricLogger.start('utlegeles');
-      const measurement2 = metricLogger.start('utlegeles');
-      clock.tick(10 * 1000);
-      const measurement3 = metricLogger.start('utlegeles');
-      metricLogger.stop(measurement1);
-
-      clock.tick(21 * 1000);
-      expect(Logger.prototype.info).toHaveBeenCalledWith('utlegeles', {
-        count: 1,
-        max: 10000,
-        min: 10000,
-        sum: 10000,
-        average: 10000,
-        is_metric: true
-      });
-
-      metricLogger.stop(measurement2);
-      metricLogger.stop(measurement3);
-      clock.tick(60 * 1000);
-      expect(Logger.prototype.info).toHaveBeenCalledWith('utlegeles', {
-        count: 2,
-        max: 31000,
-        min: 21000,
-        sum: 52000,
-        average: 26000,
-        is_metric: true
-      });
-    });
-
-    it('should log measurements for multiple tags and include extra parameters', () => {
-      const measurement1 = metricLogger.start('utlegeles', { a: 21 });
-      const measurement2 = metricLogger.start('nyavogas');
-      clock.tick(10 * 1000);
-      metricLogger.stop(measurement1);
-      clock.tick(5 * 1000);
-      metricLogger.stop(measurement2);
-      clock.tick(15 * 1000);
-
-      expect(Logger.prototype.info).toHaveBeenCalledWith('utlegeles', {
-        count: 1,
-        max: 10000,
-        min: 10000,
-        sum: 10000,
-        average: 10000,
-        a: 21,
-        is_metric: true
-      });
-      expect(Logger.prototype.info).toHaveBeenCalledWith('nyavogas', {
-        count: 1,
-        max: 15000,
-        min: 15000,
-        sum: 15000,
-        average: 15000,
-        is_metric: true
-      });
-    });
-  });
-
-  describe('simple count one log methods', () => {
-    it.each(['trace', 'debug', 'info', 'warn', 'error', 'fatal'])(
-      'should log a only the count when %s method is called',
-      method => {
-        jest.spyOn(Logger.prototype, method);
-
-        metricLogger[method]('elfogyasztot-tap', { alma: 2 });
-        metricLogger[method]('elfogyasztot-tap', { alma: 2 });
-
-        clock.tick(60 * 1000 + 11500);
-        expect(Logger.prototype[method]).toHaveBeenCalledWith('elfogyasztot-tap', {
-          count: 2,
-          alma: 2,
-          is_metric: true
-        });
-      }
-    );
-  });
-});