@emartech/aggregate-metric-logger 2.0.5 → 2.1.0

package/README.md

@@ -9,6 +9,16 @@ start collecting and logging measurements.
 To change the default `aggregate-metric-logger` namespace used for logs, set
 `METRIC_LOGGER_NAMESPACE` to a custom value.
 
+## Setup to send metrics to GCP monitoring
+
+(currently only the `metricLogger.count` method sends metrics to GCP)
+
+Set `METRIC_LOGGER_ENABLED` just us for LaaS based metrics.
+
+Set `GCP_PROJECT_ID` and `GCP_CREDENTIALS` env variables. `HOSTNAME` env variable also has to be
+set, but for a pod running on kubernetes this should already be set.
+
+
 ## Usage
 
 The package exposes similar logging methods as the `@emartech/json-logger` package:

package/env/.env

@@ -0,0 +1,3 @@
+HOSTNAME=local-machine
+GCP_PROJECT_ID=dummy-project
+METRIC_LOGGER_ENABLED=true

package/jest.config.js

@@ -0,0 +1,8 @@
+'use strict';
+
+module.exports = {
+  testEnvironment: 'node',
+  setupFilesAfterEnv: ['./test-helpers/test-setup.js'],
+  restoreMocks: true,
+  transform: {}
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@emartech/aggregate-metric-logger",
-  "version": "2.0.5",
+  "version": "2.1.0",
   "description": "aggregate metrics in memory for logging",
   "main": "src/index.js",
   "scripts": {
@@ -30,6 +30,7 @@
   },
   "dependencies": {
     "@emartech/json-logger": "^3.1.0",
+    "@google-cloud/monitoring": "^3.0.2",
     "date-fns": "^2.3.0",
     "json-stable-stringify": "^1.0.1",
     "lodash": "^4.17.15"
@@ -37,10 +38,11 @@
   "devDependencies": {
     "@sinonjs/fake-timers": "9.1.1",
     "@types/jest": "27.4.1",
+    "dotenv-haphap": "^3.1.2",
     "eslint": "^8.12.0",
     "eslint-config-emarsys": "^5.1.0",
     "eslint-plugin-jest": "^26.1.3",
-    "jest": "27.5.1",
+    "jest": "^29.2.2",
     "semantic-release": "^19.0.2"
   }
 }

package/src/gcp-metric-collector.js

@@ -0,0 +1,41 @@
+'use strict';
+
+const stringify = require('json-stable-stringify');
+const { publishCountMetrics } = require('./gcp-metric-publisher');
+
+let metrics = {};
+let initialized = false;
+let startTime = null;
+let timeoutId = null;
+
+exports.count = (metricName, labels = {}, value = 1) => {
+  if (process.env.METRIC_LOGGER_ENABLED !== 'true') return;
+
+  if (!initialized) {
+    initialized = true;
+    startTime = Date.now();
+    setupNextFlush();
+  }
+  const key = `${metricName}_${stringify(labels)}`;
+  if (!metrics[key]) {
+    metrics[key] = { name: metricName, labels, count: value };
+  } else {
+    metrics[key].count += value;
+  }
+};
+
+exports.reset = () => {
+  metrics = {};
+  initialized = false;
+  startTime = null;
+  clearTimeout(timeoutId);
+};
+
+const setupNextFlush = () => {
+  timeoutId = setTimeout(flush, 60 * 1000);
+};
+
+const flush = async () => {
+  await publishCountMetrics(metrics, startTime);
+  setupNextFlush();
+};

package/src/gcp-metric-collector.spec.js

@@ -0,0 +1,89 @@
+'use strict';
+
+const gcpMetricCollector = require('./gcp-metric-collector');
+const {
+  stubCreateTimeSeries,
+  getPublishedMetrics,
+  flushPromises
+} = require('../test-helpers/test-helpers');
+
+beforeEach(() => {
+  jest.useFakeTimers();
+  gcpMetricCollector.reset();
+});
+afterEach(() => {
+  jest.useRealTimers();
+  gcpMetricCollector.reset();
+});
+
+describe('gcpMetricCollector', () => {
+  describe('count', () => {
+    it('should send metric for a single measurement in next flush period', () => {
+      stubCreateTimeSeries();
+
+      gcpMetricCollector.count('jolo', { 'kacsa': 'A' }, 88);
+
+      expect(getPublishedMetrics()).toEqual([]);
+
+      jest.advanceTimersByTime(60 * 1000 + 1);
+
+      expect(getPublishedMetrics()).toEqual([
+        [{ name: 'jolo', labels: { kacsa: 'A' }, count: 88 }]
+      ]);
+    });
+
+    it('should sum up counts that happened and publish the sum, send a metric for each label combo', () => {
+      stubCreateTimeSeries();
+
+      gcpMetricCollector.count('jolo', { a: 1 });
+      gcpMetricCollector.count('jolo');
+      gcpMetricCollector.count('yolo');
+      gcpMetricCollector.count('yolo');
+      gcpMetricCollector.count('jolo');
+
+      expect(getPublishedMetrics()).toEqual([]);
+
+      jest.advanceTimersByTime(60 * 1000 + 1);
+
+      expect(getPublishedMetrics()).toEqual([
+        [
+          { name: 'jolo', labels: { a: 1 }, count: 1 },
+          { name: 'jolo', labels: {}, count: 2 },
+          { name: 'yolo', labels: {}, count: 2 }
+        ]
+      ]);
+    });
+
+    it('should not reset counts after sending metrics, but keep increasing them', async () => {
+      stubCreateTimeSeries();
+
+      gcpMetricCollector.count('jolo');
+      gcpMetricCollector.count('jolo');
+
+      jest.advanceTimersByTime(60 * 1000 + 1);
+      await flushPromises();
+      expect(getPublishedMetrics()).toEqual([
+        [{ name: 'jolo', labels: {}, count: 2 }]
+      ]);
+
+      gcpMetricCollector.count('jolo');
+      jest.advanceTimersByTime(60 * 1000 + 1);
+      await flushPromises();
+
+      expect(getPublishedMetrics()).toEqual([
+        [{ name: 'jolo', labels: {}, count: 2 }],
+        [{ name: 'jolo', labels: {}, count: 3 }]
+      ]);
+    });
+
+    it('should not publish metrics if metric logger is not enabled', async () => {
+      process.env.METRIC_LOGGER_ENABLED = 'false';
+      stubCreateTimeSeries();
+
+      gcpMetricCollector.count('jolo', { 'kacsa': 'A' }, 88);
+      jest.advanceTimersByTime(60 * 1000 + 1);
+
+      expect(getPublishedMetrics()).toEqual([]);
+    });
+  });
+});

package/src/gcp-metric-publisher.js

@@ -0,0 +1,32 @@
+'use strict';
+
+const monitoring = require('@google-cloud/monitoring');
+const client = new monitoring.MetricServiceClient();
+
+exports.publishCountMetrics = async (metrics, startTime) => {
+  const timeSeries = Object.values(metrics).map(metric => ({
+    metric: {
+      type: `custom.googleapis.com/${metric.name}`,
+      labels: { ...metric.labels, hostname: process.env.HOSTNAME }
+    },
+    resource: {
+      type: 'global',
+      labels: { project_id: process.env.GCP_PROJECT_ID }
+    },
+    points: [
+      {
+        interval: {
+          startTime: { seconds: startTime / 1000 },
+          endTime: { seconds: Date.now() / 1000 }
+        },
+        value: { int64Value: `${metric.count}` }
+      }
+    ],
+    metricKind: 'CUMULATIVE',
+    valueType: 'int64Value'
+  }));
+  await client.createTimeSeries({
+    name: client.projectPath(process.env.GCP_PROJECT_ID),
+    timeSeries
+  });
+};

package/src/gcp-metric-publisher.spec.js

@@ -0,0 +1,68 @@
+'use strict';
+
+const { MetricServiceClient } = require('@google-cloud/monitoring');
+const { publishCountMetrics } = require('./gcp-metric-publisher');
+const { stubCreateTimeSeries, getPublishedMetrics } = require('../test-helpers/test-helpers');
+
+jest.mock('@google-cloud/monitoring');
+
+describe('gcpMetricPublisher', () => {
+  describe('publishCountMetrics', () => {
+    it('should push a metrics to gcp in the correct format', async () => {
+      stubCreateTimeSeries();
+      jest.useFakeTimers().setSystemTime(new Date(80000));
+      process.env.HOSTNAME = 'test-host';
+      process.env.GCP_PROJECT_ID = 'test-project';
+      const metrics = {
+        key1: { name: 'kacs', labels: { a: 1 }, count: 14 }
+      };
+      const startTime = 15000;
+
+      await publishCountMetrics(metrics, startTime);
+
+      expect(MetricServiceClient.prototype.createTimeSeries).toHaveBeenCalledWith({
+        name: 'projects/test-project',
+        timeSeries: [
+          {
+            metric: {
+              type: 'custom.googleapis.com/kacs',
+              labels: { a: 1, hostname: 'test-host' }
+            },
+            resource: { type: 'global', labels: { project_id: 'test-project' } },
+            points: [
+              {
+                interval: {
+                  startTime: { seconds: 15 },
+                  endTime: { seconds: 80 }
+                },
+                value: { int64Value: '14' }
+              }
+            ],
+            metricKind: 'CUMULATIVE',
+            valueType: 'int64Value'
+          }
+        ]
+      });
+    });
+
+    it('should publish multile metrics with the same api call', async () => {
+      stubCreateTimeSeries();
+      const metrics = {
+        key1: { name: 'kacs', labels: { a: 1 }, count: 14 },
+        key2: { name: 'kacs', labels: { a: 2 }, count: 88 },
+        key3: { name: 'lacs', labels: {}, count: 1 }
+      };
+      const startTime = 15000;
+
+      await publishCountMetrics(metrics, startTime);
+
+      expect(getPublishedMetrics()).toEqual([
+        [
+          { name: 'kacs', labels: { a: 1 }, count: 14 },
+          { name: 'kacs', labels: { a: 2 }, count: 88 },
+          { name: 'lacs', labels: {}, count: 1 }
+        ]
+      ]);
+    });
+  });
+});

package/src/index.js

@@ -1,21 +1,18 @@
 'use strict';
 
 const metricLoggerFactory = require('./metric-logger');
+const gcpMetricCollector = require('./gcp-metric-collector');
 
 const enabled = process.env.METRIC_LOGGER_ENABLED === 'true';
-const inProgressMeasurementWarningLimit =
-  parseInt(process.env.METRIC_LOGGER_IN_PROGRESS_MEASUREMENT_WARNING_LIMIT, 10) || 10000;
 
 const exposedInstance = metricLoggerFactory({
   enabled,
-  namespace: process.env.METRIC_LOGGER_NAMESPACE || 'aggregate-metric-logger',
-  inProgressMeasurementWarningLimit
+  namespace: process.env.METRIC_LOGGER_NAMESPACE || 'aggregate-metric-logger'
 });
 const exposedFactoryInstanceHybrid = namespace => metricLoggerFactory({ enabled, namespace });
 
 exposedFactoryInstanceHybrid.setThresholds = exposedInstance.setThresholds;
 exposedFactoryInstanceHybrid.measure = exposedInstance.measure;
-exposedFactoryInstanceHybrid.count = exposedInstance.count;
 exposedFactoryInstanceHybrid.trace = exposedInstance.trace;
 exposedFactoryInstanceHybrid.debug = exposedInstance.debug;
 exposedFactoryInstanceHybrid.info = exposedInstance.info;
@@ -26,4 +23,6 @@ exposedFactoryInstanceHybrid.start = exposedInstance.start;
 exposedFactoryInstanceHybrid.stop = exposedInstance.stop;
 exposedFactoryInstanceHybrid.cancel = exposedInstance.cancel;
 
+exposedFactoryInstanceHybrid.count = gcpMetricCollector.count;
+
 module.exports = exposedFactoryInstanceHybrid;

package/src/metric-logger.spec.js

@@ -4,11 +4,10 @@ const metricLoggerFactory = require('./metric-logger');
 const Logger = require('@emartech/json-logger').Logger;
 const fakeTimers = require('@sinonjs/fake-timers');
 
-jest.spyOn(Logger.prototype, 'info');
 let metricLogger;
 let clock;
 beforeEach(() => {
-  jest.resetAllMocks();
+  jest.spyOn(Logger.prototype, 'info');
   metricLogger = metricLoggerFactory();
   clock = fakeTimers.install();
 });

package/test-helpers/test-helpers.js

@@ -0,0 +1,24 @@
+'use strict';
+
+const { MetricServiceClient } = require('@google-cloud/monitoring');
+const { omit } = require('lodash');
+
+exports.stubCreateTimeSeries = () => {
+  jest.spyOn(MetricServiceClient.prototype, 'createTimeSeries').mockResolvedValue();
+};
+
+exports.getPublishedMetrics = () => {
+  const { mock } = MetricServiceClient.prototype.createTimeSeries;
+
+  if (mock.calls.length === 0) return [];
+
+  return mock.calls.map(call =>
+    call[0].timeSeries.map(series => ({
+      name: series.metric.type.replace('custom.googleapis.com/', ''),
+      labels: omit(series.metric.labels, 'hostname'),
+      count: Number(series.points[0].value.int64Value)
+    }))
+  );
+};
+
+exports.flushPromises = async () => new Promise(jest.requireActual('timers').setImmediate);

package/test-helpers/test-setup.js

@@ -0,0 +1,13 @@
+'use strict';
+
+require('dotenv-haphap').config('env/.env');
+
+const env = process.env;
+
+beforeEach(() => {
+  process.env = { ...env };
+});
+
+afterEach(() => {
+  process.env = env;
+});