@emartech/aggregate-metric-logger 2.2.0 → 3.0.1

package/README.md

@@ -1,5 +1,5 @@
 # aggregate-metric-logger
-aggregate metrics in memory and log them every minute
+Aggregate metrics in memory and log them every minute
 
 ## Setup
 
@@ -11,11 +11,11 @@ To change the default `aggregate-metric-logger` namespace used for logs, set
 
 ## Setup to send metrics to GCP monitoring
 
-(currently only the `metricLogger.count`, `metricLogger.measureGcp` method sends metrics to GCP)
+(currently only the `metricLogger.count` and `metricLogger.measureGcp` methods send metrics to GCP)
 
 Set `METRIC_LOGGER_ENABLED` just us for LaaS based metrics.
 
-Set `GCP_PROJECT_ID` and `GCP_CREDENTIALS` env variables. `HOSTNAME` env variable also has to be
+Set `MONITORING_PROJECT_ID` and `GCP_CREDENTIALS` env variables. `HOSTNAME` env variable also has to be
 set, but for a pod running on kubernetes this should already be set.
 
 

package/env/.env

@@ -1,3 +1,3 @@
 HOSTNAME=local-machine
-GCP_PROJECT_ID=dummy-project
+MONITORING_PROJECT_ID=dummy-project
 METRIC_LOGGER_ENABLED=true

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@emartech/aggregate-metric-logger",
-  "version": "2.2.0",
+  "version": "3.0.1",
   "description": "aggregate metrics in memory for logging",
   "main": "src/index.js",
   "scripts": {

package/src/gcp-metric-collector/gcp-metric-collector.spec.js

@@ -5,8 +5,9 @@ const {
   stubCreateTimeSeries,
   getPublishedMetrics,
   flushPromises,
-  stubCreateTimeSeriesWithCallbackDuringPublish
+  resolvablePromise
 } = require('../../test-helpers/test-helpers');
+const { MetricServiceClient } = require('@google-cloud/monitoring');
 
 beforeEach(() => {
   jest.useFakeTimers();
@@ -123,9 +124,9 @@ describe('gcpMetricCollector', () => {
                 mean: 7,
                 sumOfSquaredDeviation: 0,
                 bucketOptions: {
-                  exponentialBuckets: { numFiniteBuckets: 20, growthFactor: 2, scale: 1 }
+                  exponentialBuckets: { numFiniteBuckets: 25, growthFactor: 2, scale: 1 }
                 },
-                bucketCounts: [0, 0, 0, 1, ...Array(18).fill(0)]
+                bucketCounts: [0, 0, 0, 1, ...Array(23).fill(0)]
               }
             },
             metricKind: 'GAUGE',
@@ -151,7 +152,7 @@ describe('gcpMetricCollector', () => {
                 count: 2,
                 mean: 9.5,
                 sumOfSquaredDeviation: 12.5,
-                bucketCounts: [0, 0, 0, 1, 1, ...Array(17).fill(0)]
+                bucketCounts: [0, 0, 0, 1, 1, ...Array(22).fill(0)]
               }
             }
           }
@@ -182,7 +183,7 @@ describe('gcpMetricCollector', () => {
                 count: 1,
                 mean: 5,
                 sumOfSquaredDeviation: 0,
-                bucketCounts: [0, 0, 0, 1, ...Array(18).fill(0)]
+                bucketCounts: [0, 0, 0, 1, ...Array(23).fill(0)]
               }
             }
           }
@@ -190,16 +191,27 @@ describe('gcpMetricCollector', () => {
       ]);
     });
 
-    it('should not remove metrics data during publishing', async () => {
-      stubCreateTimeSeriesWithCallbackDuringPublish(() => gcpMetricCollector.measure('jolo', { kacsa: 'A' }, 5));
+    it('should not lose metrics data logged during publishing', async () => {
+      const firstCreateTimeSeriesPromise = resolvablePromise();
+      const secondCreateTimeSeriesPromise = resolvablePromise();
+      jest.spyOn(MetricServiceClient.prototype, 'createTimeSeries')
+        .mockReturnValueOnce(firstCreateTimeSeriesPromise)
+        .mockReturnValueOnce(secondCreateTimeSeriesPromise);
 
       gcpMetricCollector.measure('jolo', { kacsa: 'A' }, 7);
 
       jest.advanceTimersByTime(60 * 1000 + 1);
+
+      gcpMetricCollector.measure('jolo', { kacsa: 'A' }, 5);
+      firstCreateTimeSeriesPromise.resolve();
+
       await flushPromises();
+
       jest.advanceTimersByTime(60 * 1000 + 1);
       await flushPromises();
 
+      secondCreateTimeSeriesPromise.resolve();
+
       expect(getPublishedMetrics()).toMatchObject([
         [{}],
         [
@@ -209,7 +221,7 @@ describe('gcpMetricCollector', () => {
                 count: 1,
                 mean: 5,
                 sumOfSquaredDeviation: 0,
-                bucketCounts: [0, 0, 0, 1, ...Array(18).fill(0)]
+                bucketCounts: [0, 0, 0, 1, ...Array(23).fill(0)]
               }
             }
           }

package/src/gcp-time-series-builder/gcp-time-series-builder.js

@@ -10,7 +10,7 @@ exports.buildTimeSeries = metrics =>
       },
       resource: {
         type: 'global',
-        labels: { project_id: process.env.GCP_PROJECT_ID }
+        labels: { project_id: process.env.MONITORING_PROJECT_ID }
       },
       points: [
         {

package/src/gcp-time-series-publisher/gcp-time-series-publisher.js

@@ -2,14 +2,20 @@
 
 const logger = require('@emartech/json-logger')('aggregate-metric-logger');
 const monitoring = require('@google-cloud/monitoring');
+const { chunk } = require('lodash');
+
+const MAX_TIME_SERIES_COUNT = 200;
 
 exports.publishTimeSeries = async timeSeries => {
   try {
     const client = getMonitoringClient();
-    await client.createTimeSeries({
-      name: client.projectPath(process.env.GCP_PROJECT_ID),
-      timeSeries
-    });
+
+    for (const timeSeriesChunk of chunk(timeSeries, MAX_TIME_SERIES_COUNT)) {
+      await client.createTimeSeries({
+        name: client.projectPath(process.env.MONITORING_PROJECT_ID),
+        timeSeries: timeSeriesChunk
+      });
+    }
   } catch (error) {
     logger.warnFromError('pushing-metrics-failed', error, { do_not_alert: true });
   }
@@ -19,7 +25,7 @@ let monitoringClientInstance = null;
 const getMonitoringClient = () => {
   if (!monitoringClientInstance) {
     monitoringClientInstance = new monitoring.MetricServiceClient({
-      projectId: process.env.GCP_PROJECT_ID
+      projectId: process.env.MONITORING_PROJECT_ID
     });
   }
   return monitoringClientInstance;

package/src/gcp-time-series-publisher/gcp-time-series-publisher.spec.js

@@ -1,6 +1,7 @@
 'use strict';
 
 const { MetricServiceClient } = require('@google-cloud/monitoring');
+const { range } = require('lodash');
 const { publishTimeSeries } = require('./gcp-time-series-publisher');
 const { stubCreateTimeSeries } = require('../../test-helpers/test-helpers');
 
@@ -8,10 +9,10 @@ jest.mock('@google-cloud/monitoring');
 
 describe('gcpMetricPublisher', () => {
   describe('publishTimeSeries', () => {
-    it('should push a metrics to gcp in the correct format', async () => {
+    it('should push metrics to gcp to the correct project', async () => {
       stubCreateTimeSeries();
       process.env.HOSTNAME = 'test-host';
-      process.env.GCP_PROJECT_ID = 'test-project';
+      process.env.MONITORING_PROJECT_ID = 'test-project';
       const timeSeries = [1, 2];
 
       await publishTimeSeries(timeSeries);
@@ -21,5 +22,23 @@ describe('gcpMetricPublisher', () => {
         timeSeries
       });
     });
+
+    it('should push metrics to gcp in batches', async () => {
+      stubCreateTimeSeries();
+      process.env.HOSTNAME = 'test-host';
+      process.env.MONITORING_PROJECT_ID = 'test-project';
+
+      await publishTimeSeries(range(1, 300));
+
+      expect(MetricServiceClient.prototype.createTimeSeries).toHaveBeenCalledWith({
+        name: 'projects/test-project',
+        timeSeries: range(1, 201)
+      });
+      expect(MetricServiceClient.prototype.createTimeSeries).toHaveBeenCalledWith({
+        name: 'projects/test-project',
+        timeSeries: range(201, 300)
+      });
+    });
+
   });
 });

package/src/metric-types/distribution/distribution.js

@@ -2,7 +2,7 @@
 
 // exponential bucket rules: https://cloud.google.com/monitoring/api/ref_v3/rest/v3/TypedValue#exponential
 
-exports.createDistribution = ({ scale = 1, growthFactor = 2, numFiniteBuckets = 20 } = {}) => {
+exports.createDistribution = ({ scale = 1, growthFactor = 2, numFiniteBuckets = 25 } = {}) => {
   const bucketUpperBounds = Array(numFiniteBuckets + 2)
     .fill(null)
     .map((value, index, array) =>

package/test-helpers/test-helpers.js

@@ -3,16 +3,6 @@
 const { MetricServiceClient } = require('@google-cloud/monitoring');
 const { omit, pick } = require('lodash');
 
-exports.stubCreateTimeSeriesWithCallbackDuringPublish = (createMetric) => {
-  jest.spyOn(MetricServiceClient.prototype, 'createTimeSeries').mockImplementation(
-    () =>
-      new Promise(resolve => {
-        createMetric();
-        resolve();
-      })
-  );
-};
-
 exports.stubCreateTimeSeries = () => {
   jest.spyOn(MetricServiceClient.prototype, 'createTimeSeries').mockResolvedValue();
 };
@@ -35,3 +25,18 @@ exports.getPublishedMetrics = () => {
 };
 
 exports.flushPromises = async () => new Promise(jest.requireActual('timers').setImmediate);
+
+exports.resolvablePromise = () => {
+  let resolvePromise;
+  let rejectPromise;
+
+  const promise = new Promise((resolve, reject) => {
+    resolvePromise = resolve;
+    rejectPromise = reject;
+  });
+
+  promise.resolve = resolvePromise;
+  promise.reject = rejectPromise;
+
+  return promise;
+};