@emartech/aggregate-metric-logger 5.0.2 → 5.2.0

package/README.md

@@ -18,6 +18,8 @@ Set `METRIC_LOGGER_ENABLED` just us for LaaS based metrics.
 Set `MONITORING_PROJECT_ID` and `GCP_CREDENTIALS` env variables. `HOSTNAME` env variable also has to be
 set, but for a pod running on kubernetes this should already be set.
 
+Add `Monitoring Metrics Writer` Role to the service account in the GCP project.
+
 
 ## Usage
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@emartech/aggregate-metric-logger",
-  "version": "5.0.2",
+  "version": "5.2.0",
   "description": "aggregate metrics in memory for logging",
   "main": "src/index.js",
   "scripts": {
@@ -30,19 +30,19 @@
   },
   "dependencies": {
     "@emartech/json-logger": "^8.0.3",
-    "@google-cloud/monitoring": "^3.0.2",
-    "date-fns": "^2.3.0",
+    "@google-cloud/monitoring": "^4.1.0",
+    "date-fns": "^3.6.0",
     "json-stable-stringify": "^1.1.1",
     "lodash": "^4.17.15",
     "ms": "^2.1.3"
   },
   "devDependencies": {
-    "@emartech/rti-server-tools": "^6.13.8",
+    "@emartech/rti-server-tools": "^6.17.0",
     "@sinonjs/fake-timers": "^11.2.2",
     "@types/jest": "^29.5.12",
     "dotenv-haphap": "^3.1.2",
-    "eslint": "^9.4.0",
-    "globals": "15.4.0",
+    "eslint": "^9.7.0",
+    "globals": "15.8.0",
     "jest": "^29.2.2",
     "semantic-release": "^24.0.0"
   }

package/src/gcp-metric-collector/gcp-metric-collector.js

@@ -1,71 +1,88 @@
 'use strict';
 
 const stringify = require('json-stable-stringify');
-const { publishTimeSeries } = require('../gcp-time-series-publisher/gcp-time-series-publisher');
-const { buildTimeSeries } = require('../gcp-time-series-builder/gcp-time-series-builder');
+const { createGcpTimeSeriesPublisher } = require('../gcp-time-series-publisher/gcp-time-series-publisher');
+const { createGcpTimeSeriesBuilder } = require('../gcp-time-series-builder/gcp-time-series-builder');
 const { createCounter } = require('../metric-types/counter/counter');
 const { createDistribution } = require('../metric-types/distribution/distribution');
 const { isNumber, isPlainObject } = require('lodash');
 
-let metrics = {};
-let initialized = false;
-let timeoutId = null;
+class GcpMetricCollector {
+  metrics = {};
+  initialized = false;
+  timeoutId = null;
+  projectId = null;
 
-exports.count = (metricName, labels = {}, value = 1) => {
-  if (!isNumber(value)) {
-    throw new Error(`${ value } is not a number`);
+  constructor(projectId = null) {
+    this.projectId = projectId ?? process.env.MONITORING_PROJECT_ID;
   }
 
-  if (!isPlainObject(labels)) {
-    throw new Error('labels is not a plain object');
-  }
+  count(metricName, labels = {}, value = 1) {
+    if (!isNumber(value)) {
+      throw new Error(`${ value } is not a number`);
+    }
 
-  addToMetric(metricName, labels, 'counter', value);
-};
+    if (!isPlainObject(labels)) {
+      throw new Error('labels is not a plain object');
+    }
 
-exports.measure = (metricName, labels = {}, value = 1) =>
-  addToMetric(metricName, labels, 'distribution', value);
+    this.addToMetric(metricName, labels, 'counter', value);
+  }
 
-exports.start = (metricName, labels = {}) => ({ metricName, labels, start: Date.now() });
+  measure(metricName, labels = {}, value = 1) {
+    return this.addToMetric(metricName, labels, 'distribution', value);
+  }
+
+  start(metricName, labels = {}) {
+    return { metricName, labels, start: Date.now() };
+  }
 
-exports.stop = measurement =>
-  addToMetric(
-    measurement.metricName,
-    measurement.labels,
-    'distribution',
-    Date.now() - measurement.start
-  );
+  stop(measurement) {
+    this.addToMetric(
+      measurement.metricName,
+      measurement.labels,
+      'distribution',
+      Date.now() - measurement.start
+    );
+  }
 
-const addToMetric = (metricName, labels = {}, type, value) => {
-  if (process.env.METRIC_LOGGER_ENABLED !== 'true') return;
+  addToMetric(metricName, labels = {}, type, value) {
+    if (process.env.METRIC_LOGGER_ENABLED !== 'true') return;
 
-  if (!initialized) {
-    initialized = true;
-    setupNextFlush();
+    if (!this.initialized) {
+      this.initialized = true;
+      this.setupNextFlush();
+    }
+    const key = `${metricName}_${stringify(labels)}`;
+    if (!this.metrics[key]) {
+      const aggregator = type === 'counter' ? createCounter() : createDistribution();
+      this.metrics[key] = { name: metricName, labels, aggregator };
+    }
+    this.metrics[key].aggregator.add(value);
   }
-  const key = `${metricName}_${stringify(labels)}`;
-  if (!metrics[key]) {
-    const aggregator = type === 'counter' ? createCounter() : createDistribution();
-    metrics[key] = { name: metricName, labels, aggregator };
+
+  reset() {
+    this.metrics = {};
+    this.initialized = false;
+    clearTimeout(this.timeoutId);
   }
-  metrics[key].aggregator.add(value);
-};
 
-exports.reset = () => {
-  metrics = {};
-  initialized = false;
-  clearTimeout(timeoutId);
-};
+  setupNextFlush() {
+    this.timeoutId = setTimeout(this.flush.bind(this), 60 * 1000);
+  }
 
-const setupNextFlush = () => {
-  timeoutId = setTimeout(flush, 60 * 1000);
-};
+  async flush() {
+    const timeSeries = createGcpTimeSeriesBuilder(this.projectId).buildTimeSeries(this.metrics);
+    for (const [, metric] of Object.entries(this.metrics)) {
+      metric.aggregator.advanceTimer();
+    }
+    await createGcpTimeSeriesPublisher(this.projectId).publishTimeSeries(timeSeries);
+    this.setupNextFlush();
+  }
 
-const flush = async () => {
-  const timeSeries = buildTimeSeries(metrics);
-  for (const [, metric] of Object.entries(metrics)) {
-    metric.aggregator.advanceTimer();
+  createGcpMetricsCollector(projectId) {
+    return new GcpMetricCollector(projectId);
   }
-  await publishTimeSeries(timeSeries);
-  setupNextFlush();
-};
+}
+
+module.exports = new GcpMetricCollector();

package/src/gcp-metric-collector/gcp-metric-collector.spec.js

@@ -33,6 +33,12 @@ describe('gcpMetricCollector', () => {
       expect(getPublishedMetrics()).toEqual([
         [
           {
+            resource: {
+              labels: {
+                project_id: 'dummy-project'
+              },
+              type: 'global'
+            },
             name: 'jolo',
             labels: { kacsa: 'A' },
             value: { int64Value: '88' },
@@ -45,6 +51,81 @@ describe('gcpMetricCollector', () => {
       ]);
     });
 
+    it('should send metric for given gcp project id', () => {
+      jest.useFakeTimers().setSystemTime(new Date(20000));
+      stubCreateTimeSeries();
+
+      gcpMetricCollector.createGcpMetricsCollector('test-project').count('jolo', { kacsa: 'A' }, 88);
+
+      expect(getPublishedMetrics()).toEqual([]);
+
+      jest.advanceTimersByTime(60 * 1000 + 1);
+
+      expect(getPublishedMetrics()).toEqual([
+        [
+          expect.objectContaining({
+            resource: {
+              labels: {
+                project_id: 'test-project'
+              },
+              type: 'global'
+            }
+          })
+        ]
+      ]);
+    });
+
+    it('should measure seperate metrics for each gcp projects', () => {
+      jest.useFakeTimers().setSystemTime(new Date(20000));
+      stubCreateTimeSeries();
+
+      const anotherCollector = gcpMetricCollector.createGcpMetricsCollector('another-project');
+
+      gcpMetricCollector.count('jolo', { kacsa: 'A' }, 88);
+      anotherCollector.count('jolo', { kacsa: 'A' }, 66);
+
+      expect(getPublishedMetrics()).toEqual([]);
+
+      jest.advanceTimersByTime(60 * 1000 + 1);
+
+      expect(getPublishedMetrics()).toEqual([
+        [
+          {
+            resource: {
+              labels: {
+                project_id: 'dummy-project'
+              },
+              type: 'global'
+            },
+            name: 'jolo',
+            labels: { kacsa: 'A' },
+            value: { int64Value: '88' },
+            startTime: 19000,
+            endTime: 80000,
+            metricKind: 'CUMULATIVE',
+            valueType: 'int64Value'
+          }
+        ],
+        [
+          {
+            resource: {
+              labels: {
+                project_id: 'another-project'
+              },
+              type: 'global'
+            },
+            name: 'jolo',
+            labels: { kacsa: 'A' },
+            value: { int64Value: '66' },
+            startTime: 19000,
+            endTime: 80000,
+            metricKind: 'CUMULATIVE',
+            valueType: 'int64Value'
+          }
+        ]
+      ]);
+    })
+
     it('should sum up counts that happened and publish the sum, send a metric for each label combo', () => {
       stubCreateTimeSeries();
 
@@ -134,6 +215,12 @@ describe('gcpMetricCollector', () => {
             labels: { kacsa: 'A' },
             startTime: 80000,
             endTime: 80000,
+            resource: {
+              labels: {
+                project_id: 'dummy-project'
+              },
+              type: 'global'
+            },
             value: {
               distributionValue: {
                 count: 1,

package/src/gcp-time-series-builder/gcp-time-series-builder.js

@@ -1,27 +1,44 @@
 'use strict';
 
-exports.buildTimeSeries = metrics =>
-  Object.values(metrics).map(metric => {
-    const state = metric.aggregator.getState();
-    return {
-      metric: {
-        type: `custom.googleapis.com/${metric.name}`,
-        labels: { ...metric.labels, hostname: process.env.HOSTNAME }
-      },
-      resource: {
-        type: 'global',
-        labels: { project_id: process.env.MONITORING_PROJECT_ID }
-      },
-      points: [
-        {
-          interval: {
-            startTime: { seconds: state.startTime / 1000 },
-            endTime: { seconds: state.endTime / 1000 }
-          },
-          value: state.value
-        }
-      ],
-      metricKind: state.metricKind,
-      valueType: state.valueType
-    };
-  });
+class GcpTimeSeriesBuilder {
+  projectId = null;
+
+  constructor(projectId = null) {
+    this.projectId = projectId ?? process.env.MONITORING_PROJECT_ID;
+  }
+  buildTimeSeries(metrics) {
+    return Object.values(metrics).map(metric => {
+      const state = metric.aggregator.getState();
+      return {
+        metric: {
+          type: `custom.googleapis.com/${metric.name}`,
+          labels: { ...metric.labels, hostname: process.env.HOSTNAME }
+        },
+        resource: {
+          type: 'global',
+          labels: { project_id: this.projectId }
+        },
+        points: [
+          {
+            interval: {
+              startTime: { seconds: state.startTime / 1000 },
+              endTime: { seconds: state.endTime / 1000 }
+            },
+            value: state.value
+          }
+        ],
+        metricKind: state.metricKind,
+        valueType: state.valueType
+      };
+    });
+  }
+}
+
+exports.createGcpTimeSeriesBuilder = projectId => {
+  return new GcpTimeSeriesBuilder(projectId);
+}
+
+exports.buildTimeSeries = metrics => {
+  const builder = new GcpTimeSeriesBuilder();
+  return builder.buildTimeSeries(metrics);
+}

package/src/gcp-time-series-builder/gcp-time-series-builder.spec.js

@@ -1,6 +1,6 @@
 'use strict';
 
-const { buildTimeSeries } = require('./gcp-time-series-builder');
+const { buildTimeSeries, createGcpTimeSeriesBuilder } = require('./gcp-time-series-builder');
 
 jest.mock('@google-cloud/monitoring');
 
@@ -69,6 +69,27 @@ describe('gcpTimeSeriesBuilder', () => {
       ]);
     });
 
+    it('should create time series from metric for given gcp project id', async () => {
+      const metrics = {
+        key1: dummyMetric({ name: 'kacs', labels: { a: 1 }, count: 14 })
+      };
+
+      expect(createGcpTimeSeriesBuilder('test-project').buildTimeSeries(metrics)).toEqual([
+        {
+          metric: expect.anything(),
+          metricKind: expect.anything(),
+          points: expect.anything(),
+          resource: {
+            labels: {
+              project_id: 'test-project'
+            },
+            type: 'global'
+          },
+          valueType: expect.anything()
+        }
+      ]);
+    });
+
     it('should create time series from all the metrics', async () => {
       const metrics = {
         key1: dummyMetric({ name: 'kacs', labels: { a: 1 }, count: 14 }),

package/src/gcp-time-series-publisher/gcp-time-series-publisher.js

@@ -8,33 +8,48 @@ const { setTimeout } = require('timers/promises');
 
 const MAX_TIME_SERIES_COUNT = 200;
 
-exports.publishTimeSeries = async timeSeries => {
-  try {
-    const client = getMonitoringClient();
-    for (const timeSeriesChunk of chunk(timeSeries, MAX_TIME_SERIES_COUNT)) {
-      await createTimeSeries(client, timeSeriesChunk);
-    }
-  } catch (error) {
-    logger.warnFromError('pushing-metrics-failed', error, { tags: ['alert_never'] });
-  }
-};
+class GcpTimeSeriesPublisher {
+  monitoringClientInstance = null;
+  projectId = null;
 
-const createTimeSeries = async (client, timeSeries) => {
-  const name = client.projectPath(process.env.MONITORING_PROJECT_ID);
-  try {
-    await client.createTimeSeries({ name, timeSeries });
-  } catch (error) {
-    await setTimeout(ms('10 sec'));
-    await client.createTimeSeries({ name, timeSeries });
+  constructor(projectId = null) {
+    this.projectId = projectId ?? process.env.MONITORING_PROJECT_ID;
   }
-};
 
-let monitoringClientInstance = null;
-const getMonitoringClient = () => {
-  if (!monitoringClientInstance) {
-    monitoringClientInstance = new monitoring.MetricServiceClient({
-      projectId: process.env.MONITORING_PROJECT_ID
-    });
-  }
-  return monitoringClientInstance;
+  async publishTimeSeries(timeSeries) {
+    try {
+      const client = this.getMonitoringClient();
+      for (const timeSeriesChunk of chunk(timeSeries, MAX_TIME_SERIES_COUNT)) {
+        await this.createTimeSeries(client, timeSeriesChunk);
+      }
+    } catch (error) {
+      logger.warnFromError('pushing-metrics-failed', error, { tags: ['alert_never'] });
+    }
+  };
+
+  async createTimeSeries(client, timeSeries) {
+    const name = client.projectPath(this.projectId);
+    try {
+      await client.createTimeSeries({ name, timeSeries });
+    } catch (error) {
+      await setTimeout(ms('10 sec'));
+      await client.createTimeSeries({ name, timeSeries });
+    }
+  };
+
+  getMonitoringClient() {
+    if (!this.monitoringClientInstance) {
+      this.monitoringClientInstance = new monitoring.MetricServiceClient({
+        projectId: this.projectId
+      });
+    }
+    return this.monitoringClientInstance;
+  };
+}
+
+exports.createGcpTimeSeriesPublisher = projectId => new GcpTimeSeriesPublisher(projectId)
+
+exports.publishTimeSeries = async timeSeries => {
+  const publisher = new GcpTimeSeriesPublisher()
+  await publisher.publishTimeSeries(timeSeries);
 };

package/src/gcp-time-series-publisher/gcp-time-series-publisher.spec.js

@@ -2,7 +2,7 @@
 
 const { MetricServiceClient } = require('@google-cloud/monitoring');
 const { range } = require('lodash');
-const { publishTimeSeries } = require('./gcp-time-series-publisher');
+const { publishTimeSeries, createGcpTimeSeriesPublisher } = require('./gcp-time-series-publisher');
 const { stubCreateTimeSeries } = require('../../test-helpers/test-helpers');
 
 jest.mock('@google-cloud/monitoring');
@@ -23,6 +23,20 @@ describe('gcpMetricPublisher', () => {
       });
     });
 
+    it('should push metrics to gcp to given project', async () => {
+      stubCreateTimeSeries();
+      process.env.HOSTNAME = 'test-host';
+      process.env.MONITORING_PROJECT_ID = 'test-project-1';
+      const timeSeries = [1, 2];
+
+      await createGcpTimeSeriesPublisher('test-project-2').publishTimeSeries(timeSeries);
+
+      expect(MetricServiceClient.prototype.createTimeSeries).toHaveBeenCalledWith({
+        name: 'projects/test-project-2',
+        timeSeries
+      });
+    });
+
     it('should push metrics to gcp in batches', async () => {
       stubCreateTimeSeries();
       process.env.HOSTNAME = 'test-host';

package/src/index.js

@@ -26,5 +26,6 @@ exposedFactoryInstanceHybrid.count = gcpMetricCollector.count;
 exposedFactoryInstanceHybrid.measureGcp = gcpMetricCollector.measure;
 exposedFactoryInstanceHybrid.startGcp = gcpMetricCollector.start;
 exposedFactoryInstanceHybrid.stopGcp = gcpMetricCollector.stop;
+exposedFactoryInstanceHybrid.createGcpMetricsCollector = gcpMetricCollector.createGcpMetricsCollector;
 
 module.exports = exposedFactoryInstanceHybrid;

package/test-helpers/test-helpers.js

@@ -14,7 +14,7 @@ exports.getPublishedMetrics = () => {
 
   return mock.calls.map(call =>
     call[0].timeSeries.map(series => ({
-      ...pick(series, ['metricKind', 'valueType']),
+      ...pick(series, ['metricKind', 'valueType', 'resource']),
       name: series.metric.type.replace('custom.googleapis.com/', ''),
       labels: omit(series.metric.labels, 'hostname'),
       startTime: series.points[0].interval.startTime.seconds * 1000,