@orion-js/echoes 2.10.3 → 2.11.0

package/lib/index.js

@@ -2,6 +2,8 @@
 
 var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
 
+var _interopRequireWildcard = require("@babel/runtime/helpers/interopRequireWildcard");
+
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
@@ -11,6 +13,12 @@ Object.defineProperty(exports, "startService", {
     return _startService.default;
   }
 });
+Object.defineProperty(exports, "stopService", {
+  enumerable: true,
+  get: function () {
+    return _startService.stopService;
+  }
+});
 Object.defineProperty(exports, "publish", {
   enumerable: true,
   get: function () {
@@ -30,7 +38,7 @@ Object.defineProperty(exports, "request", {
   }
 });
 
-var _startService = _interopRequireDefault(require("./startService"));
+var _startService = _interopRequireWildcard(require("./startService"));
 
 var _publish = _interopRequireDefault(require("./publish"));
 

package/lib/publish/index.js

@@ -18,7 +18,7 @@ var _serialize = _interopRequireDefault(require("./serialize"));
  */
 async function _default(options) {
   if (!_config.default.producer) {
-    throw new Error('You must initialize echoes configruation to use publish');
+    throw new Error('You must initialize echoes configuration to use publish');
   }
 
   const payload = {

package/lib/startService/KafkaManager.js

@@ -0,0 +1,181 @@
+"use strict";
+
+var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = void 0;
+
+var _kafkajs = require("kafkajs");
+
+var _types = _interopRequireDefault(require("../echo/types"));
+
+const HEARTBEAT_INTERVAL_SECONDS = 5; // This value must be less than the kafkajs session timeout
+
+const CHECK_JOIN_CONSUMER_INTERVAL_SECONDS = 30;
+const DEFAULT_PARTITIONS_CONSUMED_CONCURRENTLY = 4; // How many partitions to consume concurrently, adjust this with the members to partitions ratio to avoid idle consumers.
+
+const DEFAULT_MEMBERS_TO_PARTITIONS_RATIO = 1; // How many members are in comparison to partitions, this is used to determine if the consumer group has room for more members. Numbers over 1 leads to idle consumers. Numbers under 1 needs partitionsConsumedConcurrently to be more than 1.
+
+class KafkaManager {
+  constructor(options) {
+    this.kafka = new _kafkajs.Kafka(options.client);
+    this.options = options;
+    this.producer = this.kafka.producer(options.producer);
+    this.consumer = this.kafka.consumer(options.consumer);
+    this.topics = Object.keys(options.echoes).filter(key => options.echoes[key].type === _types.default.event);
+  }
+
+  async checkJoinConsumerGroupConditions() {
+    const admin = this.kafka.admin();
+
+    try {
+      await admin.connect();
+      const groupDescriptions = await admin.describeGroups([this.options.consumer.groupId]);
+      const group = groupDescriptions.groups[0];
+
+      if (group.state === 'Empty') {
+        console.info(`Echoes: Consumer group ${this.options.consumer.groupId} is empty, joining`);
+        return true;
+      }
+
+      const topicsMetadata = await admin.fetchTopicMetadata({
+        topics: this.topics
+      });
+      const totalPartitions = topicsMetadata.topics.reduce((acc, t) => acc + t.partitions.length, 0);
+      console.info(`Echoes: Consumer group ${this.options.consumer.groupId} has ${group.members.length} members and ${totalPartitions} partitions`);
+      const partitionsRatio = this.options.membersToPartitionsRatio || DEFAULT_MEMBERS_TO_PARTITIONS_RATIO;
+      const partitionsThreshold = Math.ceil(totalPartitions * partitionsRatio);
+
+      if (partitionsThreshold > group.members.length) {
+        console.info(`Echoes: Consumer group ${this.options.consumer.groupId} has room for more members ${group.members.length}/${partitionsThreshold}, joining`);
+        return true;
+      }
+    } catch (error) {
+      console.error(`Echoes: Error checking consumer group conditions, joining: ${error.message}`);
+      return true;
+    } finally {
+      await admin.disconnect().catch(error => {
+        console.error(`Echoes: Error disconnecting admin client: ${error.message}`);
+      });
+    }
+  }
+
+  async joinConsumerGroup() {
+    await this.consumer.connect();
+    await this.consumer.subscribe({
+      topics: this.topics
+    });
+    this.consumer.run({
+      partitionsConsumedConcurrently: this.options.partitionsConsumedConcurrently || DEFAULT_PARTITIONS_CONSUMED_CONCURRENTLY,
+      eachMessage: params => this.handleMessage(params)
+    });
+  }
+
+  async conditionalStart() {
+    if (await this.checkJoinConsumerGroupConditions()) {
+      await this.joinConsumerGroup();
+      return true;
+    }
+  }
+
+  async start() {
+    if (this.started) return;
+    await this.producer.connect();
+    this.started = await this.conditionalStart();
+    if (this.started) return;
+    console.info('Echoes: Delaying consumer group join, waiting for conditions to be met');
+    this.interval = setInterval(async () => {
+      this.started = await this.conditionalStart();
+      if (this.started) clearInterval(this.interval);
+    }, CHECK_JOIN_CONSUMER_INTERVAL_SECONDS * 1000);
+  }
+
+  async stop() {
+    console.warn("Echoes: Stopping echoes");
+
+    if (this.interval) {
+      clearInterval(this.interval);
+    }
+
+    if (this.consumer) {
+      await this.consumer.disconnect();
+    }
+
+    if (this.producer) {
+      await this.producer.disconnect();
+    }
+  }
+
+  async handleMessage(params) {
+    const echo = this.options.echoes[params.topic];
+
+    if (!echo || echo.type !== _types.default.event) {
+      console.warn(`Echoes: Received a message for an unknown topic: ${params.topic}, ignoring it`);
+      return;
+    }
+
+    let intervalsCount = 0;
+    const hInterval = setInterval(async () => {
+      await params.heartbeat().catch(error => {
+        console.warn(`Echoes: Error sending heartbeat: ${error.message}`);
+      });
+      intervalsCount++;
+
+      if (intervalsCount * HEARTBEAT_INTERVAL_SECONDS % 30 === 0) {
+        console.warn(`Echoes: Event is taking too long to process:  ${params.topic} ${intervalsCount * HEARTBEAT_INTERVAL_SECONDS}s`);
+      }
+    }, HEARTBEAT_INTERVAL_SECONDS * 1000);
+
+    try {
+      await echo.onMessage(params).catch(error => this.handleRetries(echo, params, error));
+    } catch (error) {
+      console.error(`Echoes: error processing a message: ${params.topic} ${error.message}`);
+      throw error;
+    } finally {
+      clearInterval(hInterval);
+    }
+  }
+
+  async handleRetries(echo, params, error) {
+    const {
+      message,
+      topic
+    } = params;
+    const retries = getRetries(message);
+
+    if (echo.attemptsBeforeDeadLetter === undefined || echo.attemptsBeforeDeadLetter === null) {
+      throw error;
+    }
+
+    const maxRetries = echo.attemptsBeforeDeadLetter || 0;
+    const exceededMaxRetries = retries >= maxRetries;
+    const nextTopic = exceededMaxRetries ? `DLQ-${topic}` : topic;
+    await this.producer.send({
+      topic: nextTopic,
+      messages: [{
+        value: message.value.toString(),
+        headers: {
+          retries: String(retries + 1),
+          errorMessage: error.message
+        }
+      }]
+    });
+
+    if (exceededMaxRetries) {
+      console.error(`Echoes: a message has reached the maximum number of retries, sending it to DLQ: ${nextTopic}`);
+    } else {
+      console.warn(`Echoes: a retryable message failed "${error.message}", re-sending it to topic: ${nextTopic}`);
+    }
+  }
+
+}
+
+function getRetries(message) {
+  if (!message || !message.headers || !message.headers.retries) return 0;
+  return Number.parseInt(message.headers.retries.toString());
+}
+
+var _default = KafkaManager;
+exports.default = _default;

package/lib/startService/index.js

@@ -6,57 +6,22 @@ Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.default = startService;
-
-var _kafkajs = require("kafkajs");
+exports.stopService = stopService;
 
 var _config = _interopRequireDefault(require("../config"));
 
 var _requestsHandler = _interopRequireDefault(require("../requestsHandler"));
 
-var _types = _interopRequireDefault(require("../echo/types"));
+var _KafkaManager = _interopRequireDefault(require("./KafkaManager"));
 
-const HEARTBEAT_INTERVAL_SECONDS = 3;
+let kafkaManager = null;
 
 async function startService(options) {
   if (options.client) {
-    const kafka = new _kafkajs.Kafka(options.client);
-    _config.default.producer = kafka.producer(options.producer);
-    _config.default.consumer = kafka.consumer(options.consumer);
-    await _config.default.producer.connect();
-    await _config.default.consumer.connect();
-
-    for (const topic in options.echoes) {
-      const echo = options.echoes[topic];
-      if (echo.type !== _types.default.event) continue;
-      await _config.default.consumer.subscribe({
-        topic,
-        fromBeginning: options.readTopicsFromBeginning || false
-      });
-    }
-
-    _config.default.consumer.run({
-      eachMessage: async params => {
-        const echo = options.echoes[params.topic];
-        if (!echo) return;
-        if (echo.type !== _types.default.event) return;
-        let intervalsCount = 0;
-        const interval = setInterval(async () => {
-          await params.heartbeat().catch(error => {
-            console.warn('Echoes: Error sending heartbeat:', error);
-          });
-          intervalsCount++;
-
-          if (intervalsCount % 10 === 0) {
-            console.warn(`Echoes: Event ${params.topic} is taking too long to process: ${intervalsCount * HEARTBEAT_INTERVAL_SECONDS}s`);
-          }
-        }, HEARTBEAT_INTERVAL_SECONDS * 1000);
-        await echo.onMessage(params).catch(error => {
-          clearInterval(interval);
-          throw error;
-        });
-        clearInterval(interval);
-      }
-    });
+    kafkaManager = new _KafkaManager.default(options);
+    await kafkaManager.start();
+    _config.default.producer = kafkaManager.producer;
+    _config.default.consumer = kafkaManager.consumer;
   }
 
   if (options.requests) {
@@ -67,4 +32,12 @@ async function startService(options) {
       _config.default.requests.startHandler(_requestsHandler.default);
     }
   }
+}
+
+async function stopService() {
+  if (kafkaManager) {
+    console.info("Stoping echoes...");
+    await kafkaManager.stop();
+    console.info("Echoes stopped");
+  }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@orion-js/echoes",
-  "version": "2.10.3",
+  "version": "2.11.0",
   "main": "lib/index.js",
   "author": "nicolaslopezj",
   "license": "MIT",
@@ -29,5 +29,5 @@
   "publishConfig": {
     "access": "public"
   },
-  "gitHead": "30a4b8f24d4fd596818058305488368844920479"
+  "gitHead": "7985a356f3e823b338d3d1c478acc29c5b10e1de"
 }