@orion-js/echoes 3.10.0 → 3.11.1

package/lib/index.d.ts

@@ -1,7 +1,7 @@
-import startService from './startService';
+import startService, { stopService } from './startService';
 import publish from './publish';
 import echo from './echo';
 import request from './request';
 export * from './types';
 export * from './service';
-export { publish, startService, echo, request };
+export { publish, startService, stopService, echo, request };

package/lib/index.js

@@ -1,11 +1,27 @@
 "use strict";
 var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
     if (k2 === undefined) k2 = k;
-    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
 }) : (function(o, m, k, k2) {
     if (k2 === undefined) k2 = k;
     o[k2] = m[k];
 }));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
 var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
@@ -13,9 +29,10 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.request = exports.echo = exports.startService = exports.publish = void 0;
-const startService_1 = __importDefault(require("./startService"));
+exports.request = exports.echo = exports.stopService = exports.startService = exports.publish = void 0;
+const startService_1 = __importStar(require("./startService"));
 exports.startService = startService_1.default;
+Object.defineProperty(exports, "stopService", { enumerable: true, get: function () { return startService_1.stopService; } });
 const publish_1 = __importDefault(require("./publish"));
 exports.publish = publish_1.default;
 const echo_1 = __importDefault(require("./echo"));

package/lib/publish/index.js

@@ -13,7 +13,7 @@ async function publish(options) {
         throw new Error('You must initialize echoes configruation to use publish');
     }
     const payload = {
-        params: options.params
+        params: options.params,
     };
     return await config_1.default.producer.send({
         acks: options.acks,
@@ -21,9 +21,9 @@ async function publish(options) {
         topic: options.topic,
         messages: [
             {
-                value: (0, serialize_1.default)(payload)
-            }
-        ]
+                value: (0, serialize_1.default)(payload),
+            },
+        ],
     });
 }
 exports.default = publish;

package/lib/startService/KafkaManager.d.ts

@@ -0,0 +1,24 @@
+/// <reference types="node" />
+import { Kafka, EachMessagePayload, Producer, Consumer } from 'kafkajs';
+import { EchoesOptions, EchoType } from '../types';
+/**
+ * Manages the Kafka connection and the consumers.
+ */
+declare class KafkaManager {
+    kafka: Kafka;
+    options: EchoesOptions;
+    producer: Producer;
+    consumer: Consumer;
+    topics: string[];
+    started: boolean;
+    interval: NodeJS.Timeout;
+    constructor(options: EchoesOptions);
+    checkJoinConsumerGroupConditions(): Promise<boolean>;
+    joinConsumerGroup(): Promise<void>;
+    conditionalStart(): Promise<boolean>;
+    start(): Promise<void>;
+    stop(): Promise<void>;
+    handleMessage(params: EachMessagePayload): Promise<void>;
+    handleRetries(echo: EchoType, params: EachMessagePayload, error: Error): Promise<void>;
+}
+export default KafkaManager;

package/lib/startService/KafkaManager.js

@@ -0,0 +1,146 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const kafkajs_1 = require("kafkajs");
+const types_1 = __importDefault(require("../echo/types"));
+const HEARTBEAT_INTERVAL_SECONDS = 5; // This value must be less than the kafkajs session timeout
+const CHECK_JOIN_CONSUMER_INTERVAL_SECONDS = 30;
+const DEFAULT_PARTITIONS_CONSUMED_CONCURRENTLY = 4; // How many partitions to consume concurrently, adjust this with the members to partitions ratio to avoid idle consumers.
+const DEFAULT_MEMBERS_TO_PARTITIONS_RATIO = 1; // How many members are in comparison to partitions, this is used to determine if the consumer group has room for more members. Numbers over 1 leads to idle consumers. Numbers under 1 needs partitionsConsumedConcurrently to be more than 1.
+/**
+ * Manages the Kafka connection and the consumers.
+ */
+class KafkaManager {
+    constructor(options) {
+        this.kafka = new kafkajs_1.Kafka(options.client);
+        this.options = options;
+        this.producer = this.kafka.producer(options.producer);
+        this.consumer = this.kafka.consumer(options.consumer);
+        this.topics = Object.keys(options.echoes).filter(key => options.echoes[key].type === types_1.default.event);
+    }
+    async checkJoinConsumerGroupConditions() {
+        const admin = this.kafka.admin();
+        try {
+            await admin.connect();
+            const groupDescriptions = await admin.describeGroups([this.options.consumer.groupId]);
+            const group = groupDescriptions.groups[0];
+            if (group.state === 'Empty') {
+                console.info(`Echoes: Consumer group ${this.options.consumer.groupId} is empty, joining`);
+                return true;
+            }
+            const topicsMetadata = await admin.fetchTopicMetadata({ topics: this.topics });
+            const totalPartitions = topicsMetadata.topics.reduce((acc, topic) => acc + topic.partitions.length, 0);
+            console.info(`Echoes: Consumer group ${this.options.consumer.groupId} has ${group.members.length} members and ${totalPartitions} partitions`);
+            const partitionsRatio = this.options.membersToPartitionsRatio || DEFAULT_MEMBERS_TO_PARTITIONS_RATIO;
+            const partitionsThreshold = Math.ceil(totalPartitions * partitionsRatio);
+            if (partitionsThreshold > group.members.length) {
+                console.info(`Echoes: Consumer group ${this.options.consumer.groupId} has room for more members ${group.members.length}/${partitionsThreshold}, joining`);
+                return true;
+            }
+        }
+        catch (error) {
+            console.error(`Echoes: Error checking consumer group conditions, join: ${error.message}`);
+            return true;
+        }
+        finally {
+            await admin.disconnect().catch(error => {
+                console.error(`Echoes: Error disconnecting admin client: ${error.message}`);
+            });
+        }
+    }
+    async joinConsumerGroup() {
+        await this.consumer.connect();
+        await this.consumer.subscribe({ topics: this.topics });
+        await this.consumer.run({
+            partitionsConsumedConcurrently: this.options.partitionsConsumedConcurrently || DEFAULT_PARTITIONS_CONSUMED_CONCURRENTLY,
+            eachMessage: params => this.handleMessage(params),
+        });
+    }
+    async conditionalStart() {
+        if (await this.checkJoinConsumerGroupConditions()) {
+            await this.joinConsumerGroup();
+            return true;
+        }
+    }
+    async start() {
+        if (this.started)
+            return;
+        await this.producer.connect();
+        this.started = await this.conditionalStart();
+        if (this.started)
+            return;
+        console.info('Echoes: Delaying consumer group join, waiting for conditions to be met');
+        this.interval = setInterval(async () => {
+            this.started = await this.conditionalStart();
+            if (this.started)
+                clearInterval(this.interval);
+        }, CHECK_JOIN_CONSUMER_INTERVAL_SECONDS * 1000);
+    }
+    async stop() {
+        console.warn('Echoes: Stopping echoes');
+        if (this.interval)
+            clearInterval(this.interval);
+        if (this.consumer)
+            await this.consumer.disconnect();
+        if (this.producer)
+            await this.producer.disconnect();
+    }
+    async handleMessage(params) {
+        const echo = this.options.echoes[params.topic];
+        if (!echo || echo.type !== types_1.default.event) {
+            console.warn(`Echoes: Received a message for an unknown topic: ${params.topic}, ignoring it`);
+            return;
+        }
+        let intervalsCount = 0;
+        const heartbeatInterval = setInterval(async () => {
+            await params.heartbeat().catch(error => {
+                console.warn(`Echoes: Error sending heartbeat: ${error.message}`);
+            });
+            intervalsCount++;
+            if ((intervalsCount * HEARTBEAT_INTERVAL_SECONDS) % 30 === 0) {
+                console.warn(`Echoes: Event is taking too long to process: ${params.topic} ${intervalsCount * HEARTBEAT_INTERVAL_SECONDS}s`);
+            }
+        }, HEARTBEAT_INTERVAL_SECONDS * 1000);
+        try {
+            await echo.onMessage(params).catch(error => this.handleRetries(echo, params, error));
+        }
+        catch (error) {
+            console.error(`Echoes: error processing a message: ${params.topic} ${error.message}`);
+            throw error;
+        }
+        finally {
+            clearInterval(heartbeatInterval);
+        }
+    }
+    async handleRetries(echo, params, error) {
+        const { message, topic } = params;
+        const retries = Number.parseInt(message?.headers?.retries?.toString() || '0', 10);
+        if (echo.attemptsBeforeDeadLetter === undefined || echo.attemptsBeforeDeadLetter === null) {
+            throw error;
+        }
+        const maxRetries = echo.attemptsBeforeDeadLetter || 0;
+        const exceededMaxRetries = retries >= maxRetries;
+        const nextTopic = exceededMaxRetries ? `DLQ-${topic}` : topic;
+        await this.producer.send({
+            topic: nextTopic,
+            messages: [
+                {
+                    value: message.value.toString(),
+                    headers: {
+                        retries: String(retries + 1),
+                        error: error.message,
+                    },
+                },
+            ],
+        });
+        if (exceededMaxRetries) {
+            console.error(`Echoes: a message has reached the maximum number of retries, sending it to DLQ: ${nextTopic}`);
+        }
+        else {
+            console.warn(`Echoes: a retryable message failed "${error.message}", re-sending it to topic: ${nextTopic}`);
+        }
+    }
+}
+exports.default = KafkaManager;

package/lib/startService/index.d.ts

@@ -1,2 +1,3 @@
 import { EchoesOptions } from '../types';
 export default function startService(options: EchoesOptions): Promise<void>;
+export declare function stopService(): Promise<void>;

package/lib/startService/index.js

@@ -3,11 +3,12 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-const kafkajs_1 = require("kafkajs");
+exports.stopService = void 0;
 const config_1 = __importDefault(require("../config"));
 const requestsHandler_1 = __importDefault(require("../requestsHandler"));
-const types_1 = __importDefault(require("../echo/types"));
+const KafkaManager_1 = __importDefault(require("./KafkaManager"));
 const http_1 = require("@orion-js/http");
+let kafkaManager = null;
 async function startService(options) {
     config_1.default.echoes = options.echoes;
     if (options.requests) {
@@ -15,30 +16,18 @@ async function startService(options) {
         (0, http_1.registerRoute)((0, requestsHandler_1.default)(options));
     }
     if (options.client) {
-        const kafka = new kafkajs_1.Kafka(options.client);
-        config_1.default.producer = kafka.producer(options.producer);
-        config_1.default.consumer = kafka.consumer(options.consumer);
-        await config_1.default.producer.connect();
-        await config_1.default.consumer.connect();
-        for (const topic in options.echoes) {
-            const echo = options.echoes[topic];
-            if (echo.type !== types_1.default.event)
-                continue;
-            await config_1.default.consumer.subscribe({
-                topic,
-                fromBeginning: options.readTopicsFromBeginning === false ? false : true
-            });
-        }
-        config_1.default.consumer.run({
-            eachMessage: async (payload) => {
-                const echo = options.echoes[payload.topic];
-                if (!echo)
-                    return;
-                if (echo.type !== types_1.default.event)
-                    return;
-                await echo.onMessage(payload);
-            }
-        });
+        kafkaManager = new KafkaManager_1.default(options);
+        await kafkaManager.start();
+        config_1.default.producer = kafkaManager.producer;
+        config_1.default.consumer = kafkaManager.consumer;
     }
 }
 exports.default = startService;
+async function stopService() {
+    if (kafkaManager) {
+        console.info('Stoping echoes...');
+        await kafkaManager.stop();
+        console.info('Echoes stopped');
+    }
+}
+exports.stopService = stopService;

package/lib/types.d.ts

@@ -2,6 +2,7 @@ import { ConsumerConfig, KafkaConfig, ProducerConfig, Consumer, Producer, EachMe
 export interface EchoConfig {
     type: 'event' | 'request';
     resolve(params: any, context?: any): Promise<any>;
+    attemptsBeforeDeadLetter?: number;
 }
 export interface EchoType extends EchoConfig {
     onMessage(messageData: EachMessagePayload): Promise<void>;
@@ -45,7 +46,7 @@ export interface RequestMakerResult {
     statusCode: number;
     data: object;
 }
-export declare type RequestMaker = (options: MakeRequestParams) => Promise<RequestMakerResult>;
+export type RequestMaker = (options: MakeRequestParams) => Promise<RequestMakerResult>;
 export interface RequestsConfig {
     /**
      * The secret key used to sign all requests. Shared between all your services.
@@ -80,6 +81,14 @@ export interface EchoesOptions {
      * Defaults to true. When true, allows a reconnecting service to read missed messages.
      */
     readTopicsFromBeginning?: boolean;
+    /**
+     * Defaults to 4. How many partitions to consume concurrently, adjust this with the members to partitions ratio to avoid idle consumers.
+     */
+    partitionsConsumedConcurrently?: number;
+    /**
+     * Defaults to 1. How many members are in comparison to partitions, this is used to determine if the consumer group has room for more members. Numbers over 1 leads to idle consumers. Numbers under 1 needs partitionsConsumedConcurrently to be more than 1.
+     */
+    membersToPartitionsRatio?: number;
 }
 export interface EchoesConfigHandler {
     producer?: Producer;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@orion-js/echoes",
-  "version": "3.10.0",
+  "version": "3.11.1",
   "main": "lib/index.js",
   "types": "lib/index.d.ts",
   "files": [
@@ -19,17 +19,16 @@
   "dependencies": {
     "@orion-js/env": "^3.10.0",
     "@orion-js/helpers": "^3.10.0",
-    "@orion-js/http": "^3.10.0",
+    "@orion-js/http": "^3.11.1",
     "@orion-js/schema": "^3.10.0",
-    "@orion-js/services": "^3.10.0",
+    "@orion-js/services": "^3.11.1",
     "axios": "^0.24.0",
     "jssha": "^3.2.0",
-    "kafkajs": "^1.15.0",
+    "kafkajs": "^2.2.4",
     "lodash": "^4.17.21",
     "serialize-javascript": "^6.0.0"
   },
   "devDependencies": {
-    "@shelf/jest-mongodb": "^2.1.0",
     "@types/jest": "27.0.2",
     "@types/supertest": "2.0.11",
     "jest": "27.3.1",
@@ -41,5 +40,5 @@
   "publishConfig": {
     "access": "public"
   },
-  "gitHead": "68036a0293066cc23feb3000ab44e58600540615"
+  "gitHead": "6d60b64d2b61ff2a74884cf4eb4eea806c2231c2"
 }