@orion-js/echoes 3.12.0 → 3.13.1

package/lib/startService/KafkaManager.js

@@ -0,0 +1,146 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const kafkajs_1 = require("kafkajs");
+const types_1 = __importDefault(require("../echo/types"));
+const HEARTBEAT_INTERVAL_SECONDS = 5; // This value must be less than the kafkajs session timeout
+const CHECK_JOIN_CONSUMER_INTERVAL_SECONDS = 30;
+const DEFAULT_PARTITIONS_CONSUMED_CONCURRENTLY = 4; // How many partitions to consume concurrently, adjust this with the members to partitions ratio to avoid idle consumers.
+const DEFAULT_MEMBERS_TO_PARTITIONS_RATIO = 1; // How many members are in comparison to partitions, this is used to determine if the consumer group has room for more members. Numbers over 1 leads to idle consumers. Numbers under 1 needs partitionsConsumedConcurrently to be more than 1.
+/**
+ * Manages the Kafka connection and the consumers.
+ */
+class KafkaManager {
+    constructor(options) {
+        this.kafka = new kafkajs_1.Kafka(options.client);
+        this.options = options;
+        this.producer = this.kafka.producer(options.producer);
+        this.consumer = this.kafka.consumer(options.consumer);
+        this.topics = Object.keys(options.echoes).filter(key => options.echoes[key].type === types_1.default.event);
+    }
+    async checkJoinConsumerGroupConditions() {
+        const admin = this.kafka.admin();
+        try {
+            await admin.connect();
+            const groupDescriptions = await admin.describeGroups([this.options.consumer.groupId]);
+            const group = groupDescriptions.groups[0];
+            if (group.state === 'Empty') {
+                console.info(`Echoes: Consumer group ${this.options.consumer.groupId} is empty, joining`);
+                return true;
+            }
+            const topicsMetadata = await admin.fetchTopicMetadata({ topics: this.topics });
+            const totalPartitions = topicsMetadata.topics.reduce((acc, topic) => acc + topic.partitions.length, 0);
+            console.info(`Echoes: Consumer group ${this.options.consumer.groupId} has ${group.members.length} members and ${totalPartitions} partitions`);
+            const partitionsRatio = this.options.membersToPartitionsRatio || DEFAULT_MEMBERS_TO_PARTITIONS_RATIO;
+            const partitionsThreshold = Math.ceil(totalPartitions * partitionsRatio);
+            if (partitionsThreshold > group.members.length) {
+                console.info(`Echoes: Consumer group ${this.options.consumer.groupId} has room for more members ${group.members.length}/${partitionsThreshold}, joining`);
+                return true;
+            }
+        }
+        catch (error) {
+            console.error(`Echoes: Error checking consumer group conditions, join: ${error.message}`);
+            return true;
+        }
+        finally {
+            await admin.disconnect().catch(error => {
+                console.error(`Echoes: Error disconnecting admin client: ${error.message}`);
+            });
+        }
+    }
+    async joinConsumerGroup() {
+        await this.consumer.connect();
+        await this.consumer.subscribe({ topics: this.topics });
+        await this.consumer.run({
+            partitionsConsumedConcurrently: this.options.partitionsConsumedConcurrently || DEFAULT_PARTITIONS_CONSUMED_CONCURRENTLY,
+            eachMessage: params => this.handleMessage(params),
+        });
+    }
+    async conditionalStart() {
+        if (await this.checkJoinConsumerGroupConditions()) {
+            await this.joinConsumerGroup();
+            return true;
+        }
+    }
+    async start() {
+        if (this.started)
+            return;
+        await this.producer.connect();
+        this.started = await this.conditionalStart();
+        if (this.started)
+            return;
+        console.info('Echoes: Delaying consumer group join, waiting for conditions to be met');
+        this.interval = setInterval(async () => {
+            this.started = await this.conditionalStart();
+            if (this.started)
+                clearInterval(this.interval);
+        }, CHECK_JOIN_CONSUMER_INTERVAL_SECONDS * 1000);
+    }
+    async stop() {
+        console.warn('Echoes: Stopping echoes');
+        if (this.interval)
+            clearInterval(this.interval);
+        if (this.consumer)
+            await this.consumer.disconnect();
+        if (this.producer)
+            await this.producer.disconnect();
+    }
+    async handleMessage(params) {
+        const echo = this.options.echoes[params.topic];
+        if (!echo || echo.type !== types_1.default.event) {
+            console.warn(`Echoes: Received a message for an unknown topic: ${params.topic}, ignoring it`);
+            return;
+        }
+        let intervalsCount = 0;
+        const heartbeatInterval = setInterval(async () => {
+            await params.heartbeat().catch(error => {
+                console.warn(`Echoes: Error sending heartbeat: ${error.message}`);
+            });
+            intervalsCount++;
+            if ((intervalsCount * HEARTBEAT_INTERVAL_SECONDS) % 30 === 0) {
+                console.warn(`Echoes: Event is taking too long to process: ${params.topic} ${intervalsCount * HEARTBEAT_INTERVAL_SECONDS}s`);
+            }
+        }, HEARTBEAT_INTERVAL_SECONDS * 1000);
+        try {
+            await echo.onMessage(params).catch(error => this.handleRetries(echo, params, error));
+        }
+        catch (error) {
+            console.error(`Echoes: error processing a message: ${params.topic} ${error.message}`);
+            throw error;
+        }
+        finally {
+            clearInterval(heartbeatInterval);
+        }
+    }
+    async handleRetries(echo, params, error) {
+        const { message, topic } = params;
+        const retries = Number.parseInt(message?.headers?.retries?.toString() || '0', 10);
+        if (echo.attemptsBeforeDeadLetter === undefined || echo.attemptsBeforeDeadLetter === null) {
+            throw error;
+        }
+        const maxRetries = echo.attemptsBeforeDeadLetter || 0;
+        const exceededMaxRetries = retries >= maxRetries;
+        const nextTopic = exceededMaxRetries ? `DLQ-${topic}` : topic;
+        await this.producer.send({
+            topic: nextTopic,
+            messages: [
+                {
+                    value: message.value.toString(),
+                    headers: {
+                        retries: String(retries + 1),
+                        error: error.message,
+                    },
+                },
+            ],
+        });
+        if (exceededMaxRetries) {
+            console.error(`Echoes: a message has reached the maximum number of retries, sending it to DLQ: ${nextTopic}`);
+        }
+        else {
+            console.warn(`Echoes: a retryable message failed "${error.message}", re-sending it to topic: ${nextTopic}`);
+        }
+    }
+}
+exports.default = KafkaManager;

package/lib/startService/index.d.ts

@@ -0,0 +1,3 @@
+import { EchoesOptions } from '../types';
+export default function startService(options: EchoesOptions): Promise<void>;
+export declare function stopService(): Promise<void>;

package/lib/startService/index.js

@@ -0,0 +1,33 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.stopService = void 0;
+const config_1 = __importDefault(require("../config"));
+const requestsHandler_1 = __importDefault(require("../requestsHandler"));
+const KafkaManager_1 = __importDefault(require("./KafkaManager"));
+const http_1 = require("@orion-js/http");
+let kafkaManager = null;
+async function startService(options) {
+    config_1.default.echoes = options.echoes;
+    if (options.requests) {
+        config_1.default.requests = options.requests;
+        (0, http_1.registerRoute)((0, requestsHandler_1.default)(options));
+    }
+    if (options.client) {
+        kafkaManager = new KafkaManager_1.default(options);
+        await kafkaManager.start();
+        config_1.default.producer = kafkaManager.producer;
+        config_1.default.consumer = kafkaManager.consumer;
+    }
+}
+exports.default = startService;
+async function stopService() {
+    if (kafkaManager) {
+        console.info('Stoping echoes...');
+        await kafkaManager.stop();
+        console.info('Echoes stopped');
+    }
+}
+exports.stopService = stopService;

package/lib/tests/requests/index.test.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/tests/requests/index.test.js

@@ -0,0 +1,99 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const __1 = require("../..");
+const supertest_1 = __importDefault(require("supertest"));
+const http_1 = require("@orion-js/http");
+const schema_1 = require("@orion-js/schema");
+describe('Test echoes requests', () => {
+    const makeRequest = async (options) => {
+        const app = (0, http_1.getApp)();
+        const response = await (0, supertest_1.default)(app).post('/echoes-services').send(options.data);
+        return {
+            statusCode: response.statusCode,
+            data: response.body,
+        };
+    };
+    it('Should start echoes requests service', () => {
+        const echoes = {
+            test: (0, __1.echo)({
+                type: 'request',
+                async resolve() { },
+            }),
+        };
+        (0, __1.startService)({
+            echoes,
+            requests: {
+                key: 'secret',
+                services: {},
+            },
+        });
+    });
+    it('Should be able to make a echoes request passing dates as params', async () => {
+        expect.assertions(5);
+        const echoes = {
+            test: (0, __1.echo)({
+                type: 'request',
+                async resolve(params) {
+                    expect(params.hello).toBe('world');
+                    expect(params.date).toBeInstanceOf(Date);
+                    return {
+                        text: 'Hello world',
+                        date: params.date,
+                    };
+                },
+            }),
+        };
+        (0, __1.startService)({
+            echoes,
+            requests: {
+                key: 'secret',
+                services: { test: 'mockURL' },
+                makeRequest,
+            },
+        });
+        const date = new Date();
+        const result = await (0, __1.request)({
+            method: 'test',
+            service: 'test',
+            params: {
+                hello: 'world',
+                date,
+            },
+        });
+        expect(result.text).toBe('Hello world');
+        expect(result.date).toBeInstanceOf(Date);
+        expect(result.date.getTime()).toBe(date.getTime());
+    });
+    it('should pass errors to Orion errors', async () => {
+        const echoes = {
+            test: (0, __1.echo)({
+                type: 'request',
+                async resolve() {
+                    throw new schema_1.ValidationError({ hello: 'world' });
+                },
+            }),
+        };
+        (0, __1.startService)({
+            echoes,
+            requests: {
+                key: 'secret',
+                services: { test: 'mockURL' },
+                makeRequest,
+            },
+        });
+        expect.assertions(1);
+        try {
+            await (0, __1.request)({
+                method: 'test',
+                service: 'test',
+                params: {},
+            });
+        }
+        catch (error) {
+            expect(error.validationErrors).toEqual({ hello: 'world' });
+        }
+    });
+});

package/lib/types.d.ts

@@ -0,0 +1,98 @@
+import { ConsumerConfig, KafkaConfig, ProducerConfig, Consumer, Producer, EachMessagePayload } from 'kafkajs';
+export interface EchoConfig {
+    type: 'event' | 'request';
+    resolve(params: any, context?: any): Promise<any>;
+    attemptsBeforeDeadLetter?: number;
+}
+export interface EchoType extends EchoConfig {
+    onMessage(messageData: EachMessagePayload): Promise<void>;
+    onRequest(serializedParams: string): any;
+}
+export interface PublishOptions<TParams = any> {
+    topic: string;
+    params: TParams;
+    acks?: number;
+    timeout?: number;
+}
+export interface RequestOptions<TParams> {
+    method: string;
+    service: string;
+    params: TParams;
+    retries?: number;
+    timeout?: number;
+}
+export interface RequestHandlerResponse {
+    result?: any;
+    error?: any;
+    isUserError?: boolean;
+    isValidationError?: boolean;
+    errorInfo?: {
+        error: string;
+        message: string;
+        extra?: any;
+        validationErrors?: any;
+    };
+}
+export interface MakeRequestParams {
+    url: string;
+    retries?: number;
+    timeout?: number;
+    data: {
+        body: object;
+        signature: string;
+    };
+}
+export interface RequestMakerResult {
+    statusCode: number;
+    data: object;
+}
+export type RequestMaker = (options: MakeRequestParams) => Promise<RequestMakerResult>;
+export interface RequestsConfig {
+    /**
+     * The secret key used to sign all requests. Shared between all your services.
+     * You can also set the env var echoes_password or process.env.ECHOES_PASSWORD
+     */
+    key?: string;
+    /**
+     * The path of the echoes http receiver. Defaults to /echoes-services
+     */
+    handlerPath?: string;
+    /**
+     * Map of all the services that have echoes requests handlers
+     */
+    services?: {
+        [key: string]: string;
+    };
+    /**
+     * A custom function that make the requests to the services. Uses axios by default
+     */
+    makeRequest?: RequestMaker;
+}
+export interface EchoesMap {
+    [key: string]: EchoType;
+}
+export interface EchoesOptions {
+    client?: KafkaConfig;
+    producer?: ProducerConfig;
+    consumer?: ConsumerConfig;
+    requests?: RequestsConfig;
+    echoes: EchoesMap;
+    /**
+     * Defaults to true. When true, allows a reconnecting service to read missed messages.
+     */
+    readTopicsFromBeginning?: boolean;
+    /**
+     * Defaults to 4. How many partitions to consume concurrently, adjust this with the members to partitions ratio to avoid idle consumers.
+     */
+    partitionsConsumedConcurrently?: number;
+    /**
+     * Defaults to 1. How many members are in comparison to partitions, this is used to determine if the consumer group has room for more members. Numbers over 1 leads to idle consumers. Numbers under 1 needs partitionsConsumedConcurrently to be more than 1.
+     */
+    membersToPartitionsRatio?: number;
+}
+export interface EchoesConfigHandler {
+    producer?: Producer;
+    consumer?: Consumer;
+    requests?: RequestsConfig;
+    echoes?: EchoesMap;
+}

package/lib/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/package.json

@@ -1,28 +1,27 @@
 {
   "name": "@orion-js/echoes",
-  "version": "3.12.0",
-  "main": "./dist/index.cjs",
-  "types": "./dist/index.d.ts",
+  "version": "3.13.1",
+  "main": "lib/index.js",
+  "types": "lib/index.d.ts",
   "files": [
-    "dist"
+    "/lib"
   ],
   "author": "nicolaslopezj",
   "license": "MIT",
   "scripts": {
-    "test": "bun test",
+    "test": "jest",
     "prepare": "yarn run build",
-    "clean": "rm -rf ./dist",
-    "build": "bun run build.ts",
+    "clean": "rm -rf ./lib",
+    "build": "yarn run clean && tsc",
     "watch": "tsc -w",
-    "upgrade-interactive": "yarn upgrade-interactive",
-    "dev": "bun --watch src/index.ts"
+    "upgrade-interactive": "yarn upgrade-interactive"
   },
   "dependencies": {
-    "@orion-js/env": "^4.0.0-alpha.2",
-    "@orion-js/helpers": "^4.0.0-alpha.2",
-    "@orion-js/http": "^4.0.0-alpha.2",
-    "@orion-js/schema": "^4.0.0-alpha.2",
-    "@orion-js/services": "^4.0.0-alpha.2",
+    "@orion-js/env": "^3.13.1",
+    "@orion-js/helpers": "^3.13.1",
+    "@orion-js/http": "^3.13.1",
+    "@orion-js/schema": "^3.13.1",
+    "@orion-js/services": "^3.13.1",
     "axios": "^0.24.0",
     "jssha": "^3.2.0",
     "kafkajs": "^2.2.4",
@@ -30,21 +29,16 @@
     "serialize-javascript": "^6.0.0"
   },
   "devDependencies": {
+    "@types/jest": "27.0.2",
     "@types/supertest": "2.0.11",
+    "jest": "27.3.1",
     "reflect-metadata": "^0.1.13",
     "supertest": "^6.1.6",
-    "typescript": "^5.4.5",
-    "@types/bun": "^1.2.4"
+    "ts-jest": "27.0.7",
+    "typescript": "^4.4.4"
   },
   "publishConfig": {
     "access": "public"
   },
-  "gitHead": "9fd28b6f6b348cebc9f0dc207805647969277372",
-  "type": "module",
-  "module": "./dist/index.js",
-  "exports": {
-    "types": "./dist/index.d.ts",
-    "import": "./dist/index.js",
-    "require": "./dist/index.cjs"
-  }
-}
+  "gitHead": "18ab5a9711863e6bd3d3c61e7bb8f9067afca82c"
+}