@autofleet/rabbit 2.3.4 → 2.3.6

package/.env

@@ -0,0 +1 @@
+RABBITMQ_SERVICE_HOST=127.0.0.1:5673

package/dist/index-new.d.ts

@@ -0,0 +1,85 @@
+/// <reference types="node" />
+import { Options, ConsumeMessage } from 'amqplib';
+import { AmqpConnectionManager, ChannelWrapper } from 'amqp-connection-manager';
+import { EventEmitter } from 'events';
+import { RedisConfig } from './redis';
+export interface IAfRabbitMq {
+    ack: any;
+    nack: any;
+    assertChannel: any;
+    assertExchange: any;
+    assertQueue: any;
+    consume: any;
+    consumeFromExchange: any;
+    publish: any;
+    sendToQueue: any;
+    redisClient?: any;
+}
+interface ExchangesCache {
+    [key: string]: any;
+}
+interface QueuesCache {
+    [key: string]: any;
+}
+declare type CustomMessageHeaders = {
+    redisTimestampValidationKey?: string;
+};
+declare type RedisLockType = (args0?: string, arg1?: number) => Promise<any>;
+declare type ConsumeMessageOrNull = ConsumeMessage | null;
+interface ConsumeOptions {
+    retries?: number;
+    deadMessageTtl?: number;
+    limit?: number;
+    lockTimeout?: number;
+    useConsumeWithLock?: boolean;
+}
+declare type CallbackFunction = (msg: ConsumeMessage, ack: Function, nack: Function) => Promise<any>;
+export interface AfRabbitOptions {
+    disableReconnect?: boolean;
+    host?: string;
+}
+declare class RabbitMq implements IAfRabbitMq {
+    static parseMsg(msg: any): any;
+    static validateName(type: string, name: string): void;
+    static getPublishOptions(customHeaders?: CustomMessageHeaders): {
+        timestamp: number;
+        headers: {
+            redisTimestampValidationKey?: string | undefined;
+            creationTimestamp: number;
+        };
+    };
+    PUBLISH_TIMEOUT: number;
+    PUBLISH_ERROR_MSG: string;
+    DISCONNECT_MSG: string;
+    RESCONNECT_MSG: string;
+    channel: ChannelWrapper | null;
+    connection: AmqpConnectionManager | null;
+    em: EventEmitter;
+    creatingConnection: boolean;
+    exchanges: ExchangesCache;
+    queues: QueuesCache;
+    host: string | null;
+    options: AfRabbitOptions | undefined;
+    redisClient: any;
+    redisLock?: RedisLockType;
+    constructor(options?: AfRabbitOptions, redisConfig?: RedisConfig);
+    private shouldConsumeMessageByTimestamp;
+    ack: (channel: ChannelWrapper, msg: ConsumeMessageOrNull, shouldUpdateRedisTimestamp?: boolean, releaseLock?: null) => (userMsg: ConsumeMessage) => Promise<any>;
+    nack: (channel: ChannelWrapper, queue: string, options: any, deadQueueOptions: Options.AssertQueue, msg: ConsumeMessageOrNull, releaseLock: any) => (userMsg: ConsumeMessageOrNull, { skipRetry, }?: any) => Promise<any>;
+    getConnection(): Promise<AmqpConnectionManager>;
+    getNewChannel(): Promise<ChannelWrapper>;
+    assertChannel(): Promise<ChannelWrapper>;
+    assertExchange(exchangeName: string, options?: any): Promise<any>;
+    getQueueLength(queue: string): Promise<import("amqplib").Replies.AssertQueue>;
+    bindQueue(queue: string, exchange: string): Promise<void>;
+    assertQueue(queueName: string, options?: Options.AssertQueue): Promise<any>;
+    consume(queue: string, callback: CallbackFunction, options?: ConsumeOptions): Promise<any>;
+    private lockRedisIfNeeded;
+    private unlockRedisIfNeeded;
+    private consumeFromRabbit;
+    consumeFromExchange(queue: string, exchange: string, callback: CallbackFunction, options?: ConsumeOptions): Promise<any>;
+    publish(exchange: string, content: any, customHeaders?: any): Promise<boolean>;
+    sendToQueue(queue: string, content: any, options?: any, customHeaders?: any): Promise<boolean>;
+    isConnected(): Promise<boolean>;
+}
+export default RabbitMq;

package/dist/index-new.js

@@ -0,0 +1,327 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+/* eslint-disable @typescript-eslint/ban-ts-comment,@typescript-eslint/ban-types,@typescript-eslint/no-unused-vars,consistent-return,no-async-promise-executor,no-param-reassign,no-empty */
+// eslint-disable-next-line max-classes-per-file
+const moment_1 = __importDefault(require("moment"));
+const redis_lock_1 = __importDefault(require("redis-lock"));
+const amqp_connection_manager_1 = require("amqp-connection-manager");
+const events_1 = require("events");
+// import { newTrace, traceTypes } from '@autofleet/outbreak';
+const util_1 = require("util");
+const logger_1 = __importDefault(require("./logger"));
+const rabbitError_1 = __importDefault(require("./rabbitError"));
+const redis_1 = __importDefault(require("./redis"));
+const DEFAULT_DEAD_TTL_TWO_DAYS = 60000 * 60 * 48;
+const DEFAULT_LOCK_TIMEOUT = 1000 * 5;
+const RETRY_HEADER = 'x-retry-count';
+const DEFAULT_USE_CONSUME_WITH_LOCK = false;
+const defaultOptions = {
+    limit: 1,
+    retries: 1,
+    deadMessageTtl: DEFAULT_DEAD_TTL_TWO_DAYS,
+    lockTimeout: DEFAULT_LOCK_TIMEOUT,
+    useConsumeWithLock: DEFAULT_USE_CONSUME_WITH_LOCK,
+};
+const assertExchangeFanout = (c, exchangeName) => __awaiter(void 0, void 0, void 0, function* () { return c.assertExchange(exchangeName, 'fanout'); });
+const connectionCreatedConst = 'connectionCreated';
+const wrapSetImmediate = (callback) => new Promise((resolve, reject) => __awaiter(void 0, void 0, void 0, function* () {
+    setImmediate(() => __awaiter(void 0, void 0, void 0, function* () {
+        try {
+            const value = yield callback();
+            resolve(value);
+        }
+        catch (error) {
+            reject(error);
+        }
+    }));
+}));
+class RabbitMq {
+    constructor(options, redisConfig) {
+        var _a;
+        this.PUBLISH_TIMEOUT = Number(process.env.RABBITMQ_PUBLISH_TIMEOUT) || 60000;
+        this.PUBLISH_ERROR_MSG = `rabbit: publish timeout(${this.PUBLISH_TIMEOUT}ms) has pass, exchange: `;
+        this.DISCONNECT_MSG = 'rabbit: connection disconnect';
+        this.RESCONNECT_MSG = 'rabbit: reconnecting';
+        this.shouldConsumeMessageByTimestamp = (msg) => __awaiter(this, void 0, void 0, function* () {
+            if (msg) {
+                const { properties: { headers } } = msg;
+                const timestamp = headers === null || headers === void 0 ? void 0 : headers.creationTimestamp;
+                if (timestamp && (headers === null || headers === void 0 ? void 0 : headers.redisTimestampValidationKey) && this.redisClient) {
+                    const lastMessageTimestamp = yield this.redisClient.getAsync(headers.redisTimestampValidationKey);
+                    return !lastMessageTimestamp || (parseInt(lastMessageTimestamp, 10) <= parseInt(timestamp, 10));
+                }
+                return true;
+            }
+            return false;
+        });
+        this.ack = (channel, msg, shouldUpdateRedisTimestamp = false, releaseLock = null) => (userMsg) => __awaiter(this, void 0, void 0, function* () {
+            if (msg) {
+                yield channel.ack(msg);
+                const { properties: { headers } } = msg;
+                const timestamp = headers === null || headers === void 0 ? void 0 : headers.creationTimestamp;
+                if (shouldUpdateRedisTimestamp && timestamp && (headers === null || headers === void 0 ? void 0 : headers.redisTimestampValidationKey) && this.redisClient) {
+                    const parsedTimestamp = parseInt(timestamp, 10);
+                    yield this.redisClient.setAsync(headers.redisTimestampValidationKey, parsedTimestamp, 'EX', 3600);
+                    yield this.unlockRedisIfNeeded(releaseLock);
+                }
+            }
+        });
+        this.nack = (channel, queue, options, deadQueueOptions, msg, releaseLock) => (userMsg, { skipRetry = false, } = {}) => __awaiter(this, void 0, void 0, function* () {
+            yield this.unlockRedisIfNeeded(releaseLock);
+            if (channel && msg) {
+                if (!skipRetry
+                    && (!msg.properties.headers[RETRY_HEADER]
+                        || parseInt(msg.properties.headers[RETRY_HEADER], 10) < options.retries)) {
+                    yield this.sendToQueue(queue, RabbitMq.parseMsg(msg).content, options, Object.assign(Object.assign({}, msg.properties.headers), { [RETRY_HEADER]: msg.properties.headers[RETRY_HEADER]
+                            ? msg.properties.headers[RETRY_HEADER] + 1
+                            : 1 }));
+                }
+                else {
+                    const deadQueue = `${queue}-dead`;
+                    yield this.sendToQueue(deadQueue, RabbitMq.parseMsg(msg).content, deadQueueOptions, Object.assign(Object.assign({}, msg.properties.headers), { [RETRY_HEADER]: msg.properties.headers[RETRY_HEADER]
+                            ? msg.properties.headers[RETRY_HEADER] + 1
+                            : 1 }));
+                }
+                yield channel.ack(msg);
+            }
+        });
+        this.em = new events_1.EventEmitter();
+        this.channel = null;
+        this.connection = null;
+        this.creatingConnection = false;
+        this.exchanges = {};
+        this.queues = {};
+        this.options = options;
+        this.host = (_a = options === null || options === void 0 ? void 0 : options.host) !== null && _a !== void 0 ? _a : null;
+        this.redisClient = redisConfig && redis_1.default(redisConfig);
+        if (this.redisClient) {
+            this.redisLock = util_1.promisify(redis_lock_1.default(this.redisClient));
+        }
+    }
+    static parseMsg(msg) {
+        let { content } = msg;
+        content = content.toString();
+        try {
+            content = JSON.parse(content);
+        }
+        catch (e) { }
+        return Object.assign(Object.assign({}, msg), { content });
+    }
+    static validateName(type, name) {
+        if (!name || name === '') {
+            throw new rabbitError_1.default(`error while using ${type} with no name`);
+        }
+    }
+    static getPublishOptions(customHeaders = {}) {
+        return {
+            timestamp: moment_1.default().unix(),
+            headers: Object.assign({ creationTimestamp: moment_1.default().valueOf() }, customHeaders),
+        };
+    }
+    getConnection() {
+        return __awaiter(this, void 0, void 0, function* () {
+            return new Promise((resolve) => __awaiter(this, void 0, void 0, function* () {
+                if (this.connection !== null) {
+                    return resolve(this.connection);
+                }
+                if (this.creatingConnection) {
+                    this.em.once(connectionCreatedConst, resolve);
+                    return;
+                }
+                this.creatingConnection = true;
+                logger_1.default.info('rabbit: env username', { userName: (process.env.RABBITMQ_USERNAME || 'doesnt exist') });
+                const username = process.env.RABBITMQ_USERNAME || 'guest';
+                const password = process.env.RABBITMQ_PASSWORD || 'guest';
+                const host = this.host || process.env.RABBITMQ_SERVICE_HOST || '';
+                const connection = yield amqp_connection_manager_1.connect([`amqp://${username}:${password}@${host}`], { reconnectTimeInSeconds: 0.5 });
+                this.connection = connection;
+                this.creatingConnection = false;
+                this.em.emit(connectionCreatedConst, connection);
+                resolve(connection);
+            }));
+        });
+    }
+    getNewChannel() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const connection = yield this.getConnection();
+            return connection.createChannel({});
+        });
+    }
+    assertChannel() {
+        return __awaiter(this, void 0, void 0, function* () {
+            return new Promise((resolve, reject) => __awaiter(this, void 0, void 0, function* () {
+                if (this.channel) {
+                    return resolve(this.channel);
+                }
+                try {
+                    const connection = yield this.getConnection();
+                    if (this.channel === null) {
+                        this.channel = yield connection.createChannel({});
+                    }
+                    resolve(this.channel);
+                }
+                catch (e) {
+                    reject(e);
+                }
+            }));
+        });
+    }
+    assertExchange(exchangeName, options) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const channel = yield this.assertChannel();
+            if (this.exchanges[exchangeName]) {
+                return this.exchanges[exchangeName];
+            }
+            const exchange = yield assertExchangeFanout(channel, exchangeName);
+            this.exchanges[exchangeName] = exchange;
+            return exchange;
+        });
+    }
+    getQueueLength(queue) {
+        return __awaiter(this, void 0, void 0, function* () {
+            RabbitMq.validateName('queue', queue);
+            const channel = yield this.assertChannel();
+            // @ts-ignore
+            return channel.checkQueue(queue);
+        });
+    }
+    bindQueue(queue, exchange) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const channel = yield this.assertChannel();
+            yield channel.addSetup((setupChannel) => setupChannel.bindQueue(queue, exchange, ''));
+            // @ts-ignore
+            return channel.bindQueue(queue, exchange, '');
+        });
+    }
+    assertQueue(queueName, options) {
+        return __awaiter(this, void 0, void 0, function* () {
+            RabbitMq.validateName('queue', queueName);
+            const channel = yield this.assertChannel();
+            if (this.queues[queueName]) {
+                return this.queues[queueName];
+            }
+            yield channel.addSetup((setupChannel) => setupChannel.assertQueue(queueName, options));
+            const queue = yield channel.assertQueue(queueName, options);
+            this.queues[queueName] = queueName;
+            return queue;
+        });
+    }
+    consume(queue, callback, options) {
+        return __awaiter(this, void 0, void 0, function* () {
+            return this.consumeFromRabbit(queue, callback, options);
+        });
+    }
+    lockRedisIfNeeded(msg, options) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const { properties: { headers } } = msg;
+            const timestamp = headers === null || headers === void 0 ? void 0 : headers.creationTimestamp;
+            let releaseLock = null;
+            if (options.useConsumeWithLock && timestamp && (headers === null || headers === void 0 ? void 0 : headers.redisTimestampValidationKey) && this.redisLock) {
+                releaseLock = yield this.redisLock(headers.redisTimestampValidationKey, (options === null || options === void 0 ? void 0 : options.lockTimeout) || DEFAULT_LOCK_TIMEOUT);
+            }
+            return releaseLock;
+        });
+    }
+    unlockRedisIfNeeded(releaseLock) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (this.redisLock && releaseLock) {
+                yield releaseLock();
+            }
+        });
+    }
+    consumeFromRabbit(queue, callback, options) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const optionsWithDefaults = Object.assign(Object.assign({}, defaultOptions), options);
+            RabbitMq.validateName('queue', queue);
+            const { limit, deadMessageTtl, useConsumeWithLock, lockTimeout, } = optionsWithDefaults;
+            if (useConsumeWithLock) {
+                if (!this.redisLock) {
+                    throw new Error('Usage of consumeWithLock requires RedisInstance');
+                }
+                logger_1.default.info(`Consuming with lock from queue ${queue} with lockTimeout: ${lockTimeout}ms`);
+            }
+            const channel = yield this.getNewChannel();
+            return channel.addSetup((c) => __awaiter(this, void 0, void 0, function* () {
+                yield c.assertQueue(queue);
+                yield c.prefetch(limit, true);
+                return Promise.all([
+                    c.consume(queue, (msg) => __awaiter(this, void 0, void 0, function* () {
+                        if (!msg) {
+                            return null;
+                        }
+                        const parsedMessage = RabbitMq.parseMsg(msg);
+                        const releaseLock = yield this.lockRedisIfNeeded(parsedMessage, optionsWithDefaults);
+                        const shouldConsume = yield this.shouldConsumeMessageByTimestamp(parsedMessage);
+                        if (!shouldConsume) {
+                            yield this.unlockRedisIfNeeded(releaseLock);
+                            return this.ack(channel, msg)(msg);
+                        }
+                        try {
+                            yield callback(parsedMessage, this.ack(channel, msg, true, releaseLock), this.nack(channel, queue, optionsWithDefaults, { messageTtl: deadMessageTtl }, msg, releaseLock));
+                        }
+                        catch (e) {
+                            yield this.nack(channel, queue, optionsWithDefaults, { messageTtl: deadMessageTtl }, msg, releaseLock)(msg);
+                        }
+                    })),
+                ]);
+            }));
+        });
+    }
+    consumeFromExchange(queue, exchange, callback, options) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const optionsWithDefaults = Object.assign(Object.assign({}, defaultOptions), options);
+            RabbitMq.validateName('exchange', exchange);
+            RabbitMq.validateName('queue', queue);
+            const { limit, deadMessageTtl } = optionsWithDefaults;
+            const channel = yield this.getNewChannel();
+            return channel.addSetup((c) => __awaiter(this, void 0, void 0, function* () {
+                yield assertExchangeFanout(c, exchange);
+                yield c.assertQueue(queue);
+                yield c.prefetch(limit, true);
+                return Promise.all([
+                    c.bindQueue(queue, exchange, ''),
+                    this.consume(queue, callback, options),
+                ]);
+            }));
+        });
+    }
+    publish(exchange, content, customHeaders) {
+        return __awaiter(this, void 0, void 0, function* () {
+            return wrapSetImmediate(() => __awaiter(this, void 0, void 0, function* () {
+                RabbitMq.validateName('exchange', exchange);
+                const channel = yield this.assertChannel();
+                yield this.assertExchange(exchange);
+                yield channel.publish(exchange, '', Buffer.from(JSON.stringify(content)), RabbitMq.getPublishOptions(customHeaders));
+            }));
+        });
+    }
+    sendToQueue(queue, content, options, customHeaders) {
+        return __awaiter(this, void 0, void 0, function* () {
+            return wrapSetImmediate(() => __awaiter(this, void 0, void 0, function* () {
+                RabbitMq.validateName('queue', queue);
+                const channel = yield this.assertChannel();
+                yield this.assertQueue(queue, options);
+                return channel.sendToQueue(queue, Buffer.from(JSON.stringify(content)), RabbitMq.getPublishOptions(customHeaders));
+            }));
+        });
+    }
+    isConnected() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const connection = yield this.getConnection();
+            return connection.isConnected();
+        });
+    }
+}
+exports.default = RabbitMq;

package/dist/index.js

@@ -34,7 +34,7 @@ const defaultOptions = {
     lockTimeout: DEFAULT_LOCK_TIMEOUT,
     useConsumeWithLock: DEFAULT_USE_CONSUME_WITH_LOCK,
 };
-const assertExchangeFanout = (c, exchangeName) => c.assertExchange(exchangeName, 'fanout');
+const assertExchangeFanout = (c, exchangeName) => __awaiter(void 0, void 0, void 0, function* () { return c.assertExchange(exchangeName, 'fanout'); });
 const connectionCreatedConst = 'connectionCreated';
 const wrapSetImmediate = (callback) => new Promise((resolve, reject) => __awaiter(void 0, void 0, void 0, function* () {
     setImmediate(() => __awaiter(void 0, void 0, void 0, function* () {
@@ -178,7 +178,7 @@ class RabbitMq {
                 try {
                     const connection = yield this.getConnection();
                     if (this.channel === null) {
-                        this.channel = connection.createChannel({});
+                        this.channel = yield connection.createChannel({});
                     }
                     resolve(this.channel);
                 }
@@ -204,12 +204,13 @@ class RabbitMq {
             RabbitMq.validateName('queue', queue);
             const channel = yield this.assertChannel();
             // @ts-ignore
-            return channel.checkQueue(queue);
+            return channel.assertQueue(queue);
         });
     }
     bindQueue(queue, exchange) {
         return __awaiter(this, void 0, void 0, function* () {
             const channel = yield this.assertChannel();
+            yield channel.addSetup((setupChannel) => setupChannel.bindQueue(queue, exchange, ''));
             // @ts-ignore
             return channel.bindQueue(queue, exchange, '');
         });
@@ -221,6 +222,7 @@ class RabbitMq {
             if (this.queues[queueName]) {
                 return this.queues[queueName];
             }
+            yield channel.addSetup((setupChannel) => setupChannel.assertQueue(queueName, options));
             const queue = yield channel.assertQueue(queueName, options);
             this.queues[queueName] = queueName;
             return queue;
@@ -262,6 +264,7 @@ class RabbitMq {
             }
             const channel = yield this.getNewChannel();
             return channel.addSetup((c) => __awaiter(this, void 0, void 0, function* () {
+                console.log('addSetup');
                 yield c.assertQueue(queue);
                 yield c.prefetch(limit, true);
                 return Promise.all([

package/dist/memory-test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/memory-test.js

@@ -0,0 +1,54 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+/* eslint-disable no-await-in-loop */
+// eslint-disable-next-line import/no-extraneous-dependencies
+const node_tcp_proxy_1 = __importDefault(require("node-tcp-proxy"));
+const index_new_1 = __importDefault(require("./index-new"));
+const delay = (ms = 1000) => new Promise((resolve) => setTimeout(() => resolve(), ms));
+const callback = (eq, timeout = 0) => (msg, ack, nack) => __awaiter(void 0, void 0, void 0, function* () {
+    setTimeout(() => __awaiter(void 0, void 0, void 0, function* () {
+        // console.log(msg)
+        yield ack(msg);
+    }), timeout);
+    return null;
+});
+const payload = { ttt: 123 };
+const testId = 'www';
+const main = () => __awaiter(void 0, void 0, void 0, function* () {
+    const port = 6672;
+    const createProxy = () => node_tcp_proxy_1.default.createProxy(port, '0.0.0.0', 5672);
+    const restartProxy = (currentProxy, sleep) => __awaiter(void 0, void 0, void 0, function* () {
+        currentProxy.end();
+        yield delay(sleep);
+        return node_tcp_proxy_1.default.createProxy(port, '0.0.0.0', 5672);
+    });
+    let currentProxy = createProxy();
+    const rabbit = new index_new_1.default({ host: `127.0.0.1:${port}` });
+    const queue = `test-timestamp-validation-${testId}`;
+    const mockFn = callback(payload);
+    yield rabbit.consume(queue, mockFn);
+    console.error('start');
+    const usedB = process.memoryUsage().heapUsed / 1024 / 1024;
+    console.log('Used before', usedB);
+    for (let i = 0; i < 10000; i += 1) {
+        // currentProxy = await restartProxy(currentProxy, 100);
+        yield rabbit.sendToQueue(queue, payload);
+        yield delay(10);
+        // expect(mockFn).toBeCalledTimes(i);
+        const used = process.memoryUsage().heapUsed / 1024 / 1024;
+        console.error('Used', i, used);
+    }
+});
+main();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@autofleet/rabbit",
-  "version": "2.3.4",
+  "version": "2.3.6",
   "description": "",
   "main": "dist/index.js",
   "scripts": {

package/src/index-new.ts

@@ -0,0 +1,427 @@
+/* eslint-disable @typescript-eslint/ban-ts-comment,@typescript-eslint/ban-types,@typescript-eslint/no-unused-vars,consistent-return,no-async-promise-executor,no-param-reassign,no-empty */
+// eslint-disable-next-line max-classes-per-file
+import moment from 'moment';
+import RedisLock from 'redis-lock';
+import { ConfirmChannel, Options, ConsumeMessage } from 'amqplib';
+import { AmqpConnectionManager, ChannelWrapper, connect } from 'amqp-connection-manager';
+import { EventEmitter } from 'events';
+// import { newTrace, traceTypes } from '@autofleet/outbreak';
+import { promisify } from 'util';
+import logger from './logger';
+import RabbitError from './rabbitError';
+import getRedisInstance, { RedisConfig } from './redis';
+
+export interface IAfRabbitMq {
+  ack: any;
+  nack: any;
+  assertChannel: any;
+  assertExchange: any;
+  assertQueue: any;
+  consume: any;
+  consumeFromExchange: any;
+  publish: any;
+  sendToQueue: any;
+  redisClient?: any;
+}
+
+interface ExchangesCache {
+  [key: string]: any
+}
+interface QueuesCache {
+  [key: string]: any
+}
+
+type CustomMessageHeaders = {
+  redisTimestampValidationKey?: string;
+}
+
+type RedisLockType = (args0?: string, arg1?: number) => Promise<any>
+
+type ConsumeMessageOrNull = ConsumeMessage | null;
+interface ConsumeOptions {
+  retries?: number;
+  deadMessageTtl?: number;
+  limit?: number;
+  lockTimeout?: number;
+  useConsumeWithLock?: boolean;
+}
+
+type CallbackFunction = (msg: ConsumeMessage, ack: Function, nack: Function) => Promise<any>
+
+export interface AfRabbitOptions {
+  disableReconnect?: boolean;
+  host?: string;
+}
+
+const DEFAULT_DEAD_TTL_TWO_DAYS = 60000 * 60 * 48;
+const DEFAULT_LOCK_TIMEOUT = 1000 * 5;
+const RETRY_HEADER = 'x-retry-count';
+const DEFAULT_USE_CONSUME_WITH_LOCK = false;
+
+const defaultOptions = {
+  limit: 1,
+  retries: 1,
+  deadMessageTtl: DEFAULT_DEAD_TTL_TWO_DAYS,
+  lockTimeout: DEFAULT_LOCK_TIMEOUT,
+  useConsumeWithLock: DEFAULT_USE_CONSUME_WITH_LOCK,
+};
+
+const assertExchangeFanout = async (c: ConfirmChannel | ChannelWrapper, exchangeName: string) => c.assertExchange(exchangeName, 'fanout');
+
+const connectionCreatedConst = 'connectionCreated';
+
+const wrapSetImmediate = (callback: () => any) => new Promise<any>(async (resolve, reject) => {
+  setImmediate(async () => {
+    try {
+      const value = await callback();
+      resolve(value);
+    } catch (error) {
+      reject(error);
+    }
+  });
+});
+
+class RabbitMq implements IAfRabbitMq {
+  static parseMsg(msg: any) : any {
+    let { content } = msg;
+    content = content.toString();
+
+    try {
+      content = JSON.parse(content);
+    } catch (e) { }
+
+    return {
+      ...msg,
+      content,
+    };
+  }
+
+  static validateName(type: string, name: string) {
+    if (!name || name === '') {
+      throw new RabbitError(`error while using ${type} with no name`);
+    }
+  }
+
+  static getPublishOptions(customHeaders: CustomMessageHeaders = {}) {
+    return {
+      timestamp: moment().unix(),
+      headers: {
+        creationTimestamp: moment().valueOf(),
+        ...customHeaders,
+      },
+    };
+  }
+
+  PUBLISH_TIMEOUT = Number(process.env.RABBITMQ_PUBLISH_TIMEOUT) || 60000;
+
+  PUBLISH_ERROR_MSG = `rabbit: publish timeout(${this.PUBLISH_TIMEOUT}ms) has pass, exchange: `;
+
+  DISCONNECT_MSG = 'rabbit: connection disconnect';
+
+  RESCONNECT_MSG = 'rabbit: reconnecting';
+
+  channel: ChannelWrapper | null;
+
+  connection: AmqpConnectionManager | null;
+
+  em: EventEmitter;
+
+  creatingConnection: boolean;
+
+  exchanges: ExchangesCache;
+
+  queues: QueuesCache;
+
+  host: string | null;
+
+  options: AfRabbitOptions | undefined;
+
+  redisClient: any;
+
+  redisLock?: RedisLockType;
+
+  constructor(options?: AfRabbitOptions, redisConfig?: RedisConfig) {
+    this.em = new EventEmitter();
+    this.channel = null;
+    this.connection = null;
+    this.creatingConnection = false;
+    this.exchanges = {};
+    this.queues = {};
+    this.options = options;
+    this.host = options?.host ?? null;
+    this.redisClient = redisConfig && getRedisInstance(redisConfig);
+    if (this.redisClient) {
+      this.redisLock = promisify(RedisLock(this.redisClient)) as RedisLockType;
+    }
+  }
+
+  private shouldConsumeMessageByTimestamp = async (msg: ConsumeMessageOrNull) => {
+    if (msg) {
+      const { properties: { headers } } = msg;
+      const timestamp = headers?.creationTimestamp;
+
+      if (timestamp && headers?.redisTimestampValidationKey && this.redisClient) {
+        const lastMessageTimestamp = await this.redisClient.getAsync(headers.redisTimestampValidationKey);
+        return !lastMessageTimestamp || (parseInt(lastMessageTimestamp, 10) <= parseInt(timestamp, 10));
+      }
+      return true;
+    }
+    return false;
+  }
+
+  public ack = (channel: ChannelWrapper, msg: ConsumeMessageOrNull, shouldUpdateRedisTimestamp = false, releaseLock = null) => async (userMsg: ConsumeMessage) : Promise<any> => {
+    if (msg) {
+      await channel.ack(msg);
+      const { properties: { headers } } = msg;
+      const timestamp = headers?.creationTimestamp;
+
+      if (shouldUpdateRedisTimestamp && timestamp && headers?.redisTimestampValidationKey && this.redisClient) {
+        const parsedTimestamp = parseInt(timestamp, 10);
+        await this.redisClient.setAsync(headers.redisTimestampValidationKey, parsedTimestamp, 'EX', 3600);
+        await this.unlockRedisIfNeeded(releaseLock);
+      }
+    }
+  }
+
+  public nack = (
+    channel: ChannelWrapper,
+    queue: string,
+    options: any,
+    deadQueueOptions: Options.AssertQueue,
+    msg: ConsumeMessageOrNull,
+    releaseLock: any,
+  ) => async (
+    userMsg: ConsumeMessageOrNull,
+    {
+      skipRetry = false,
+    }: any = { },
+  ) : Promise<any> => {
+    await this.unlockRedisIfNeeded(releaseLock);
+    if (channel && msg) {
+      if (
+        !skipRetry
+        && (
+          !msg.properties.headers[RETRY_HEADER]
+          || parseInt(msg.properties.headers[RETRY_HEADER], 10) < options.retries
+        )
+      ) {
+        await this.sendToQueue(queue, RabbitMq.parseMsg(msg).content, options, {
+          ...msg.properties.headers,
+          [RETRY_HEADER]: msg.properties.headers[RETRY_HEADER]
+            ? msg.properties.headers[RETRY_HEADER] + 1
+            : 1,
+        });
+      } else {
+        const deadQueue = `${queue}-dead`;
+        await this.sendToQueue(deadQueue, RabbitMq.parseMsg(msg).content, deadQueueOptions, {
+          ...msg.properties.headers,
+          [RETRY_HEADER]: msg.properties.headers[RETRY_HEADER]
+            ? msg.properties.headers[RETRY_HEADER] + 1
+            : 1,
+        });
+      }
+      await channel.ack(msg);
+    }
+  }
+
+  async getConnection() {
+    return new Promise<AmqpConnectionManager>(async (resolve) => {
+      if (this.connection !== null) {
+        return resolve(this.connection);
+      }
+      if (this.creatingConnection) {
+        this.em.once(connectionCreatedConst, resolve);
+        return;
+      }
+      this.creatingConnection = true;
+      logger.info('rabbit: env username', { userName: (process.env.RABBITMQ_USERNAME || 'doesnt exist') });
+      const username: string = process.env.RABBITMQ_USERNAME || 'guest';
+      const password: string = process.env.RABBITMQ_PASSWORD || 'guest';
+      const host: string = this.host || process.env.RABBITMQ_SERVICE_HOST || '';
+      const connection: AmqpConnectionManager = await connect([`amqp://${username}:${password}@${host}`], { reconnectTimeInSeconds: 0.5 });
+      this.connection = connection;
+      this.creatingConnection = false;
+      this.em.emit(connectionCreatedConst, connection);
+      resolve(connection);
+    });
+  }
+
+  async getNewChannel() {
+    const connection: AmqpConnectionManager = await this.getConnection();
+    return connection.createChannel({});
+  }
+
+  async assertChannel() {
+    return new Promise<ChannelWrapper>(async (resolve, reject) => {
+      if (this.channel) {
+        return resolve(this.channel);
+      }
+
+      try {
+        const connection: AmqpConnectionManager = await this.getConnection();
+        if (this.channel === null) {
+          this.channel = await connection.createChannel({});
+        }
+        resolve(this.channel);
+      } catch (e) {
+        reject(e);
+      }
+    });
+  }
+
+  async assertExchange(exchangeName: string, options?: any) {
+    const channel: ChannelWrapper = await this.assertChannel();
+    if (this.exchanges[exchangeName]) {
+      return this.exchanges[exchangeName];
+    }
+    const exchange = await assertExchangeFanout(channel, exchangeName);
+    this.exchanges[exchangeName] = exchange;
+    return exchange;
+  }
+
+  async getQueueLength(queue: string) {
+    RabbitMq.validateName('queue', queue);
+    const channel: ChannelWrapper = await this.assertChannel();
+    // @ts-ignore
+    return channel.checkQueue(queue);
+  }
+
+  async bindQueue(queue: string, exchange: string) {
+    const channel: ChannelWrapper = await this.assertChannel();
+    await channel.addSetup((setupChannel: ConfirmChannel) => setupChannel.bindQueue(queue, exchange, ''));
+    // @ts-ignore
+    return channel.bindQueue(queue, exchange, '');
+  }
+
+  async assertQueue(queueName: string, options?: Options.AssertQueue) {
+    RabbitMq.validateName('queue', queueName);
+    const channel: ChannelWrapper = await this.assertChannel();
+    if (this.queues[queueName]) {
+      return this.queues[queueName];
+    }
+    await channel.addSetup((setupChannel: ConfirmChannel) => setupChannel.assertQueue(queueName, options));
+    const queue = await channel.assertQueue(queueName, options);
+    this.queues[queueName] = queueName;
+    return queue;
+  }
+
+  async consume(queue: string, callback: CallbackFunction, options?: ConsumeOptions): Promise<any> {
+    return this.consumeFromRabbit(queue, callback, options);
+  }
+
+  private async lockRedisIfNeeded(msg: any, options: any) {
+    const { properties: { headers } } = msg;
+    const timestamp = headers?.creationTimestamp;
+    let releaseLock = null;
+
+    if (options.useConsumeWithLock && timestamp && headers?.redisTimestampValidationKey && this.redisLock) {
+      releaseLock = await this.redisLock(
+        headers.redisTimestampValidationKey,
+        options?.lockTimeout || DEFAULT_LOCK_TIMEOUT,
+      );
+    }
+    return releaseLock;
+  }
+
+  private async unlockRedisIfNeeded(releaseLock: any) {
+    if (this.redisLock && releaseLock) {
+      await releaseLock();
+    }
+  }
+
+  private async consumeFromRabbit(queue: string, callback: CallbackFunction, options?: ConsumeOptions): Promise<any> {
+    const optionsWithDefaults = { ...defaultOptions, ...options };
+    RabbitMq.validateName('queue', queue);
+    const {
+      limit, deadMessageTtl, useConsumeWithLock, lockTimeout,
+    } = optionsWithDefaults;
+    if (useConsumeWithLock) {
+      if (!this.redisLock) {
+        throw new Error('Usage of consumeWithLock requires RedisInstance');
+      }
+      logger.info(`Consuming with lock from queue ${queue} with lockTimeout: ${lockTimeout}ms`);
+    }
+    const channel: ChannelWrapper = await this.getNewChannel();
+    return channel.addSetup(async (c: ConfirmChannel) => {
+      await c.assertQueue(queue);
+      await c.prefetch(limit, true);
+      return Promise.all([
+        c.consume(
+          queue,
+          async (msg: ConsumeMessageOrNull) => {
+            if (!msg) {
+              return null;
+            }
+            const parsedMessage = RabbitMq.parseMsg(msg);
+            const releaseLock = await this.lockRedisIfNeeded(parsedMessage, optionsWithDefaults);
+            const shouldConsume = await this.shouldConsumeMessageByTimestamp(parsedMessage);
+            if (!shouldConsume) {
+              await this.unlockRedisIfNeeded(releaseLock);
+              return this.ack(channel, msg)(msg);
+            }
+            try {
+              await callback(
+                parsedMessage,
+                this.ack(channel, msg, true, releaseLock),
+                this.nack(channel, queue, optionsWithDefaults, { messageTtl: deadMessageTtl }, msg, releaseLock),
+              );
+            } catch (e) {
+              await this.nack(channel, queue, optionsWithDefaults, { messageTtl: deadMessageTtl }, msg, releaseLock)(msg);
+            }
+          },
+        ),
+      ]);
+    });
+  }
+
+  async consumeFromExchange(queue: string, exchange: string, callback: CallbackFunction, options?: ConsumeOptions) : Promise<any> {
+    const optionsWithDefaults = { ...defaultOptions, ...options };
+    RabbitMq.validateName('exchange', exchange);
+    RabbitMq.validateName('queue', queue);
+    const { limit, deadMessageTtl } = optionsWithDefaults;
+    const channel: ChannelWrapper = await this.getNewChannel();
+
+    return channel.addSetup(async (c: ConfirmChannel) => {
+      await assertExchangeFanout(c, exchange);
+      await c.assertQueue(queue);
+      await c.prefetch(limit, true);
+      return Promise.all([
+        c.bindQueue(queue, exchange, ''),
+        this.consume(
+          queue,
+          callback,
+          options,
+        ),
+      ]);
+    });
+  }
+
+  async publish(exchange: string, content: any, customHeaders?: any) : Promise<boolean> {
+    return wrapSetImmediate(async () => {
+      RabbitMq.validateName('exchange', exchange);
+      const channel: ChannelWrapper = await this.assertChannel();
+      await this.assertExchange(exchange);
+      await channel.publish(exchange, '',
+        Buffer.from(JSON.stringify(content)),
+        RabbitMq.getPublishOptions(customHeaders));
+    });
+  }
+
+  async sendToQueue(queue: string, content: any, options?: any, customHeaders?: any) : Promise<boolean> {
+    return wrapSetImmediate(async () => {
+      RabbitMq.validateName('queue', queue);
+      const channel: ChannelWrapper = await this.assertChannel();
+      await this.assertQueue(queue, options);
+      return channel.sendToQueue(queue,
+        Buffer.from(JSON.stringify(content)),
+        RabbitMq.getPublishOptions(customHeaders));
+    });
+  }
+
+  async isConnected() : Promise<boolean> {
+    const connection = await this.getConnection();
+    return connection.isConnected();
+  }
+}
+
+export default RabbitMq;

package/src/index.ts

@@ -65,7 +65,7 @@ const defaultOptions = {
   useConsumeWithLock: DEFAULT_USE_CONSUME_WITH_LOCK,
 };
 
-const assertExchangeFanout = (c: any, exchangeName: string) => c.assertExchange(exchangeName, 'fanout');
+const assertExchangeFanout = async (c: any, exchangeName: string) => c.assertExchange(exchangeName, 'fanout');
 
 const connectionCreatedConst = 'connectionCreated';
 
@@ -267,7 +267,7 @@ class RabbitMq implements IAfRabbitMq {
       try {
         const connection: AmqpConnectionManager = await this.getConnection();
         if (this.channel === null) {
-          this.channel = connection.createChannel({});
+          this.channel = await connection.createChannel({});
         }
         resolve(this.channel);
       } catch (e) {
@@ -295,6 +295,7 @@ class RabbitMq implements IAfRabbitMq {
 
   async bindQueue(queue: string, exchange: string) {
     const channel: ChannelWrapper = await this.assertChannel();
+    await channel.addSetup((setupChannel: ConfirmChannel) => setupChannel.bindQueue(queue, exchange, ''));
     // @ts-ignore
     return channel.bindQueue(queue, exchange, '');
   }
@@ -305,6 +306,7 @@ class RabbitMq implements IAfRabbitMq {
     if (this.queues[queueName]) {
       return this.queues[queueName];
     }
+    await channel.addSetup((setupChannel: ConfirmChannel) => setupChannel.assertQueue(queueName, options));
     const queue = await channel.assertQueue(queueName, options);
     this.queues[queueName] = queueName;
     return queue;

package/src/memory-test.ts

@@ -0,0 +1,44 @@
+/* eslint-disable no-await-in-loop */
+// eslint-disable-next-line import/no-extraneous-dependencies
+import proxy from 'node-tcp-proxy';
+import RabbitMq from './index-new';
+
+const delay = (ms = 1000) => new Promise((resolve) => setTimeout(() => resolve(), ms));
+const callback = (eq: any, timeout = 0) => async (msg: any, ack: any, nack: any) => {
+  setTimeout(async () => {
+    // console.log(msg)
+    await ack(msg);
+  }, timeout);
+  return null;
+};
+
+const payload = { ttt: 123 };
+const testId = 'www';
+
+const main = async () => {
+  const port = 6672;
+  const createProxy = () => proxy.createProxy(port, '0.0.0.0', 5672);
+  const restartProxy = async (currentProxy, sleep) => {
+    currentProxy.end();
+    await delay(sleep);
+    return proxy.createProxy(port, '0.0.0.0', 5672);
+  };
+  let currentProxy = createProxy();
+  const rabbit = new RabbitMq({ host: `127.0.0.1:${port}` });
+  const queue = `test-timestamp-validation-${testId}`;
+  const mockFn = callback(payload);
+  await rabbit.consume(queue, mockFn);
+  console.error('start');
+  const usedB = process.memoryUsage().heapUsed / 1024 / 1024;
+  console.log('Used before', usedB);
+  for (let i = 0; i < 10000; i += 1) {
+    // currentProxy = await restartProxy(currentProxy, 100);
+    await rabbit.sendToQueue(queue, payload);
+    await delay(10);
+    // expect(mockFn).toBeCalledTimes(i);
+    const used = process.memoryUsage().heapUsed / 1024 / 1024;
+    console.error('Used', i, used);
+  }
+};
+
+main();

package/tsconfig.json

@@ -1,6 +1,6 @@
 {
   "compilerOptions": {
-    "target": "es6",
+    "target": "es2020",
     "module": "commonjs",
     "declaration": true,
     "outDir": "./dist",