@autofleet/rabbit 2.1.16-beta → 2.2.0-beta-1

package/dist/index.d.ts

@@ -1,5 +1,5 @@
 /// <reference types="node" />
-import { Options } from 'amqplib';
+import { Options, ConsumeMessage } from 'amqplib';
 import { AmqpConnectionManager, ChannelWrapper } from 'amqp-connection-manager';
 import { EventEmitter } from 'events';
 import { RedisConfig } from './redis';
@@ -24,6 +24,15 @@ interface QueuesCache {
 declare type CustomMessageHeaders = {
     redisTimestampValidationKey?: string;
 };
+declare type RedisLockType = (args0?: string, arg1?: number) => Promise<any>;
+interface ConsumeOptions {
+    retries?: number;
+    deadMessageTtl?: number;
+    limit?: number;
+    lockTimeout?: number;
+    useConsumeWithLock?: boolean;
+}
+declare type CallbackFunction = (msg: ConsumeMessage, ack: Function, nack: Function) => Promise<any>;
 export interface AfRabbitOptions {
     reconnect: boolean;
 }
@@ -46,9 +55,10 @@ declare class RabbitMq implements IAfRabbitMq {
     queues: QueuesCache;
     options: AfRabbitOptions | undefined;
     redisClient: any;
+    redisLock?: RedisLockType;
     constructor(options?: AfRabbitOptions, redisConfig?: RedisConfig);
     private shouldConsumeMessageByTimestamp;
-    ack: (channel: ChannelWrapper, shouldUpdateRedisTimestamp?: boolean) => (msg: any) => Promise<void>;
+    ack: (channel: ChannelWrapper, shouldUpdateRedisTimestamp?: boolean) => (msg: ConsumeMessage) => Promise<any>;
     nack: (channel: ChannelWrapper, queue: string, options: any, deadQueueOptions: Options.AssertQueue) => (msg: any, { skipRetry, }?: any) => Promise<void>;
     getConnection(): Promise<AmqpConnectionManager>;
     getNewChannel(): Promise<ChannelWrapper>;
@@ -57,9 +67,11 @@ declare class RabbitMq implements IAfRabbitMq {
     getQueueLength(queue: string): Promise<import("amqplib").Replies.AssertQueue>;
     bindQueue(queue: string, exchange: string): Promise<void>;
     assertQueue(queueName: string, options?: Options.AssertQueue): Promise<any>;
-    consume(queue: string, callback: (msg: any, ack: Function, nack: Function) => any, options?: any): Promise<void>;
-    consumeFromExchange(queue: string, exchange: string, callback: (msg: any, ack: Function, nack: Function) => any, options?: any): Promise<void>;
-    publish(exchange: string, content: any, customHeaders?: any): Promise<void>;
+    consume(queue: string, callback: CallbackFunction, options?: ConsumeOptions): Promise<any>;
+    private consumeWithLock;
+    private consumeFromRabbit;
+    consumeFromExchange(queue: string, exchange: string, callback: CallbackFunction, options?: ConsumeOptions): Promise<any>;
+    publish(exchange: string, content: any, customHeaders?: any): Promise<any>;
     sendToQueue(queue: string, content: any, options?: any, customHeaders?: any): Promise<boolean>;
     isConnected(): Promise<boolean>;
 }

package/dist/index.js

@@ -15,9 +15,11 @@ Object.defineProperty(exports, "__esModule", { value: true });
 /* eslint-disable @typescript-eslint/ban-ts-comment,@typescript-eslint/ban-types,@typescript-eslint/no-unused-vars,consistent-return,no-async-promise-executor,no-param-reassign,no-empty */
 // eslint-disable-next-line max-classes-per-file
 const moment_1 = __importDefault(require("moment"));
+const redis_lock_1 = __importDefault(require("redis-lock"));
 const amqp_connection_manager_1 = require("amqp-connection-manager");
 const events_1 = require("events");
 // import { newTrace, traceTypes } from '@autofleet/outbreak';
+const util_1 = require("util");
 const logger_1 = __importDefault(require("./logger"));
 const rabbitError_1 = __importDefault(require("./rabbitError"));
 const redis_1 = __importDefault(require("./redis"));
@@ -27,9 +29,9 @@ const defaultOptions = {
     retries: 1,
     deadMessageTtl: DEFAULT_DEAD_TTL_TWO_DAYS,
 };
-const SECONDS_TO_MILLISECONDS = 1000;
 const assertExchangeFanout = (c, exchangeName) => c.assertExchange(exchangeName, 'fanout');
 const connectionCreatedConst = 'connectionCreated';
+const DEFAULT_LOCK_TIMEOUT = 1000;
 class RabbitMq {
     constructor(options, redisConfig) {
         this.PUBLISH_TIMEOUT = Number(process.env.RABBITMQ_PUBLISH_TIMEOUT) || 60000;
@@ -37,15 +39,18 @@ class RabbitMq {
         this.DISCONNECT_MSG = 'rabbit: connection disconnect';
         this.RESCONNECT_MSG = 'rabbit: reconnecting';
         this.shouldConsumeMessageByTimestamp = (msg) => __awaiter(this, void 0, void 0, function* () {
-            const { properties: { timestamp, headers } } = msg;
-            if (timestamp && (headers === null || headers === void 0 ? void 0 : headers.redisTimestampValidationKey) && this.redisClient) {
-                const lastMessageTimestamp = yield this.redisClient.getAsync(headers.redisTimestampValidationKey);
-                logger_1.default.info('rabbit: checking if should consume params', {
-                    msg, timestamp, headers, lastMessageTimestamp,
-                });
-                return !lastMessageTimestamp || (parseInt(lastMessageTimestamp, 10) <= parseInt(timestamp, 10));
+            if (msg) {
+                const { properties: { timestamp, headers } } = msg;
+                if (timestamp && (headers === null || headers === void 0 ? void 0 : headers.redisTimestampValidationKey) && this.redisClient) {
+                    const lastMessageTimestamp = yield this.redisClient.getAsync(headers.redisTimestampValidationKey);
+                    logger_1.default.info('rabbit: checking if should consume params', {
+                        msg, timestamp, headers, lastMessageTimestamp,
+                    });
+                    return !lastMessageTimestamp || (parseInt(lastMessageTimestamp, 10) <= parseInt(timestamp, 10));
+                }
+                return true;
             }
-            return true;
+            return false;
         });
         this.ack = (channel, shouldUpdateRedisTimestamp = false) => (msg) => __awaiter(this, void 0, void 0, function* () {
             yield channel.ack(msg);
@@ -79,6 +84,9 @@ class RabbitMq {
         this.queues = {};
         this.options = options;
         this.redisClient = redisConfig && redis_1.default(redisConfig);
+        if (this.redisClient) {
+            this.redisLock = util_1.promisify(redis_lock_1.default(this.redisClient));
+        }
     }
     static parseMsg(msg) {
         let { content } = msg;
@@ -121,11 +129,11 @@ class RabbitMq {
                     this.exchanges = {};
                     this.queues = {};
                     if ((_a = this.options) === null || _a === void 0 ? void 0 : _a.reconnect) {
-                        console.error(`${this.RESCONNECT_MSG}${err && ` - ${err}`}`);
+                        logger_1.default.error(`${this.RESCONNECT_MSG}${err && ` - ${err}`}`);
                         this.connection = null;
                     }
                     else {
-                        console.error(`${this.DISCONNECT_MSG}${err && ` - ${err}`}`);
+                        logger_1.default.error(`${this.DISCONNECT_MSG}${err && ` - ${err}`}`);
                     }
                 });
                 this.creatingConnection = false;
@@ -198,6 +206,45 @@ class RabbitMq {
         });
     }
     consume(queue, callback, options) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (options && options.useConsumeWithLock) {
+                return this.consumeWithLock(queue, callback, options);
+            }
+            return this.consumeFromRabbit(queue, callback, options);
+        });
+    }
+    consumeWithLock(queue, callback, options) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.redisLock) {
+                throw new Error('Usage of consumeWithLock requires RedisInstance');
+            }
+            const lockTimeout = options && options.lockTimeout ? options.lockTimeout : DEFAULT_LOCK_TIMEOUT;
+            logger_1.default.info(`Consuming with lock from queue ${queue} with lockTimeout: ${lockTimeout}ms`);
+            return this.consumeFromRabbit(queue, (msg, ack, nack) => __awaiter(this, void 0, void 0, function* () {
+                if (!msg) {
+                    return null;
+                }
+                const { properties: { timestamp, headers } } = msg;
+                let releaseLock = null;
+                try {
+                    if (timestamp && (headers === null || headers === void 0 ? void 0 : headers.redisTimestampValidationKey)) {
+                        if (this.redisLock) {
+                            releaseLock = yield this.redisLock(headers.redisTimestampValidationKey, lockTimeout);
+                            yield callback(msg, ack, nack);
+                            yield releaseLock();
+                        }
+                    }
+                }
+                catch (err) {
+                    if (releaseLock) {
+                        yield releaseLock();
+                    }
+                    throw err;
+                }
+            }), options);
+        });
+    }
+    consumeFromRabbit(queue, callback, options) {
         return __awaiter(this, void 0, void 0, function* () {
             const optionsWithDefaults = Object.assign(Object.assign({}, defaultOptions), options);
             RabbitMq.validateName('queue', queue);
@@ -208,20 +255,20 @@ class RabbitMq {
                 yield c.prefetch(limit, true);
                 return Promise.all([
                     c.consume(queue, (msg) => __awaiter(this, void 0, void 0, function* () {
-                        if (msg) {
-                            const parsedMessage = RabbitMq.parseMsg(msg);
-                            // newTrace(traceTypes.RABBIT);
-                            const shouldConsume = yield this.shouldConsumeMessageByTimestamp(parsedMessage);
-                            logger_1.default.info('rabbit: trying to consume', { queue, shouldConsume, msg });
-                            if (!shouldConsume) {
-                                return this.ack(channel)(msg);
-                            }
-                            try {
-                                yield callback(parsedMessage, this.ack(channel, true), this.nack(channel, queue, optionsWithDefaults, { messageTtl: deadMessageTtl }));
-                            }
-                            catch (e) {
-                                yield this.nack(channel, queue, optionsWithDefaults, { messageTtl: deadMessageTtl })(msg);
-                            }
+                        if (!msg) {
+                            return null;
+                        }
+                        const parsedMessage = RabbitMq.parseMsg(msg);
+                        const shouldConsume = yield this.shouldConsumeMessageByTimestamp(parsedMessage);
+                        logger_1.default.info('rabbit: trying to consume', { queue, shouldConsume, msg });
+                        if (!shouldConsume) {
+                            return this.ack(channel)(msg);
+                        }
+                        try {
+                            yield callback(parsedMessage, this.ack(channel, true), this.nack(channel, queue, optionsWithDefaults, { messageTtl: deadMessageTtl }));
+                        }
+                        catch (e) {
+                            yield this.nack(channel, queue, optionsWithDefaults, { messageTtl: deadMessageTtl })(msg);
                         }
                     })),
                 ]);
@@ -242,16 +289,7 @@ class RabbitMq {
                 yield c.prefetch(limit, true);
                 return Promise.all([
                     c.bindQueue(queue, exchange, ''),
-                    c.consume(queue, (msg) => __awaiter(this, void 0, void 0, function* () {
-                        const parsedMessage = RabbitMq.parseMsg(msg);
-                        // newTrace(traceTypes.RABBIT);
-                        const shouldConsume = yield this.shouldConsumeMessageByTimestamp(parsedMessage);
-                        logger_1.default.info('rabbit: trying to consume', { queue, shouldConsume, msg });
-                        if (!shouldConsume) {
-                            return this.ack(channel)(msg);
-                        }
-                        callback(parsedMessage, this.ack(channel, true), this.nack(channel, queue, optionsWithDefaults, { messageTtl: deadMessageTtl }));
-                    })),
+                    this.consume(queue, callback, options),
                 ]);
             }));
         });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@autofleet/rabbit",
-  "version": "2.1.16-beta",
+  "version": "2.2.0-beta-1",
   "description": "",
   "main": "dist/index.js",
   "scripts": {
@@ -16,7 +16,7 @@
   },
   "dependencies": {
     "@autofleet/logger": "^1.2.8",
-    "@types/amqp-connection-manager": "^2.0.10",
+    "@types/amqp-connection-manager": "^2.0.12",
     "@types/amqplib": "^0.5.16",
     "@types/uuid": "^8.3.3",
     "amqp-connection-manager": "^3.7.0",
@@ -24,6 +24,7 @@
     "bluebird": "^3.7.2",
     "moment": "^2.29.1",
     "redis": "^3.1.2",
+    "redis-lock": "^0.1.4",
     "uuid": "^8.3.2"
   },
   "devDependencies": {

package/src/index.ts

@@ -1,10 +1,12 @@
 /* eslint-disable @typescript-eslint/ban-ts-comment,@typescript-eslint/ban-types,@typescript-eslint/no-unused-vars,consistent-return,no-async-promise-executor,no-param-reassign,no-empty */
 // eslint-disable-next-line max-classes-per-file
 import moment from 'moment';
-import { ConfirmChannel, Options } from 'amqplib';
+import RedisLock from 'redis-lock';
+import { ConfirmChannel, Options, ConsumeMessage } from 'amqplib';
 import { AmqpConnectionManager, ChannelWrapper, connect } from 'amqp-connection-manager';
 import { EventEmitter } from 'events';
 // import { newTrace, traceTypes } from '@autofleet/outbreak';
+import { promisify } from 'util';
 import logger from './logger';
 import RabbitError from './rabbitError';
 import getRedisInstance, { RedisConfig } from './redis';
@@ -33,6 +35,19 @@ type CustomMessageHeaders = {
   redisTimestampValidationKey?: string;
 }
 
+type RedisLockType = (args0?: string, arg1?: number) => Promise<any>
+
+type ConsumeMessageOrNull = ConsumeMessage | null;
+interface ConsumeOptions {
+  retries?: number;
+  deadMessageTtl?: number;
+  limit?: number;
+  lockTimeout?: number;
+  useConsumeWithLock?: boolean;
+}
+
+type CallbackFunction = (msg: ConsumeMessage, ack: Function, nack: Function) => Promise<any>
+
 export interface AfRabbitOptions {
   reconnect: boolean;
 }
@@ -44,13 +59,15 @@ const defaultOptions = {
   retries: 1,
   deadMessageTtl: DEFAULT_DEAD_TTL_TWO_DAYS,
 };
-const SECONDS_TO_MILLISECONDS = 1000;
 
 const assertExchangeFanout = (c: any, exchangeName: string) => c.assertExchange(exchangeName, 'fanout');
 
 const connectionCreatedConst = 'connectionCreated';
+
+const DEFAULT_LOCK_TIMEOUT = 1000;
+
 class RabbitMq implements IAfRabbitMq {
-  static parseMsg(msg: any) {
+  static parseMsg(msg: any) : any {
     let { content } = msg;
     content = content.toString();
 
@@ -101,6 +118,8 @@ class RabbitMq implements IAfRabbitMq {
 
   redisClient: any;
 
+  redisLock?: RedisLockType;
+
   constructor(options?: AfRabbitOptions, redisConfig?: RedisConfig) {
     this.em = new EventEmitter();
     this.channel = null;
@@ -110,21 +129,27 @@ class RabbitMq implements IAfRabbitMq {
     this.queues = {};
     this.options = options;
     this.redisClient = redisConfig && getRedisInstance(redisConfig);
+    if (this.redisClient) {
+      this.redisLock = promisify(RedisLock(this.redisClient)) as RedisLockType;
+    }
   }
 
-  private shouldConsumeMessageByTimestamp = async (msg: any) => {
-    const { properties: { timestamp, headers } } = msg;
-    if (timestamp && headers?.redisTimestampValidationKey && this.redisClient) {
-      const lastMessageTimestamp = await this.redisClient.getAsync(headers.redisTimestampValidationKey);
-      logger.info('rabbit: checking if should consume params', {
-        msg, timestamp, headers, lastMessageTimestamp,
-      });
-      return !lastMessageTimestamp || (parseInt(lastMessageTimestamp, 10) <= parseInt(timestamp, 10));
+  private shouldConsumeMessageByTimestamp = async (msg: ConsumeMessageOrNull) => {
+    if (msg) {
+      const { properties: { timestamp, headers } } = msg;
+      if (timestamp && headers?.redisTimestampValidationKey && this.redisClient) {
+        const lastMessageTimestamp = await this.redisClient.getAsync(headers.redisTimestampValidationKey);
+        logger.info('rabbit: checking if should consume params', {
+          msg, timestamp, headers, lastMessageTimestamp,
+        });
+        return !lastMessageTimestamp || (parseInt(lastMessageTimestamp, 10) <= parseInt(timestamp, 10));
+      }
+      return true;
     }
-    return true;
+    return false;
   }
 
-  public ack = (channel: ChannelWrapper, shouldUpdateRedisTimestamp = false) => async (msg: any) => {
+  public ack = (channel: ChannelWrapper, shouldUpdateRedisTimestamp = false) => async (msg: ConsumeMessage) : Promise<any> => {
     await channel.ack(msg);
     const { properties: { timestamp, headers } } = msg;
     if (shouldUpdateRedisTimestamp && timestamp && headers?.redisTimestampValidationKey && this.redisClient) {
@@ -181,10 +206,10 @@ class RabbitMq implements IAfRabbitMq {
           this.exchanges = {};
           this.queues = {};
           if (this.options?.reconnect) {
-            console.error(`${this.RESCONNECT_MSG}${err && ` - ${err}`}`);
+            logger.error(`${this.RESCONNECT_MSG}${err && ` - ${err}`}`);
             this.connection = null;
           } else {
-            console.error(`${this.DISCONNECT_MSG}${err && ` - ${err}`}`);
+            logger.error(`${this.DISCONNECT_MSG}${err && ` - ${err}`}`);
           }
         });
       this.creatingConnection = false;
@@ -250,7 +275,46 @@ class RabbitMq implements IAfRabbitMq {
     return queue;
   }
 
-  async consume(queue: string, callback: (msg: any, ack: Function, nack: Function) => any, options?: any) {
+  async consume(queue: string, callback: CallbackFunction, options?: ConsumeOptions): Promise<any> {
+    if (options && options.useConsumeWithLock) {
+      return this.consumeWithLock(queue, callback, options);
+    }
+    return this.consumeFromRabbit(queue, callback, options);
+  }
+
+  private async consumeWithLock(queue: string, callback: CallbackFunction, options?: ConsumeOptions) : Promise<any> {
+    if (!this.redisLock) {
+      throw new Error('Usage of consumeWithLock requires RedisInstance');
+    }
+    const lockTimeout = options && options.lockTimeout ? options.lockTimeout : DEFAULT_LOCK_TIMEOUT;
+    logger.info(`Consuming with lock from queue ${queue} with lockTimeout: ${lockTimeout}ms`);
+    return this.consumeFromRabbit(queue, async (msg, ack, nack) => {
+      if (!msg) {
+        return null;
+      }
+      const { properties: { timestamp, headers } } = msg;
+      let releaseLock = null;
+      try {
+        if (timestamp && headers?.redisTimestampValidationKey) {
+          if (this.redisLock) {
+            releaseLock = await this.redisLock(
+              headers.redisTimestampValidationKey,
+              lockTimeout,
+            );
+            await callback(msg, ack, nack);
+            await releaseLock();
+          }
+        }
+      } catch (err) {
+        if (releaseLock) {
+          await releaseLock();
+        }
+        throw err;
+      }
+    }, options);
+  }
+
+  private async consumeFromRabbit(queue: string, callback: CallbackFunction, options?: ConsumeOptions): Promise<any> {
     const optionsWithDefaults = { ...defaultOptions, ...options };
     RabbitMq.validateName('queue', queue);
     const { limit, deadMessageTtl } = optionsWithDefaults;
@@ -261,33 +325,32 @@ class RabbitMq implements IAfRabbitMq {
       return Promise.all([
         c.consume(
           queue,
-          async (msg: any) => {
-            if (msg) {
-              const parsedMessage = RabbitMq.parseMsg(msg);
-              // newTrace(traceTypes.RABBIT);
-              const shouldConsume = await this.shouldConsumeMessageByTimestamp(parsedMessage);
-              logger.info('rabbit: trying to consume', { queue, shouldConsume, msg });
-              if (!shouldConsume) {
-                return this.ack(channel)(msg);
-              }
-              try {
-                await callback(
-                  parsedMessage,
-                  this.ack(channel, true),
-                  this.nack(channel, queue, optionsWithDefaults, { messageTtl: deadMessageTtl }),
-                );
-              } catch (e) {
-                await this.nack(channel, queue, optionsWithDefaults, { messageTtl: deadMessageTtl })(msg);
-              }
+          async (msg: ConsumeMessageOrNull) => {
+            if (!msg) {
+              return null;
+            }
+            const parsedMessage = RabbitMq.parseMsg(msg);
+            const shouldConsume = await this.shouldConsumeMessageByTimestamp(parsedMessage);
+            logger.info('rabbit: trying to consume', { queue, shouldConsume, msg });
+            if (!shouldConsume) {
+              return this.ack(channel)(msg);
+            }
+            try {
+              await callback(
+                parsedMessage,
+                this.ack(channel, true),
+                this.nack(channel, queue, optionsWithDefaults, { messageTtl: deadMessageTtl }),
+              );
+            } catch (e) {
+              await this.nack(channel, queue, optionsWithDefaults, { messageTtl: deadMessageTtl })(msg);
             }
           },
-
         ),
       ]);
     });
   }
 
-  async consumeFromExchange(queue: string, exchange: string, callback: (msg: any, ack: Function, nack: Function) => any, options?: any) {
+  async consumeFromExchange(queue: string, exchange: string, callback: CallbackFunction, options?: ConsumeOptions) : Promise<any> {
     const optionsWithDefaults = { ...defaultOptions, ...options };
     RabbitMq.validateName('exchange', exchange);
     RabbitMq.validateName('queue', queue);
@@ -301,24 +364,16 @@ class RabbitMq implements IAfRabbitMq {
       await c.prefetch(limit, true);
       return Promise.all([
         c.bindQueue(queue, exchange, ''),
-        c.consume(
+        this.consume(
           queue,
-          async (msg: any) => {
-            const parsedMessage = RabbitMq.parseMsg(msg);
-            // newTrace(traceTypes.RABBIT);
-            const shouldConsume = await this.shouldConsumeMessageByTimestamp(parsedMessage);
-            logger.info('rabbit: trying to consume', { queue, shouldConsume, msg });
-            if (!shouldConsume) {
-              return this.ack(channel)(msg);
-            }
-            callback(parsedMessage, this.ack(channel, true), this.nack(channel, queue, optionsWithDefaults, { messageTtl: deadMessageTtl }));
-          },
+          callback,
+          options,
         ),
       ]);
     });
   }
 
-  async publish(exchange: string, content: any, customHeaders?: any) {
+  async publish(exchange: string, content: any, customHeaders?: any) : Promise<any> {
     RabbitMq.validateName('exchange', exchange);
     const channel: ChannelWrapper = await this.assertChannel();
     await this.assertExchange(exchange);
@@ -327,7 +382,7 @@ class RabbitMq implements IAfRabbitMq {
       RabbitMq.getPublishOptions(customHeaders));
   }
 
-  async sendToQueue(queue: string, content: any, options?: any, customHeaders?: any) {
+  async sendToQueue(queue: string, content: any, options?: any, customHeaders?: any) : Promise<boolean> {
     RabbitMq.validateName('queue', queue);
     const channel: ChannelWrapper = await this.assertChannel();
     await this.assertQueue(queue, options);

package/src/redis-lock.d.ts

@@ -0,0 +1,4 @@
+declare module 'redis-lock' {
+    const lock: any;
+    export = lock;
+  }