@autofleet/rabbit 2.3.6 → 2.3.7

package/src/index-new.ts

@@ -1,427 +0,0 @@
-/* eslint-disable @typescript-eslint/ban-ts-comment,@typescript-eslint/ban-types,@typescript-eslint/no-unused-vars,consistent-return,no-async-promise-executor,no-param-reassign,no-empty */
-// eslint-disable-next-line max-classes-per-file
-import moment from 'moment';
-import RedisLock from 'redis-lock';
-import { ConfirmChannel, Options, ConsumeMessage } from 'amqplib';
-import { AmqpConnectionManager, ChannelWrapper, connect } from 'amqp-connection-manager';
-import { EventEmitter } from 'events';
-// import { newTrace, traceTypes } from '@autofleet/outbreak';
-import { promisify } from 'util';
-import logger from './logger';
-import RabbitError from './rabbitError';
-import getRedisInstance, { RedisConfig } from './redis';
-
-export interface IAfRabbitMq {
-  ack: any;
-  nack: any;
-  assertChannel: any;
-  assertExchange: any;
-  assertQueue: any;
-  consume: any;
-  consumeFromExchange: any;
-  publish: any;
-  sendToQueue: any;
-  redisClient?: any;
-}
-
-interface ExchangesCache {
-  [key: string]: any
-}
-interface QueuesCache {
-  [key: string]: any
-}
-
-type CustomMessageHeaders = {
-  redisTimestampValidationKey?: string;
-}
-
-type RedisLockType = (args0?: string, arg1?: number) => Promise<any>
-
-type ConsumeMessageOrNull = ConsumeMessage | null;
-interface ConsumeOptions {
-  retries?: number;
-  deadMessageTtl?: number;
-  limit?: number;
-  lockTimeout?: number;
-  useConsumeWithLock?: boolean;
-}
-
-type CallbackFunction = (msg: ConsumeMessage, ack: Function, nack: Function) => Promise<any>
-
-export interface AfRabbitOptions {
-  disableReconnect?: boolean;
-  host?: string;
-}
-
-const DEFAULT_DEAD_TTL_TWO_DAYS = 60000 * 60 * 48;
-const DEFAULT_LOCK_TIMEOUT = 1000 * 5;
-const RETRY_HEADER = 'x-retry-count';
-const DEFAULT_USE_CONSUME_WITH_LOCK = false;
-
-const defaultOptions = {
-  limit: 1,
-  retries: 1,
-  deadMessageTtl: DEFAULT_DEAD_TTL_TWO_DAYS,
-  lockTimeout: DEFAULT_LOCK_TIMEOUT,
-  useConsumeWithLock: DEFAULT_USE_CONSUME_WITH_LOCK,
-};
-
-const assertExchangeFanout = async (c: ConfirmChannel | ChannelWrapper, exchangeName: string) => c.assertExchange(exchangeName, 'fanout');
-
-const connectionCreatedConst = 'connectionCreated';
-
-const wrapSetImmediate = (callback: () => any) => new Promise<any>(async (resolve, reject) => {
-  setImmediate(async () => {
-    try {
-      const value = await callback();
-      resolve(value);
-    } catch (error) {
-      reject(error);
-    }
-  });
-});
-
-class RabbitMq implements IAfRabbitMq {
-  static parseMsg(msg: any) : any {
-    let { content } = msg;
-    content = content.toString();
-
-    try {
-      content = JSON.parse(content);
-    } catch (e) { }
-
-    return {
-      ...msg,
-      content,
-    };
-  }
-
-  static validateName(type: string, name: string) {
-    if (!name || name === '') {
-      throw new RabbitError(`error while using ${type} with no name`);
-    }
-  }
-
-  static getPublishOptions(customHeaders: CustomMessageHeaders = {}) {
-    return {
-      timestamp: moment().unix(),
-      headers: {
-        creationTimestamp: moment().valueOf(),
-        ...customHeaders,
-      },
-    };
-  }
-
-  PUBLISH_TIMEOUT = Number(process.env.RABBITMQ_PUBLISH_TIMEOUT) || 60000;
-
-  PUBLISH_ERROR_MSG = `rabbit: publish timeout(${this.PUBLISH_TIMEOUT}ms) has pass, exchange: `;
-
-  DISCONNECT_MSG = 'rabbit: connection disconnect';
-
-  RESCONNECT_MSG = 'rabbit: reconnecting';
-
-  channel: ChannelWrapper | null;
-
-  connection: AmqpConnectionManager | null;
-
-  em: EventEmitter;
-
-  creatingConnection: boolean;
-
-  exchanges: ExchangesCache;
-
-  queues: QueuesCache;
-
-  host: string | null;
-
-  options: AfRabbitOptions | undefined;
-
-  redisClient: any;
-
-  redisLock?: RedisLockType;
-
-  constructor(options?: AfRabbitOptions, redisConfig?: RedisConfig) {
-    this.em = new EventEmitter();
-    this.channel = null;
-    this.connection = null;
-    this.creatingConnection = false;
-    this.exchanges = {};
-    this.queues = {};
-    this.options = options;
-    this.host = options?.host ?? null;
-    this.redisClient = redisConfig && getRedisInstance(redisConfig);
-    if (this.redisClient) {
-      this.redisLock = promisify(RedisLock(this.redisClient)) as RedisLockType;
-    }
-  }
-
-  private shouldConsumeMessageByTimestamp = async (msg: ConsumeMessageOrNull) => {
-    if (msg) {
-      const { properties: { headers } } = msg;
-      const timestamp = headers?.creationTimestamp;
-
-      if (timestamp && headers?.redisTimestampValidationKey && this.redisClient) {
-        const lastMessageTimestamp = await this.redisClient.getAsync(headers.redisTimestampValidationKey);
-        return !lastMessageTimestamp || (parseInt(lastMessageTimestamp, 10) <= parseInt(timestamp, 10));
-      }
-      return true;
-    }
-    return false;
-  }
-
-  public ack = (channel: ChannelWrapper, msg: ConsumeMessageOrNull, shouldUpdateRedisTimestamp = false, releaseLock = null) => async (userMsg: ConsumeMessage) : Promise<any> => {
-    if (msg) {
-      await channel.ack(msg);
-      const { properties: { headers } } = msg;
-      const timestamp = headers?.creationTimestamp;
-
-      if (shouldUpdateRedisTimestamp && timestamp && headers?.redisTimestampValidationKey && this.redisClient) {
-        const parsedTimestamp = parseInt(timestamp, 10);
-        await this.redisClient.setAsync(headers.redisTimestampValidationKey, parsedTimestamp, 'EX', 3600);
-        await this.unlockRedisIfNeeded(releaseLock);
-      }
-    }
-  }
-
-  public nack = (
-    channel: ChannelWrapper,
-    queue: string,
-    options: any,
-    deadQueueOptions: Options.AssertQueue,
-    msg: ConsumeMessageOrNull,
-    releaseLock: any,
-  ) => async (
-    userMsg: ConsumeMessageOrNull,
-    {
-      skipRetry = false,
-    }: any = { },
-  ) : Promise<any> => {
-    await this.unlockRedisIfNeeded(releaseLock);
-    if (channel && msg) {
-      if (
-        !skipRetry
-        && (
-          !msg.properties.headers[RETRY_HEADER]
-          || parseInt(msg.properties.headers[RETRY_HEADER], 10) < options.retries
-        )
-      ) {
-        await this.sendToQueue(queue, RabbitMq.parseMsg(msg).content, options, {
-          ...msg.properties.headers,
-          [RETRY_HEADER]: msg.properties.headers[RETRY_HEADER]
-            ? msg.properties.headers[RETRY_HEADER] + 1
-            : 1,
-        });
-      } else {
-        const deadQueue = `${queue}-dead`;
-        await this.sendToQueue(deadQueue, RabbitMq.parseMsg(msg).content, deadQueueOptions, {
-          ...msg.properties.headers,
-          [RETRY_HEADER]: msg.properties.headers[RETRY_HEADER]
-            ? msg.properties.headers[RETRY_HEADER] + 1
-            : 1,
-        });
-      }
-      await channel.ack(msg);
-    }
-  }
-
-  async getConnection() {
-    return new Promise<AmqpConnectionManager>(async (resolve) => {
-      if (this.connection !== null) {
-        return resolve(this.connection);
-      }
-      if (this.creatingConnection) {
-        this.em.once(connectionCreatedConst, resolve);
-        return;
-      }
-      this.creatingConnection = true;
-      logger.info('rabbit: env username', { userName: (process.env.RABBITMQ_USERNAME || 'doesnt exist') });
-      const username: string = process.env.RABBITMQ_USERNAME || 'guest';
-      const password: string = process.env.RABBITMQ_PASSWORD || 'guest';
-      const host: string = this.host || process.env.RABBITMQ_SERVICE_HOST || '';
-      const connection: AmqpConnectionManager = await connect([`amqp://${username}:${password}@${host}`], { reconnectTimeInSeconds: 0.5 });
-      this.connection = connection;
-      this.creatingConnection = false;
-      this.em.emit(connectionCreatedConst, connection);
-      resolve(connection);
-    });
-  }
-
-  async getNewChannel() {
-    const connection: AmqpConnectionManager = await this.getConnection();
-    return connection.createChannel({});
-  }
-
-  async assertChannel() {
-    return new Promise<ChannelWrapper>(async (resolve, reject) => {
-      if (this.channel) {
-        return resolve(this.channel);
-      }
-
-      try {
-        const connection: AmqpConnectionManager = await this.getConnection();
-        if (this.channel === null) {
-          this.channel = await connection.createChannel({});
-        }
-        resolve(this.channel);
-      } catch (e) {
-        reject(e);
-      }
-    });
-  }
-
-  async assertExchange(exchangeName: string, options?: any) {
-    const channel: ChannelWrapper = await this.assertChannel();
-    if (this.exchanges[exchangeName]) {
-      return this.exchanges[exchangeName];
-    }
-    const exchange = await assertExchangeFanout(channel, exchangeName);
-    this.exchanges[exchangeName] = exchange;
-    return exchange;
-  }
-
-  async getQueueLength(queue: string) {
-    RabbitMq.validateName('queue', queue);
-    const channel: ChannelWrapper = await this.assertChannel();
-    // @ts-ignore
-    return channel.checkQueue(queue);
-  }
-
-  async bindQueue(queue: string, exchange: string) {
-    const channel: ChannelWrapper = await this.assertChannel();
-    await channel.addSetup((setupChannel: ConfirmChannel) => setupChannel.bindQueue(queue, exchange, ''));
-    // @ts-ignore
-    return channel.bindQueue(queue, exchange, '');
-  }
-
-  async assertQueue(queueName: string, options?: Options.AssertQueue) {
-    RabbitMq.validateName('queue', queueName);
-    const channel: ChannelWrapper = await this.assertChannel();
-    if (this.queues[queueName]) {
-      return this.queues[queueName];
-    }
-    await channel.addSetup((setupChannel: ConfirmChannel) => setupChannel.assertQueue(queueName, options));
-    const queue = await channel.assertQueue(queueName, options);
-    this.queues[queueName] = queueName;
-    return queue;
-  }
-
-  async consume(queue: string, callback: CallbackFunction, options?: ConsumeOptions): Promise<any> {
-    return this.consumeFromRabbit(queue, callback, options);
-  }
-
-  private async lockRedisIfNeeded(msg: any, options: any) {
-    const { properties: { headers } } = msg;
-    const timestamp = headers?.creationTimestamp;
-    let releaseLock = null;
-
-    if (options.useConsumeWithLock && timestamp && headers?.redisTimestampValidationKey && this.redisLock) {
-      releaseLock = await this.redisLock(
-        headers.redisTimestampValidationKey,
-        options?.lockTimeout || DEFAULT_LOCK_TIMEOUT,
-      );
-    }
-    return releaseLock;
-  }
-
-  private async unlockRedisIfNeeded(releaseLock: any) {
-    if (this.redisLock && releaseLock) {
-      await releaseLock();
-    }
-  }
-
-  private async consumeFromRabbit(queue: string, callback: CallbackFunction, options?: ConsumeOptions): Promise<any> {
-    const optionsWithDefaults = { ...defaultOptions, ...options };
-    RabbitMq.validateName('queue', queue);
-    const {
-      limit, deadMessageTtl, useConsumeWithLock, lockTimeout,
-    } = optionsWithDefaults;
-    if (useConsumeWithLock) {
-      if (!this.redisLock) {
-        throw new Error('Usage of consumeWithLock requires RedisInstance');
-      }
-      logger.info(`Consuming with lock from queue ${queue} with lockTimeout: ${lockTimeout}ms`);
-    }
-    const channel: ChannelWrapper = await this.getNewChannel();
-    return channel.addSetup(async (c: ConfirmChannel) => {
-      await c.assertQueue(queue);
-      await c.prefetch(limit, true);
-      return Promise.all([
-        c.consume(
-          queue,
-          async (msg: ConsumeMessageOrNull) => {
-            if (!msg) {
-              return null;
-            }
-            const parsedMessage = RabbitMq.parseMsg(msg);
-            const releaseLock = await this.lockRedisIfNeeded(parsedMessage, optionsWithDefaults);
-            const shouldConsume = await this.shouldConsumeMessageByTimestamp(parsedMessage);
-            if (!shouldConsume) {
-              await this.unlockRedisIfNeeded(releaseLock);
-              return this.ack(channel, msg)(msg);
-            }
-            try {
-              await callback(
-                parsedMessage,
-                this.ack(channel, msg, true, releaseLock),
-                this.nack(channel, queue, optionsWithDefaults, { messageTtl: deadMessageTtl }, msg, releaseLock),
-              );
-            } catch (e) {
-              await this.nack(channel, queue, optionsWithDefaults, { messageTtl: deadMessageTtl }, msg, releaseLock)(msg);
-            }
-          },
-        ),
-      ]);
-    });
-  }
-
-  async consumeFromExchange(queue: string, exchange: string, callback: CallbackFunction, options?: ConsumeOptions) : Promise<any> {
-    const optionsWithDefaults = { ...defaultOptions, ...options };
-    RabbitMq.validateName('exchange', exchange);
-    RabbitMq.validateName('queue', queue);
-    const { limit, deadMessageTtl } = optionsWithDefaults;
-    const channel: ChannelWrapper = await this.getNewChannel();
-
-    return channel.addSetup(async (c: ConfirmChannel) => {
-      await assertExchangeFanout(c, exchange);
-      await c.assertQueue(queue);
-      await c.prefetch(limit, true);
-      return Promise.all([
-        c.bindQueue(queue, exchange, ''),
-        this.consume(
-          queue,
-          callback,
-          options,
-        ),
-      ]);
-    });
-  }
-
-  async publish(exchange: string, content: any, customHeaders?: any) : Promise<boolean> {
-    return wrapSetImmediate(async () => {
-      RabbitMq.validateName('exchange', exchange);
-      const channel: ChannelWrapper = await this.assertChannel();
-      await this.assertExchange(exchange);
-      await channel.publish(exchange, '',
-        Buffer.from(JSON.stringify(content)),
-        RabbitMq.getPublishOptions(customHeaders));
-    });
-  }
-
-  async sendToQueue(queue: string, content: any, options?: any, customHeaders?: any) : Promise<boolean> {
-    return wrapSetImmediate(async () => {
-      RabbitMq.validateName('queue', queue);
-      const channel: ChannelWrapper = await this.assertChannel();
-      await this.assertQueue(queue, options);
-      return channel.sendToQueue(queue,
-        Buffer.from(JSON.stringify(content)),
-        RabbitMq.getPublishOptions(customHeaders));
-    });
-  }
-
-  async isConnected() : Promise<boolean> {
-    const connection = await this.getConnection();
-    return connection.isConnected();
-  }
-}
-
-export default RabbitMq;

package/src/memory-test.ts

@@ -1,44 +0,0 @@
-/* eslint-disable no-await-in-loop */
-// eslint-disable-next-line import/no-extraneous-dependencies
-import proxy from 'node-tcp-proxy';
-import RabbitMq from './index-new';
-
-const delay = (ms = 1000) => new Promise((resolve) => setTimeout(() => resolve(), ms));
-const callback = (eq: any, timeout = 0) => async (msg: any, ack: any, nack: any) => {
-  setTimeout(async () => {
-    // console.log(msg)
-    await ack(msg);
-  }, timeout);
-  return null;
-};
-
-const payload = { ttt: 123 };
-const testId = 'www';
-
-const main = async () => {
-  const port = 6672;
-  const createProxy = () => proxy.createProxy(port, '0.0.0.0', 5672);
-  const restartProxy = async (currentProxy, sleep) => {
-    currentProxy.end();
-    await delay(sleep);
-    return proxy.createProxy(port, '0.0.0.0', 5672);
-  };
-  let currentProxy = createProxy();
-  const rabbit = new RabbitMq({ host: `127.0.0.1:${port}` });
-  const queue = `test-timestamp-validation-${testId}`;
-  const mockFn = callback(payload);
-  await rabbit.consume(queue, mockFn);
-  console.error('start');
-  const usedB = process.memoryUsage().heapUsed / 1024 / 1024;
-  console.log('Used before', usedB);
-  for (let i = 0; i < 10000; i += 1) {
-    // currentProxy = await restartProxy(currentProxy, 100);
-    await rabbit.sendToQueue(queue, payload);
-    await delay(10);
-    // expect(mockFn).toBeCalledTimes(i);
-    const used = process.memoryUsage().heapUsed / 1024 / 1024;
-    console.error('Used', i, used);
-  }
-};
-
-main();