@autofleet/rabbit 3.3.0-beta.0 → 3.3.0-beta.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -4,6 +4,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
4
4
  return (mod && mod.__esModule) ? mod : { "default": mod };
5
5
  };
6
6
  Object.defineProperty(exports, "__esModule", { value: true });
7
+ exports.sendCeleryTaskViaHttp = void 0;
7
8
  const events_1 = require("events");
8
9
  const util_1 = require("util");
9
10
  const moment_1 = __importDefault(require("moment"));
@@ -55,10 +56,57 @@ class RabbitMq {
55
56
  },
56
57
  };
57
58
  }
58
- constructor(options, redisConfig) {
59
+ constructor(options = {}, redisConfig) {
59
60
  this.DISCONNECT_MSG = 'rabbit: connection disconnect';
60
61
  this.RECONNECT_MSG = 'rabbit: connection disconnect - reconnecting';
61
62
  this.consumers = [];
63
+ this.doesVHostExist = false;
64
+ this.vhost = 'quorum-vhost';
65
+ this.oldConsumers = [];
66
+ this.assertVHost = async () => {
67
+ if (this.doesVHostExist) {
68
+ return;
69
+ }
70
+ const username = process.env.RABBITMQ_USERNAME || 'guest';
71
+ const password = process.env.RABBITMQ_PASSWORD || 'guest';
72
+ const credentials = Buffer.from(`${username}:${password}`).toString('base64');
73
+ const headers = {
74
+ Authorization: `Basic ${credentials}`,
75
+ 'Content-Type': 'application/json',
76
+ };
77
+ const rabbitHost = `http://${(this.options?.rabbitHost || process.env.RABBITMQ_SERVICE_HOST || 'localhost').split(':')[0]}:15672`;
78
+ const url = `${rabbitHost}/api/vhosts/${encodeURIComponent(this.vhost)}`;
79
+ try {
80
+ const response = await fetch(url, {
81
+ method: 'GET',
82
+ headers,
83
+ });
84
+ if (response.status === 200) {
85
+ this.doesVHostExist = true;
86
+ logger_1.default.info('Vhost exists', { vhost: this.vhost });
87
+ return;
88
+ }
89
+ if (response.status !== 404) {
90
+ logger_1.default.error('Failed to check vhost', { response });
91
+ throw new rabbitError_1.default('Failed to check vhost');
92
+ }
93
+ const createResponse = await fetch(url, {
94
+ method: 'PUT',
95
+ headers,
96
+ body: JSON.stringify({ default_queue_type: 'quorum' }),
97
+ });
98
+ if (!createResponse.ok) {
99
+ logger_1.default.error('Failed to create vhost', { response: createResponse });
100
+ throw new rabbitError_1.default('Failed to create vhost');
101
+ }
102
+ this.doesVHostExist = true;
103
+ logger_1.default.info('Vhost created', { vhost: this.vhost });
104
+ }
105
+ catch (error) {
106
+ logger_1.default.error('Failed to check or create vhost', { error });
107
+ throw error;
108
+ }
109
+ };
62
110
  this.shouldConsumeMessageByTimestamp = async (msg) => {
63
111
  if (msg) {
64
112
  const { properties: { headers } } = msg;
@@ -88,11 +136,11 @@ class RabbitMq {
88
136
  await this.unlockRedisIfNeeded(releaseLock);
89
137
  if (channel && msg) {
90
138
  if (!skipRetry
91
- && (!msg.properties.headers[consts_1.RETRY_HEADER]
92
- || parseInt(msg.properties.headers[consts_1.RETRY_HEADER], 10) < options.retries)) {
139
+ && (!msg.properties.headers?.[consts_1.RETRY_HEADER]
140
+ || parseInt(msg.properties.headers?.[consts_1.RETRY_HEADER], 10) < options.retries)) {
93
141
  await this.sendToQueue(queue, RabbitMq.parseMsg(msg).content, options, {
94
142
  ...msg.properties.headers,
95
- [consts_1.RETRY_HEADER]: msg.properties.headers[consts_1.RETRY_HEADER]
143
+ [consts_1.RETRY_HEADER]: msg.properties.headers?.[consts_1.RETRY_HEADER]
96
144
  ? msg.properties.headers[consts_1.RETRY_HEADER] + 1
97
145
  : 1,
98
146
  });
@@ -101,7 +149,7 @@ class RabbitMq {
101
149
  const deadQueue = `${queue}-dead`;
102
150
  await this.sendToQueue(deadQueue, RabbitMq.parseMsg(msg).content, deadQueueOptions, {
103
151
  ...msg.properties.headers,
104
- [consts_1.RETRY_HEADER]: msg.properties.headers[consts_1.RETRY_HEADER]
152
+ [consts_1.RETRY_HEADER]: msg.properties.headers?.[consts_1.RETRY_HEADER]
105
153
  ? msg.properties.headers[consts_1.RETRY_HEADER] + 1
106
154
  : 1,
107
155
  });
@@ -117,14 +165,15 @@ class RabbitMq {
117
165
  };
118
166
  this.em = new events_1.EventEmitter();
119
167
  this.channel = null;
168
+ this.publishChannelSetupPromise = null;
120
169
  this.connection = null;
121
170
  this.creatingConnection = false;
122
171
  this.exchanges = {};
123
172
  this.queues = {};
124
173
  this.queueSetupPromises = {};
174
+ this.assertExchangePromises = {};
125
175
  this.consumers = [];
126
176
  this.options = options;
127
- this.podId = `${options?.serviceName}${options?.podIp ? `-${options.podIp}` : ''}`;
128
177
  this.redisClient = redisConfig && (0, redis_1.default)(redisConfig);
129
178
  if (this.redisClient) {
130
179
  this.redisLock = (0, util_1.promisify)((0, redis_lock_1.default)(this.redisClient));
@@ -139,6 +188,18 @@ class RabbitMq {
139
188
  await this.gracefulShutdown('SIGINT');
140
189
  });
141
190
  }
191
+ // TODO: [QUORUM-PHASE-3] Delete the old properties that we use for the old consumers and publishers
192
+ this.oldEm = new events_1.EventEmitter();
193
+ this.oldChannel = null;
194
+ this.oldPublishChannelSetupPromise = null;
195
+ this.oldConnection = null;
196
+ this.oldCreatingConnection = false;
197
+ this.oldExchanges = {};
198
+ this.oldQueues = {};
199
+ this.oldQueueSetupPromises = {};
200
+ this.oldAssertExchangePromises = {};
201
+ this.oldConsumers = [];
202
+ this.oldConsumersTags = [];
142
203
  }
143
204
  async getConnection() {
144
205
  return new Promise(async (resolve, reject) => {
@@ -173,7 +234,7 @@ class RabbitMq {
173
234
  const password = process.env.RABBITMQ_PASSWORD || 'guest';
174
235
  const host = this.options?.rabbitHost || process.env.RABBITMQ_SERVICE_HOST || 'localhost';
175
236
  debug('rabbit: creating connection', { host, userName, HEARTBEAT });
176
- return [`amqp://${userName}:${password}@${host}?heartbeat=${HEARTBEAT}`];
237
+ return [`amqp://${userName}:${password}@${host}/${this.vhost}?heartbeat=${HEARTBEAT}`];
177
238
  };
178
239
  const defaultUrls = findServers();
179
240
  const connection = await (0, amqp_connection_manager_1.connect)(defaultUrls, {
@@ -190,7 +251,7 @@ class RabbitMq {
190
251
  });
191
252
  this.connection.on('connectFailed', (err) => {
192
253
  this.consumersTags = [];
193
- logger_1.default.error('rabbit: connection connectFailed', { err });
254
+ logger_1.default.error('rabbit: connection connectFailed', { err, advice: 'Check if the vhost exist', vhost: this.vhost });
194
255
  if (!isResolved) {
195
256
  isResolved = true;
196
257
  reject(err);
@@ -198,6 +259,7 @@ class RabbitMq {
198
259
  }
199
260
  });
200
261
  this.connection.on('disconnect', ({ err }) => {
262
+ // this.channel = null;
201
263
  this.consumersTags = [];
202
264
  debug('rabbit: connection closed');
203
265
  if (this.options?.disableReconnect) {
@@ -226,7 +288,7 @@ class RabbitMq {
226
288
  logger_1.default.error(`rabbit: error on get connection for new channel ${name} `, { e });
227
289
  throw e;
228
290
  }
229
- const channel = connection.createChannel({ ...options, name });
291
+ const channel = connection.createChannel({ ...options });
230
292
  (0, events_1.once)(channel, 'close').then((args) => {
231
293
  logger_1.default.error(`rabbit: channel ${name} closed`);
232
294
  onClose?.(args);
@@ -242,37 +304,45 @@ class RabbitMq {
242
304
  }
243
305
  }
244
306
  async assertChannel({ force = false } = {}) {
245
- return new Promise(async (resolve, reject) => {
246
- if (this.channel && !force) {
247
- return resolve(this.channel);
248
- }
249
- try {
250
- const channel = await this.getNewChannel({});
251
- channel.on('error', (err) => {
252
- logger_1.default.error('rabbit: channel error', { err });
253
- });
254
- this.channel = channel;
255
- resolve(channel);
256
- }
257
- catch (e) {
258
- reject(e);
259
- }
260
- });
307
+ if (!this.publishChannelSetupPromise) {
308
+ this.publishChannelSetupPromise = new Promise(async (resolve, reject) => {
309
+ if (this.channel && !force) {
310
+ return resolve(this.channel);
311
+ }
312
+ try {
313
+ const channel = await this.getNewChannel({});
314
+ channel.on('error', (err) => {
315
+ logger_1.default.error('rabbit: channel error', { err });
316
+ });
317
+ this.channel = channel;
318
+ resolve(channel);
319
+ }
320
+ catch (e) {
321
+ reject(e);
322
+ }
323
+ });
324
+ }
325
+ return this.publishChannelSetupPromise;
261
326
  }
262
327
  async assertExchange(exchangeName, options) {
263
328
  const channel = await this.assertChannel();
264
329
  if (this.exchanges[exchangeName]) {
330
+ delete this.assertExchangePromises[exchangeName];
265
331
  return this.exchanges[exchangeName];
266
332
  }
267
- const exchange = await (0, utils_1.assertExchangeFanout)(channel, exchangeName);
268
- this.exchanges[exchangeName] = exchange;
269
- return exchange;
333
+ if (this.assertExchangePromises[exchangeName]) {
334
+ return this.assertExchangePromises[exchangeName];
335
+ }
336
+ this.assertExchangePromises[exchangeName] = (0, utils_1.assertExchangeFanout)(channel, exchangeName);
337
+ this.exchanges[exchangeName] = await this.assertExchangePromises[exchangeName];
338
+ return this.exchanges[exchangeName];
270
339
  }
340
+ // TODO: [QUORUM-PHASE-2] Change the implementation to the new one
271
341
  async getQueueLength(queue) {
272
342
  RabbitMq.validateName('queue', queue);
273
- const { channel } = this;
343
+ const { oldChannel: channel } = this;
274
344
  if (!channel) {
275
- throw new Error('channel is not defined');
345
+ throw new rabbitError_1.default('channel is not defined');
276
346
  }
277
347
  debug('rabbit: getting queue length', { queue, connected: this.connection?.isConnected() });
278
348
  return channel?.checkQueue(queue);
@@ -285,19 +355,31 @@ class RabbitMq {
285
355
  debug('queue deleted', deleteQueueRes);
286
356
  return deleteQueueRes;
287
357
  }
358
+ // TODO: [QUORUM-PHASE-2] Change the implementation to the new one
288
359
  async bindQueue(queue, exchange) {
289
- const channel = await this.assertChannel();
360
+ const channel = await this.assertChannelOld();
290
361
  await channel.addSetup((setupChannel) => setupChannel.bindQueue(queue, exchange, ''));
291
362
  return channel.bindQueue(queue, exchange, '');
292
363
  }
293
364
  async setupQueue(queueName, options) {
294
365
  let queue;
366
+ const localeOptions = {
367
+ ...options,
368
+ durable: true,
369
+ arguments: {
370
+ ...options?.arguments,
371
+ 'x-consumer-timeout': 1000 * 60 * 60 * 24,
372
+ 'x-queue-type': 'quorum',
373
+ },
374
+ };
295
375
  try {
296
376
  const channel = await this.assertChannel();
297
377
  debug('assertQueue->channel.addSetup', { queueName });
298
- await channel.addSetup((setupChannel) => setupChannel.assertQueue(queueName, options));
378
+ await channel.addSetup(async (setupChannel) => {
379
+ await setupChannel.assertQueue(queueName, localeOptions);
380
+ });
299
381
  debug('assertQueue->channel.assertQueue', { queueName });
300
- queue = await channel.assertQueue(queueName, options);
382
+ queue = await channel.assertQueue(queueName, localeOptions);
301
383
  }
302
384
  catch (e) {
303
385
  logger_1.default.error('rabbit: assertQueue error', { queueName, options, error: e });
@@ -306,17 +388,29 @@ class RabbitMq {
306
388
  const channel = await this.assertChannel({ force: true });
307
389
  await this.deleteQueue(queueName);
308
390
  debug('retrying assertQueue->channel.addSetup', { queueName });
309
- await channel.addSetup((setupChannel) => setupChannel.assertQueue(queueName, options));
391
+ await channel.addSetup((setupChannel) => setupChannel.assertQueue(queueName, localeOptions));
310
392
  debug('retrying assertQueue->channel.assertQueue', { queueName });
311
- queue = await channel.assertQueue(queueName, options);
393
+ queue = await channel.assertQueue(queueName, localeOptions);
312
394
  }
313
395
  else {
314
396
  throw e;
315
397
  }
316
398
  }
317
- this.queues[queueName] = queueName;
399
+ this.queues[queueName] = queue;
318
400
  return queue;
319
401
  }
402
+ // TODO: [QUORUM-PHASE-3] Can be deleted after deleting the old consumers and publishers because all the queues are created us quorum queues
403
+ static shouldUseQuorum(queueName) {
404
+ const envQuorumQueuesWhitelist = process.env.QUORUM_QUEUES_WHITELIST;
405
+ if (envQuorumQueuesWhitelist === '*') {
406
+ return true;
407
+ }
408
+ if (envQuorumQueuesWhitelist) {
409
+ const whitelist = envQuorumQueuesWhitelist.split(',');
410
+ return whitelist.includes(queueName);
411
+ }
412
+ return false;
413
+ }
320
414
  async assertQueue(queueName, options) {
321
415
  RabbitMq.validateName('queue', queueName);
322
416
  if (this.queues[queueName]) {
@@ -340,9 +434,23 @@ class RabbitMq {
340
434
  });
341
435
  }
342
436
  }
437
+ // Used by the microservices to consume messages from the queue
343
438
  async consume(queue, callback, options) {
439
+ // TODO: [QUORUM-PHASE-3] Use only the implementation of consumeNew and delete consumeNew and consumeOld
440
+ if (options?.isQuorumQueue !== false) {
441
+ await this.assertVHost();
442
+ await this.consumeNew(queue, callback, options);
443
+ }
444
+ await this.consumeOld(queue, callback, options);
445
+ }
446
+ // TODO: [QUORUM-PHASE-3] Delete consumeNew we do not use it anymore
447
+ async consumeNew(queue, callback, options) {
344
448
  await this.consumeFromRabbit(queue, callback, options);
345
449
  }
450
+ // TODO: [QUORUM-PHASE-3] Delete consumeOld we do not use it anymore
451
+ async consumeOld(queue, callback, options) {
452
+ await this.consumeFromRabbitOld(queue, callback, options);
453
+ }
346
454
  async lockRedisIfNeeded(msg, options) {
347
455
  const { properties: { headers } } = msg;
348
456
  const timestamp = headers?.creationTimestamp;
@@ -365,20 +473,22 @@ class RabbitMq {
365
473
  const { limit, deadMessageTtl, useConsumeWithLock, lockTimeout, auditContext, enableRabbitTrace, } = optionsWithDefaults;
366
474
  if (useConsumeWithLock) {
367
475
  if (!this.redisLock) {
368
- throw new Error('Usage of consumeWithLock requires RedisInstance');
476
+ throw new rabbitError_1.default('Usage of consumeWithLock requires RedisInstance');
369
477
  }
370
478
  logger_1.default.info(`rabbit: Consuming with lock from queue ${queue} with lockTimeout: ${lockTimeout}ms`);
371
479
  }
372
- const channel = await this.getNewChannel({ name: `${this.podId}_queue_${queue}` });
480
+ const channel = await this.getNewChannel({});
373
481
  return channel.addSetup(async (confirmChannel) => {
374
- await confirmChannel.assertQueue(queue, options ? { messageTtl: options.messageTtl } : undefined);
375
- await confirmChannel.prefetch(limit, true);
482
+ logger_1.default.info(`rabbit: channel.addSetup before ${queue} assertQueueOld`);
483
+ const q = await this.assertQueue(queue, optionsWithDefaults);
484
+ await confirmChannel.prefetch(limit, false);
376
485
  const { consumerTag } = await confirmChannel.consume(queue, async (msg) => {
377
486
  if (!msg) {
378
487
  return null;
379
488
  }
380
- const traceId = msg.properties.headers[consts_1.TRACING_HEADER];
381
- const userId = msg.properties.headers[consts_1.USER_TRACING_HEADER];
489
+ const traceId = msg.properties.headers?.[consts_1.TRACING_HEADER];
490
+ const userId = msg.properties.headers?.[consts_1.USER_TRACING_HEADER];
491
+ const automationId = msg.properties.headers?.[consts_1.AUTOMATION_ID_HEADER];
382
492
  const parsedMessage = RabbitMq.parseMsg(msg);
383
493
  const releaseLock = await this.lockRedisIfNeeded(parsedMessage, optionsWithDefaults);
384
494
  const trace = (0, zehut_1.newTrace)(zehut_1.traceTypes.RABBIT);
@@ -403,7 +513,10 @@ class RabbitMq {
403
513
  outbreakTrace?.context.set(consts_1.TRACING_HEADER, traceId);
404
514
  }
405
515
  if (auditContext) {
406
- await auditContext(queue);
516
+ await auditContext(queue, {
517
+ userId,
518
+ automationId,
519
+ });
407
520
  }
408
521
  const shouldConsume = await this.shouldConsumeMessageByTimestamp(parsedMessage);
409
522
  if (!shouldConsume) {
@@ -443,64 +556,94 @@ class RabbitMq {
443
556
  }
444
557
  });
445
558
  }
559
+ // Used by the microservices to consume messages from the exchange
446
560
  async consumeFromExchange(queue, exchange, callback, options) {
447
561
  const optionsWithDefaults = { ...consts_1.DEFAULT_OPTIONS, ...options };
448
562
  RabbitMq.validateName('exchange', exchange);
449
563
  RabbitMq.validateName('queue', queue);
450
564
  const { limit, deadMessageTtl } = optionsWithDefaults;
451
- await this.saveConsumer(queue, callback, options);
452
- const channel = await this.getNewChannel({ name: `${this.podId}_exchange_${exchange}_queue_${queue}` });
453
- return channel.addSetup(async (c) => {
565
+ // TODO: [QUORUM-PHASE-3] Delete the if statement after all the queues are created as quorum queues
566
+ if (options?.isQuorumQueue !== false) {
567
+ await this.assertVHost();
568
+ await this.saveConsumer(queue, callback, options);
569
+ const channel = await this.getNewChannel({ name: `consume-exchange-${exchange}-queue-${queue}` });
570
+ await channel.addSetup(async (c) => {
571
+ const assertExchange = await (0, utils_1.assertExchangeFanout)(c, exchange);
572
+ await c.assertQueue(queue);
573
+ this.exchanges[exchange] = assertExchange;
574
+ await c.prefetch(limit, false);
575
+ return Promise.all([
576
+ c.bindQueue(queue, exchange, ''),
577
+ this.consumeNew(queue, callback, options),
578
+ ]);
579
+ });
580
+ }
581
+ // TODO: [QUORUM-PHASE-3] Delete the old implementation
582
+ await this.saveConsumerOld(queue, callback, options);
583
+ const channelOld = await this.getNewChannelOld({ name: `consume-exchange-${exchange}-queue-${queue}-old` });
584
+ await channelOld.addSetup(async (c) => {
454
585
  const assertExchange = await (0, utils_1.assertExchangeFanout)(c, exchange);
455
- await c.assertQueue(queue);
456
- this.exchanges[exchange] = assertExchange;
457
- await c.prefetch(limit, true);
586
+ // await c.assertQueue(queue);
587
+ this.oldExchanges[exchange] = assertExchange;
588
+ await c.prefetch(limit, false);
458
589
  return Promise.all([
459
- c.bindQueue(queue, exchange, ''),
460
- this.consume(queue, callback, options),
590
+ // c.bindQueue(queue, exchange, ''),
591
+ this.consumeOld(queue, callback, options),
461
592
  ]);
462
593
  });
463
594
  }
595
+ // Used by the microservices to publish messages to the exchange
596
+ // TODO: [QUORUM-PHASE-2] Change the implementation to the new one
464
597
  async publish(exchange, content, customHeaders) {
465
598
  return (0, utils_1.wrapSetImmediate)(async () => {
466
599
  RabbitMq.validateName('exchange', exchange);
467
- const channel = await this.assertChannel();
468
- await this.assertExchange(exchange);
600
+ const channel = await this.assertChannelOld();
601
+ await this.assertExchangeOld(exchange);
469
602
  await channel.publish(exchange, '', Buffer.from(JSON.stringify(content)), RabbitMq.getPublishOptions(customHeaders));
470
603
  });
471
604
  }
472
- async sendToQueue(queue, content, options, customHeaders, isBlocking) {
473
- const callback = async () => {
474
- try {
475
- RabbitMq.validateName('queue', queue);
476
- await this.assertChannel();
477
- await this.assertQueue(queue, options);
478
- const res = await this.channel?.sendToQueue(queue, Buffer.from(JSON.stringify(content)), RabbitMq.getPublishOptions(customHeaders));
479
- debug(`rabbit: sending to queue ${queue}`, { res });
480
- return res;
481
- }
482
- catch (e) {
483
- logger_1.default.error(`rabbit: failed to send to queue ${queue}`, { e });
484
- throw e;
485
- }
486
- };
487
- if (isBlocking) {
488
- return callback();
605
+ // Used by the microservices to send messages to the queue
606
+ // TODO: [QUORUM-PHASE-2] Change the implementation to the new one
607
+ async sendToQueue(queue, content, options, customHeaders) {
608
+ try {
609
+ await this.assertChannelOld();
610
+ }
611
+ catch (e) {
612
+ logger_1.default.error(`rabbit sendToQueue: failed to send assert channel when sending to queue ${queue}`, { e });
613
+ throw e;
614
+ }
615
+ try {
616
+ RabbitMq.validateName('queue', queue);
617
+ await this.assertQueueOld(queue, options);
618
+ }
619
+ catch (e) {
620
+ logger_1.default.error(`rabbit sendToQueue: failed to assert queue ${queue}`, { e });
621
+ throw e;
622
+ }
623
+ try {
624
+ const res = await this.oldChannel?.sendToQueue(queue, Buffer.from(JSON.stringify(content)), RabbitMq.getPublishOptions(customHeaders));
625
+ debug(`rabbit: sending to queue ${queue}`, { res });
626
+ return res;
627
+ }
628
+ catch (e) {
629
+ const isConnected = await this.isConnected();
630
+ logger_1.default.error(`rabbit sendToQueue: failed to send to queue ${queue}, isConnected: ${isConnected}`, { e });
631
+ throw e;
489
632
  }
490
- return (0, utils_1.wrapSetImmediate)(callback);
491
633
  }
634
+ // TODO: [QUORUM-PHASE-2] After changing the publish from old to new change from the old to the new implementation
492
635
  async isConnected() {
493
- const connection = await this.getConnection();
636
+ const connection = await this.getConnectionOld();
494
637
  const isConnected = connection.isConnected();
495
638
  if (!isConnected) {
496
639
  logger_1.default.error('rabbit: isConnected - false');
497
640
  return false;
498
641
  }
499
- const channel = await this.assertChannel();
642
+ const channel = await this.assertChannelOld();
500
643
  try {
501
644
  await Promise.all([
502
645
  channel.waitForConnect(),
503
- ...this.consumers.map((c) => channel.checkQueue(c.queue)),
646
+ ...this.oldConsumers.map((c) => channel.checkQueue(c.queue)),
504
647
  ]);
505
648
  }
506
649
  catch (e) {
@@ -511,12 +654,15 @@ class RabbitMq {
511
654
  return true;
512
655
  }
513
656
  async gracefulShutdown(signal) {
514
- const tagsNumber = this.consumersTags.length;
657
+ // TODO: [QUORUM-PHASE-2] After changing the publish from old to new change from the old to the new implementation
658
+ const tagsNumber = this.consumersTags.length + this.oldConsumersTags.length;
515
659
  logger_1.default.info(`rabbit: [gracefully-shutdown] received ${signal}! canceling #${tagsNumber} tags...`);
516
660
  const cancelTagPromises = this.consumersTags.map(([channel, tag]) => channel.cancel(tag));
661
+ const cancelTagPromisesOld = this.oldConsumersTags.map(([channel, tag]) => channel.cancel(tag));
517
662
  // Clean the array to avoid race
518
663
  this.consumersTags = [];
519
- const results = await Promise.allSettled(cancelTagPromises);
664
+ this.oldConsumersTags = [];
665
+ const results = await Promise.allSettled([...cancelTagPromises, ...cancelTagPromisesOld]);
520
666
  const rejected = results.filter((p) => p.status === 'rejected');
521
667
  if (rejected.length > 0) {
522
668
  logger_1.default.warn(`rabbit: [gracefully-shutdown] #${rejected.length}/${tagsNumber} tags failed to cancel: ${rejected}`);
@@ -525,5 +671,306 @@ class RabbitMq {
525
671
  logger_1.default.info('rabbit: [gracefully-shutdown] all tags successfully canceled.');
526
672
  }
527
673
  }
674
+ async consumeFromRabbitOld(queue, callback, options) {
675
+ const optionsWithDefaults = { ...consts_1.DEFAULT_OPTIONS, ...options };
676
+ RabbitMq.validateName('queue', queue);
677
+ this.saveConsumerOld(queue, callback, options);
678
+ const uniqueId = (0, node_crypto_1.randomUUID)();
679
+ const { limit, deadMessageTtl, useConsumeWithLock, lockTimeout, auditContext, enableRabbitTrace, } = optionsWithDefaults;
680
+ if (useConsumeWithLock) {
681
+ if (!this.redisLock) {
682
+ throw new rabbitError_1.default('Usage of consumeWithLock requires RedisInstance');
683
+ }
684
+ logger_1.default.info(`rabbit: Consuming with lock from queue ${queue} with lockTimeout: ${lockTimeout}ms`);
685
+ }
686
+ const channel = await this.getNewChannelOld({});
687
+ return channel.addSetup(async (confirmChannel) => {
688
+ const q = await this.assertQueueOld(queue, optionsWithDefaults);
689
+ await confirmChannel.prefetch(limit, false);
690
+ const { consumerTag } = await confirmChannel.consume(queue, async (msg) => {
691
+ if (!msg) {
692
+ return null;
693
+ }
694
+ const traceId = msg.properties.headers?.[consts_1.TRACING_HEADER];
695
+ const userId = msg.properties.headers?.[consts_1.USER_TRACING_HEADER];
696
+ const automationId = msg.properties.headers?.[consts_1.AUTOMATION_ID_HEADER];
697
+ const parsedMessage = RabbitMq.parseMsg(msg);
698
+ const releaseLock = await this.lockRedisIfNeeded(parsedMessage, optionsWithDefaults);
699
+ const trace = (0, zehut_1.newTrace)(zehut_1.traceTypes.RABBIT);
700
+ // setting also outbreak trace as part of legacy code
701
+ const outbreakTrace = zehut_1.outbreak.newTrace(zehut_1.traceTypes.RABBIT);
702
+ // enableRabbitTrace is a flag to protect from different flows that doesn't work with permission
703
+ // and we don't want to fail the flow because of it
704
+ if (userId && enableRabbitTrace) {
705
+ try {
706
+ await Promise.all([
707
+ (0, zehut_1.createOrSetRabbitTrace)(trace, userId),
708
+ (0, zehut_1.createOrSetRabbitTrace)(outbreakTrace, userId),
709
+ ]);
710
+ }
711
+ catch (e) {
712
+ logger_1.default.error('rabbit: failed to setRabbitTrace', { userId, e });
713
+ return this.nack(confirmChannel, queue, optionsWithDefaults, { messageTtl: deadMessageTtl }, msg, releaseLock)(msg);
714
+ }
715
+ }
716
+ if (traceId) {
717
+ trace?.context?.set(consts_1.TRACING_HEADER, traceId);
718
+ outbreakTrace?.context.set(consts_1.TRACING_HEADER, traceId);
719
+ }
720
+ if (auditContext) {
721
+ await auditContext(queue, {
722
+ userId,
723
+ automationId,
724
+ });
725
+ }
726
+ const shouldConsume = await this.shouldConsumeMessageByTimestamp(parsedMessage);
727
+ if (!shouldConsume) {
728
+ await this.unlockRedisIfNeeded(releaseLock);
729
+ return this.ack(confirmChannel, msg)(msg);
730
+ }
731
+ let messageAcked = false;
732
+ // setting the localAck function to be used in the callback
733
+ const localAck = async () => {
734
+ if (messageAcked) {
735
+ return;
736
+ }
737
+ messageAcked = true;
738
+ return this.ack(confirmChannel, msg, true, releaseLock)(msg);
739
+ };
740
+ const localNack = async (_, nackOptions = {}) => {
741
+ if (messageAcked) {
742
+ return;
743
+ }
744
+ debug('rabbit localNack', { messageAcked, uniqueId, deliveryTag: msg.fields.deliveryTag });
745
+ messageAcked = true;
746
+ return this.nack(confirmChannel, queue, optionsWithDefaults, { messageTtl: deadMessageTtl }, msg, releaseLock)(msg, nackOptions);
747
+ };
748
+ try {
749
+ await callback(parsedMessage, localAck, localNack);
750
+ }
751
+ catch (e) {
752
+ await localNack(msg);
753
+ }
754
+ }, types_1.CONSUMER_DEFAULT_OPTIONS);
755
+ if (!consumerTag) {
756
+ logger_1.default.error(`rabbit: failed to consume from queue ${queue}`);
757
+ }
758
+ else {
759
+ logger_1.default.info(`rabbit: adding tag ${consumerTag} to the array.`);
760
+ this.oldConsumersTags.push([confirmChannel, consumerTag]);
761
+ }
762
+ });
763
+ }
764
+ // TODO: [QUORUM-PHASE-3] Delete all the function under this line (getNewChannelOld, getConnectionOld, assertQueueOld, setupQueueOld, saveConsumerOld)
765
+ async getNewChannelOld({ name = (0, utils_1.rand)().toString(), onClose = null, options = {} } = {}) {
766
+ let connection;
767
+ try {
768
+ connection = await this.getConnectionOld();
769
+ }
770
+ catch (e) {
771
+ logger_1.default.error(`rabbit: error on get connection for new channel ${name} `, { e });
772
+ throw e;
773
+ }
774
+ const channel = connection.createChannel({ ...options });
775
+ (0, events_1.once)(channel, 'close').then((args) => {
776
+ logger_1.default.error(`rabbit: channel ${name} closed`);
777
+ onClose?.(args);
778
+ });
779
+ try {
780
+ await (0, events_1.once)(channel, 'connect');
781
+ debug(`rabbit: channel ${name} CONNECTED`);
782
+ return channel;
783
+ }
784
+ catch (err) {
785
+ logger_1.default.error(`rabbit: channel error ${name} error`, { err });
786
+ throw err;
787
+ }
788
+ }
789
+ async getConnectionOld() {
790
+ return new Promise(async (resolve, reject) => {
791
+ if (this.oldBlockReconnect) {
792
+ debug('rabbit: block reconnect');
793
+ // eslint-disable-next-line @typescript-eslint/ban-ts-comment
794
+ // @ts-ignore
795
+ return resolve();
796
+ }
797
+ if (this.oldConnection !== null) {
798
+ if (this.options?.disableReconnect || this.oldConnection?.isConnected()) {
799
+ debug('rabbit: connection - is connected');
800
+ // eslint-disable-next-line @typescript-eslint/ban-ts-comment
801
+ // @ts-ignore
802
+ return resolve(this.oldConnection);
803
+ }
804
+ debug('rabbit: connection - reconnecting');
805
+ }
806
+ if (this.oldCreatingConnection) {
807
+ debug('rabbit: creating connection emi');
808
+ this.oldEm.once(consts_1.CONNECTION_CREATED_CONST, resolve);
809
+ this.oldEm.once(consts_1.CONNECTION_FAILED_CONST, reject);
810
+ return;
811
+ }
812
+ this.oldCreatingConnection = true;
813
+ let isResolved = false;
814
+ // It is import to use it as a function and not as a variable
815
+ // because of k8s changes the env variables
816
+ // and we want to use the new values
817
+ const findServers = () => {
818
+ const userName = process.env.RABBITMQ_USERNAME || 'guest';
819
+ const password = process.env.RABBITMQ_PASSWORD || 'guest';
820
+ const host = this.options?.rabbitHost || process.env.RABBITMQ_SERVICE_HOST || 'localhost';
821
+ debug('rabbit: creating connection', { host, userName, HEARTBEAT });
822
+ return [`amqp://${userName}:${password}@${host}?heartbeat=${HEARTBEAT}`];
823
+ };
824
+ const defaultUrls = findServers();
825
+ const connection = await (0, amqp_connection_manager_1.connect)(defaultUrls, {
826
+ findServers,
827
+ });
828
+ this.oldConnection = connection;
829
+ this.oldConnection.on('error', (err) => {
830
+ logger_1.default.error('rabbit: connection error', { err });
831
+ if (!isResolved) {
832
+ isResolved = true;
833
+ reject(err);
834
+ this.oldEm.emit(consts_1.CONNECTION_FAILED_CONST, err);
835
+ }
836
+ });
837
+ this.oldConnection.on('connectFailed', (err) => {
838
+ this.oldConsumersTags = [];
839
+ logger_1.default.error('rabbit: connection connectFailed', { err });
840
+ if (!isResolved) {
841
+ isResolved = true;
842
+ reject(err);
843
+ this.oldEm.emit(consts_1.CONNECTION_FAILED_CONST, err);
844
+ }
845
+ });
846
+ this.oldConnection.on('disconnect', ({ err }) => {
847
+ this.oldConsumersTags = [];
848
+ debug('rabbit: connection closed');
849
+ if (this.options?.disableReconnect) {
850
+ logger_1.default.error(`${this.DISCONNECT_MSG}${err && ` - ${err}`}`);
851
+ this.oldBlockReconnect = true;
852
+ }
853
+ else {
854
+ logger_1.default.error(`${this.RECONNECT_MSG}${err && ` - ${err}`}`);
855
+ }
856
+ });
857
+ this.oldConnection.once('connect', async () => {
858
+ debug('rabbit: connection established');
859
+ this.oldCreatingConnection = false;
860
+ this.oldEm.emit(consts_1.CONNECTION_CREATED_CONST, connection);
861
+ isResolved = true;
862
+ resolve(connection);
863
+ });
864
+ });
865
+ }
866
+ saveConsumerOld(queue, callback, options) {
867
+ const isConsumerExist = this.oldConsumers.some((consumer) => consumer.queue === queue);
868
+ if (!isConsumerExist) {
869
+ logger_1.default.info(`rabbit: consumer: ${queue} saved in consumer array`);
870
+ this.oldConsumers.push({
871
+ queue,
872
+ callback,
873
+ options,
874
+ });
875
+ }
876
+ }
877
+ async assertQueueOld(queueName, options) {
878
+ RabbitMq.validateName('queue', queueName);
879
+ if (this.oldQueues[queueName]) {
880
+ delete this.oldQueueSetupPromises[queueName];
881
+ return this.oldQueues[queueName];
882
+ }
883
+ if (this.oldQueueSetupPromises[queueName]) {
884
+ return this.oldQueueSetupPromises[queueName];
885
+ }
886
+ this.oldQueueSetupPromises[queueName] = this.setupQueueOld(queueName, options);
887
+ return this.oldQueueSetupPromises[queueName];
888
+ }
889
+ async setupQueueOld(queueName, options) {
890
+ // let queue: Replies.AssertQueue;
891
+ const shouldUseQuorum = RabbitMq.shouldUseQuorum(queueName);
892
+ const localeOptions = {
893
+ ...options,
894
+ durable: true,
895
+ arguments: {
896
+ ...options?.arguments,
897
+ 'x-consumer-timeout': 1000 * 60 * 60 * 24,
898
+ 'x-queue-type': shouldUseQuorum ? 'quorum' : 'classic',
899
+ },
900
+ };
901
+ try {
902
+ const channel = await this.assertChannelOld();
903
+ debug('assertQueue->channel.addSetup', { queueName });
904
+ // await channel.addSetup((setupChannel: ConfirmChannel) => setupChannel.assertQueue(queueName, localeOptions));
905
+ debug('assertQueue->channel.assertQueue', { queueName });
906
+ // queue = await channel.assertQueue(queueName, localeOptions);
907
+ }
908
+ catch (e) {
909
+ logger_1.default.error('rabbit: assertQueue error', { queueName, options, error: e });
910
+ if (!this.options?.dontRetryAssert) {
911
+ debug('retrying assertQueue', { queueName });
912
+ const channel = await this.assertChannelOld({ force: true });
913
+ await this.deleteQueueOld(queueName);
914
+ debug('retrying assertQueue->channel.addSetup', { queueName });
915
+ await channel.addSetup((setupChannel) => setupChannel.assertQueue(queueName, localeOptions));
916
+ debug('retrying assertQueue->channel.assertQueue', { queueName });
917
+ // queue = await channel.assertQueue(queueName, localeOptions);
918
+ }
919
+ else {
920
+ throw e;
921
+ }
922
+ }
923
+ const queue = {
924
+ queue: 'bla bla queue',
925
+ messageCount: 0,
926
+ consumerCount: 2,
927
+ };
928
+ this.oldQueues[queueName] = queue;
929
+ return queue;
930
+ }
931
+ async assertChannelOld({ force = false } = {}) {
932
+ if (!this.oldPublishChannelSetupPromise) {
933
+ this.oldPublishChannelSetupPromise = new Promise(async (resolve, reject) => {
934
+ if (this.oldChannel && !force) {
935
+ return resolve(this.oldChannel);
936
+ }
937
+ try {
938
+ const channel = await this.getNewChannelOld({});
939
+ channel.on('error', (err) => {
940
+ logger_1.default.error('rabbit: channel error', { err });
941
+ });
942
+ this.oldChannel = channel;
943
+ resolve(channel);
944
+ }
945
+ catch (e) {
946
+ reject(e);
947
+ }
948
+ });
949
+ }
950
+ return this.oldPublishChannelSetupPromise;
951
+ }
952
+ async deleteQueueOld(queue) {
953
+ RabbitMq.validateName('queue', queue);
954
+ const channel = await this.assertChannelOld();
955
+ logger_1.default.info('rabbit: deleting queue', { queue });
956
+ const deleteQueueRes = await channel.deleteQueue(queue);
957
+ debug('queue deleted', deleteQueueRes);
958
+ return deleteQueueRes;
959
+ }
960
+ async assertExchangeOld(exchangeName, options) {
961
+ const channel = await this.assertChannelOld();
962
+ if (this.oldExchanges[exchangeName]) {
963
+ delete this.oldAssertExchangePromises[exchangeName];
964
+ return this.oldExchanges[exchangeName];
965
+ }
966
+ if (this.oldAssertExchangePromises[exchangeName]) {
967
+ return this.oldAssertExchangePromises[exchangeName];
968
+ }
969
+ this.oldAssertExchangePromises[exchangeName] = (0, utils_1.assertExchangeFanout)(channel, exchangeName);
970
+ this.oldExchanges[exchangeName] = await this.oldAssertExchangePromises[exchangeName];
971
+ return this.oldExchanges[exchangeName];
972
+ }
528
973
  }
529
974
  exports.default = RabbitMq;
975
+ var celery_1 = require("./lib/celery");
976
+ Object.defineProperty(exports, "sendCeleryTaskViaHttp", { enumerable: true, get: function () { return celery_1.sendCeleryTaskViaHttp; } });