@autofleet/rabbit 3.3.0 → 3.3.22-connection-test-beta
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.nvmrc +1 -1
- package/dist/index.d.ts +0 -27
- package/dist/index.js +94 -437
- package/dist/lib/types.d.ts +0 -1
- package/package.json +3 -6
- package/src/index.ts +109 -515
- package/src/lib/types.ts +0 -2
- package/dist/lib/celery.d.ts +0 -9
- package/dist/lib/celery.js +0 -54
- package/src/lib/celery.ts +0 -89
package/src/index.ts
CHANGED
|
@@ -100,6 +100,44 @@ type AfConsumer = {
|
|
|
100
100
|
}
|
|
101
101
|
|
|
102
102
|
const HEARTBEAT = '60';
|
|
103
|
+
const withTimeout = async<T>(promise: Promise<T>, timeoutMs: number): Promise<T> => new Promise((resolve, reject) => {
|
|
104
|
+
const timer = setTimeout(() => {
|
|
105
|
+
reject(new Error(`Promise timed out after ${timeoutMs} ms`));
|
|
106
|
+
}, timeoutMs);
|
|
107
|
+
|
|
108
|
+
promise
|
|
109
|
+
.then((value) => {
|
|
110
|
+
clearTimeout(timer);
|
|
111
|
+
resolve(value);
|
|
112
|
+
})
|
|
113
|
+
.catch((err) => {
|
|
114
|
+
clearTimeout(timer);
|
|
115
|
+
reject(err);
|
|
116
|
+
});
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
const withRetry = async <T>(
|
|
120
|
+
operation: () => Promise<T>,
|
|
121
|
+
retries: number,
|
|
122
|
+
timeoutMs: number,
|
|
123
|
+
): Promise<T> => new Promise(async (resolve, reject) => {
|
|
124
|
+
let lastError: any;
|
|
125
|
+
|
|
126
|
+
for (let attempt = 1; attempt <= retries; attempt += 1) {
|
|
127
|
+
try {
|
|
128
|
+
// eslint-disable-next-line no-await-in-loop
|
|
129
|
+
const result = await withTimeout(operation(), timeoutMs);
|
|
130
|
+
return resolve(result);
|
|
131
|
+
} catch (err) {
|
|
132
|
+
lastError = err;
|
|
133
|
+
logger.error(`Attempt ${attempt} failed: ${err}`);
|
|
134
|
+
if (attempt < retries) {
|
|
135
|
+
logger.info(`Retrying (${attempt}/${retries})...`);
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
reject(lastError);
|
|
140
|
+
});
|
|
103
141
|
|
|
104
142
|
class RabbitMq implements IAfRabbitMq {
|
|
105
143
|
static parseMsg(msg: any) : any {
|
|
@@ -174,36 +212,7 @@ class RabbitMq implements IAfRabbitMq {
|
|
|
174
212
|
|
|
175
213
|
private consumers: Array<AfConsumer> = [];
|
|
176
214
|
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
private vhost = 'quorum-vhost';
|
|
180
|
-
|
|
181
|
-
// TODO:[QUORUM-PHASE-3] Delete the old properties that we use for the old consumers and publishers
|
|
182
|
-
oldChannel: ChannelWrapper | null;
|
|
183
|
-
|
|
184
|
-
oldPublishChannelSetupPromise: Promise<ChannelWrapper> | null;
|
|
185
|
-
|
|
186
|
-
oldBlockReconnect: boolean | null | undefined
|
|
187
|
-
|
|
188
|
-
oldConnection: AmqpConnectionManager | null | undefined
|
|
189
|
-
|
|
190
|
-
oldEm: EventEmitter;
|
|
191
|
-
|
|
192
|
-
oldCreatingConnection: boolean;
|
|
193
|
-
|
|
194
|
-
oldExchanges: ExchangesCache;
|
|
195
|
-
|
|
196
|
-
oldQueues: QueuesCache;
|
|
197
|
-
|
|
198
|
-
oldQueueSetupPromises: QueueSetupPromisesDictionary;
|
|
199
|
-
|
|
200
|
-
oldAssertExchangePromises: AssertExchangePromisesDictionary;
|
|
201
|
-
|
|
202
|
-
oldConsumersTags: Array<[ConfirmChannel, string]>;
|
|
203
|
-
|
|
204
|
-
private oldConsumers: Array<AfConsumer> = [];
|
|
205
|
-
|
|
206
|
-
constructor(options: AfRabbitOptions = {}, redisConfig?: RedisConfig) {
|
|
215
|
+
constructor(options?: AfRabbitOptions, redisConfig?: RedisConfig) {
|
|
207
216
|
this.em = new EventEmitter();
|
|
208
217
|
this.channel = null;
|
|
209
218
|
this.publishChannelSetupPromise = null;
|
|
@@ -215,7 +224,6 @@ class RabbitMq implements IAfRabbitMq {
|
|
|
215
224
|
this.assertExchangePromises = {};
|
|
216
225
|
this.consumers = [];
|
|
217
226
|
this.options = options;
|
|
218
|
-
|
|
219
227
|
this.redisClient = redisConfig && getRedisInstance(redisConfig);
|
|
220
228
|
if (this.redisClient) {
|
|
221
229
|
this.redisLock = promisify(RedisLock(this.redisClient)) as RedisLockType;
|
|
@@ -230,74 +238,8 @@ class RabbitMq implements IAfRabbitMq {
|
|
|
230
238
|
await this.gracefulShutdown('SIGINT');
|
|
231
239
|
});
|
|
232
240
|
}
|
|
233
|
-
|
|
234
|
-
// TODO: [QUORUM-PHASE-3] Delete the old properties that we use for the old consumers and publishers
|
|
235
|
-
this.oldEm = new EventEmitter();
|
|
236
|
-
this.oldChannel = null;
|
|
237
|
-
this.oldPublishChannelSetupPromise = null;
|
|
238
|
-
this.oldConnection = null;
|
|
239
|
-
this.oldCreatingConnection = false;
|
|
240
|
-
this.oldExchanges = {};
|
|
241
|
-
this.oldQueues = {};
|
|
242
|
-
this.oldQueueSetupPromises = {};
|
|
243
|
-
this.oldAssertExchangePromises = {};
|
|
244
|
-
this.oldConsumers = [];
|
|
245
|
-
this.oldConsumersTags = [];
|
|
246
241
|
}
|
|
247
242
|
|
|
248
|
-
private assertVHost = async () => {
|
|
249
|
-
if (this.doesVHostExist) {
|
|
250
|
-
return;
|
|
251
|
-
}
|
|
252
|
-
|
|
253
|
-
const username = process.env.RABBITMQ_USERNAME || 'guest';
|
|
254
|
-
const password = process.env.RABBITMQ_PASSWORD || 'guest';
|
|
255
|
-
const credentials = Buffer.from(`${username}:${password}`).toString('base64');
|
|
256
|
-
const headers = {
|
|
257
|
-
Authorization: `Basic ${credentials}`,
|
|
258
|
-
'Content-Type': 'application/json',
|
|
259
|
-
};
|
|
260
|
-
|
|
261
|
-
const rabbitHost = `http://${(this.options?.rabbitHost || process.env.RABBITMQ_SERVICE_HOST || 'localhost').split(':')[0]}:15672`;
|
|
262
|
-
|
|
263
|
-
const url = `${rabbitHost}/api/vhosts/${encodeURIComponent(this.vhost)}`;
|
|
264
|
-
|
|
265
|
-
try {
|
|
266
|
-
const response = await fetch(url, {
|
|
267
|
-
method: 'GET',
|
|
268
|
-
headers,
|
|
269
|
-
});
|
|
270
|
-
|
|
271
|
-
if (response.status === 200) {
|
|
272
|
-
this.doesVHostExist = true;
|
|
273
|
-
logger.info('Vhost exists', { vhost: this.vhost });
|
|
274
|
-
return;
|
|
275
|
-
}
|
|
276
|
-
|
|
277
|
-
if (response.status !== 404) {
|
|
278
|
-
logger.error('Failed to check vhost', { response });
|
|
279
|
-
throw new RabbitError('Failed to check vhost');
|
|
280
|
-
}
|
|
281
|
-
|
|
282
|
-
const createResponse = await fetch(url, {
|
|
283
|
-
method: 'PUT',
|
|
284
|
-
headers,
|
|
285
|
-
body: JSON.stringify({ default_queue_type: 'quorum' }),
|
|
286
|
-
});
|
|
287
|
-
|
|
288
|
-
if (!createResponse.ok) {
|
|
289
|
-
logger.error('Failed to create vhost', { response: createResponse });
|
|
290
|
-
throw new RabbitError('Failed to create vhost');
|
|
291
|
-
}
|
|
292
|
-
|
|
293
|
-
this.doesVHostExist = true;
|
|
294
|
-
logger.info('Vhost created', { vhost: this.vhost });
|
|
295
|
-
} catch (error) {
|
|
296
|
-
logger.error('Failed to check or create vhost', { error });
|
|
297
|
-
throw error;
|
|
298
|
-
}
|
|
299
|
-
};
|
|
300
|
-
|
|
301
243
|
private shouldConsumeMessageByTimestamp = async (msg: ConsumeMessageOrNull) => {
|
|
302
244
|
if (msg) {
|
|
303
245
|
const { properties: { headers } } = msg;
|
|
@@ -408,7 +350,8 @@ class RabbitMq implements IAfRabbitMq {
|
|
|
408
350
|
const host = this.options?.rabbitHost || process.env.RABBITMQ_SERVICE_HOST || 'localhost';
|
|
409
351
|
|
|
410
352
|
debug('rabbit: creating connection', { host, userName, HEARTBEAT });
|
|
411
|
-
|
|
353
|
+
|
|
354
|
+
return [`amqp://${userName}:${password}@${host}?heartbeat=${HEARTBEAT}`];
|
|
412
355
|
};
|
|
413
356
|
|
|
414
357
|
const defaultUrls = findServers();
|
|
@@ -428,7 +371,7 @@ class RabbitMq implements IAfRabbitMq {
|
|
|
428
371
|
|
|
429
372
|
this.connection.on('connectFailed', (err) => {
|
|
430
373
|
this.consumersTags = [];
|
|
431
|
-
logger.error('rabbit: connection connectFailed', { err
|
|
374
|
+
logger.error('rabbit: connection connectFailed', { err });
|
|
432
375
|
if (!isResolved) {
|
|
433
376
|
isResolved = true;
|
|
434
377
|
reject(err);
|
|
@@ -520,12 +463,11 @@ class RabbitMq implements IAfRabbitMq {
|
|
|
520
463
|
return this.exchanges[exchangeName];
|
|
521
464
|
}
|
|
522
465
|
|
|
523
|
-
// TODO: [QUORUM-PHASE-2] Change the implementation to the new one
|
|
524
466
|
async getQueueLength(queue: string) {
|
|
525
467
|
RabbitMq.validateName('queue', queue);
|
|
526
|
-
const {
|
|
468
|
+
const { channel } = this;
|
|
527
469
|
if (!channel) {
|
|
528
|
-
throw new
|
|
470
|
+
throw new Error('channel is not defined');
|
|
529
471
|
}
|
|
530
472
|
debug('rabbit: getting queue length', { queue, connected: this.connection?.isConnected() });
|
|
531
473
|
return channel?.checkQueue(queue);
|
|
@@ -540,30 +482,28 @@ class RabbitMq implements IAfRabbitMq {
|
|
|
540
482
|
return deleteQueueRes;
|
|
541
483
|
}
|
|
542
484
|
|
|
543
|
-
// TODO: [QUORUM-PHASE-2] Change the implementation to the new one
|
|
544
485
|
async bindQueue(queue: string, exchange: string) {
|
|
545
|
-
const channel: ChannelWrapper = await this.
|
|
486
|
+
const channel: ChannelWrapper = await this.assertChannel();
|
|
546
487
|
await channel.addSetup((setupChannel: ConfirmChannel) => setupChannel.bindQueue(queue, exchange, ''));
|
|
547
488
|
return channel.bindQueue(queue, exchange, '');
|
|
548
489
|
}
|
|
549
490
|
|
|
550
491
|
async setupQueue(queueName: string, options?: Options.AssertQueue): Promise<Replies.AssertQueue> {
|
|
551
492
|
let queue: Replies.AssertQueue;
|
|
493
|
+
const shouldUseQuorum = RabbitMq.shouldUseQuorum(queueName);
|
|
552
494
|
const localeOptions = {
|
|
553
495
|
...options,
|
|
554
496
|
durable: true,
|
|
555
497
|
arguments: {
|
|
556
498
|
...options?.arguments,
|
|
557
499
|
'x-consumer-timeout': 1000 * 60 * 60 * 24,
|
|
558
|
-
'x-queue-type': 'quorum',
|
|
500
|
+
'x-queue-type': shouldUseQuorum ? 'quorum' : 'classic',
|
|
559
501
|
},
|
|
560
502
|
};
|
|
561
503
|
try {
|
|
562
504
|
const channel: ChannelWrapper = await this.assertChannel();
|
|
563
505
|
debug('assertQueue->channel.addSetup', { queueName });
|
|
564
|
-
await channel.addSetup(
|
|
565
|
-
await setupChannel.assertQueue(queueName, localeOptions);
|
|
566
|
-
});
|
|
506
|
+
await channel.addSetup((setupChannel: ConfirmChannel) => setupChannel.assertQueue(queueName, localeOptions));
|
|
567
507
|
debug('assertQueue->channel.assertQueue', { queueName });
|
|
568
508
|
queue = await channel.assertQueue(queueName, localeOptions);
|
|
569
509
|
} catch (e) {
|
|
@@ -586,7 +526,6 @@ class RabbitMq implements IAfRabbitMq {
|
|
|
586
526
|
return queue;
|
|
587
527
|
}
|
|
588
528
|
|
|
589
|
-
// TODO: [QUORUM-PHASE-3] Can be deleted after deleting the old consumers and publishers because all the queues are created us quorum queues
|
|
590
529
|
static shouldUseQuorum(queueName: string): boolean {
|
|
591
530
|
const envQuorumQueuesWhitelist = process.env.QUORUM_QUEUES_WHITELIST;
|
|
592
531
|
|
|
@@ -629,27 +568,10 @@ class RabbitMq implements IAfRabbitMq {
|
|
|
629
568
|
}
|
|
630
569
|
}
|
|
631
570
|
|
|
632
|
-
// Used by the microservices to consume messages from the queue
|
|
633
571
|
async consume(queue: string, callback: CallbackFunction, options?: ConsumeOptions): Promise<any> {
|
|
634
|
-
// TODO: [QUORUM-PHASE-3] Use only the implementation of consumeNew and delete consumeNew and consumeOld
|
|
635
|
-
if (options?.isQuorumQueue !== false) {
|
|
636
|
-
await this.assertVHost();
|
|
637
|
-
await this.consumeNew(queue, callback, options);
|
|
638
|
-
}
|
|
639
|
-
|
|
640
|
-
await this.consumeOld(queue, callback, options);
|
|
641
|
-
}
|
|
642
|
-
|
|
643
|
-
// TODO: [QUORUM-PHASE-3] Delete consumeNew we do not use it anymore
|
|
644
|
-
async consumeNew(queue: string, callback: CallbackFunction, options?: ConsumeOptions): Promise<any> {
|
|
645
572
|
await this.consumeFromRabbit(queue, callback, options);
|
|
646
573
|
}
|
|
647
574
|
|
|
648
|
-
// TODO: [QUORUM-PHASE-3] Delete consumeOld we do not use it anymore
|
|
649
|
-
async consumeOld(queue: string, callback: CallbackFunction, options?: ConsumeOptions): Promise<any> {
|
|
650
|
-
await this.consumeFromRabbitOld(queue, callback, options);
|
|
651
|
-
}
|
|
652
|
-
|
|
653
575
|
private async lockRedisIfNeeded(msg: any, options: any) {
|
|
654
576
|
const { properties: { headers } } = msg;
|
|
655
577
|
const timestamp = headers?.creationTimestamp;
|
|
@@ -680,7 +602,7 @@ class RabbitMq implements IAfRabbitMq {
|
|
|
680
602
|
} = optionsWithDefaults;
|
|
681
603
|
if (useConsumeWithLock) {
|
|
682
604
|
if (!this.redisLock) {
|
|
683
|
-
throw new
|
|
605
|
+
throw new Error('Usage of consumeWithLock requires RedisInstance');
|
|
684
606
|
}
|
|
685
607
|
logger.info(`rabbit: Consuming with lock from queue ${queue} with lockTimeout: ${lockTimeout}ms`);
|
|
686
608
|
}
|
|
@@ -774,45 +696,22 @@ class RabbitMq implements IAfRabbitMq {
|
|
|
774
696
|
});
|
|
775
697
|
}
|
|
776
698
|
|
|
777
|
-
// Used by the microservices to consume messages from the exchange
|
|
778
699
|
async consumeFromExchange(queue: string, exchange: string, callback: CallbackFunction, options?: ConsumeOptions) : Promise<any> {
|
|
779
700
|
const optionsWithDefaults = { ...DEFAULT_OPTIONS, ...options };
|
|
780
701
|
RabbitMq.validateName('exchange', exchange);
|
|
781
702
|
RabbitMq.validateName('queue', queue);
|
|
782
703
|
const { limit, deadMessageTtl } = optionsWithDefaults;
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
await this.assertVHost();
|
|
786
|
-
await this.saveConsumer(queue, callback, options);
|
|
787
|
-
const channel: ChannelWrapper = await this.getNewChannel({ name: `consume-exchange-${exchange}-queue-${queue}` });
|
|
788
|
-
|
|
789
|
-
await channel.addSetup(async (c: ConfirmChannel) => {
|
|
790
|
-
const assertExchange = await assertExchangeFanout(c, exchange);
|
|
791
|
-
await c.assertQueue(queue);
|
|
792
|
-
this.exchanges[exchange] = assertExchange;
|
|
793
|
-
await c.prefetch(limit, false);
|
|
794
|
-
return Promise.all([
|
|
795
|
-
c.bindQueue(queue, exchange, ''),
|
|
796
|
-
this.consumeNew(
|
|
797
|
-
queue,
|
|
798
|
-
callback,
|
|
799
|
-
options,
|
|
800
|
-
),
|
|
801
|
-
]);
|
|
802
|
-
});
|
|
803
|
-
}
|
|
704
|
+
await this.saveConsumer(queue, callback, options);
|
|
705
|
+
const channel: ChannelWrapper = await this.getNewChannel({ name: `consume-exchange-${exchange}-queue-${queue}` });
|
|
804
706
|
|
|
805
|
-
|
|
806
|
-
await this.saveConsumerOld(queue, callback, options);
|
|
807
|
-
const channelOld: ChannelWrapper = await this.getNewChannelOld({ name: `consume-exchange-${exchange}-queue-${queue}-old` });
|
|
808
|
-
await channelOld.addSetup(async (c: ConfirmChannel) => {
|
|
707
|
+
return channel.addSetup(async (c: ConfirmChannel) => {
|
|
809
708
|
const assertExchange = await assertExchangeFanout(c, exchange);
|
|
810
709
|
await c.assertQueue(queue);
|
|
811
|
-
this.
|
|
812
|
-
await c.prefetch(limit,
|
|
710
|
+
this.exchanges[exchange] = assertExchange;
|
|
711
|
+
await c.prefetch(limit, true);
|
|
813
712
|
return Promise.all([
|
|
814
713
|
c.bindQueue(queue, exchange, ''),
|
|
815
|
-
this.
|
|
714
|
+
this.consume(
|
|
816
715
|
queue,
|
|
817
716
|
callback,
|
|
818
717
|
options,
|
|
@@ -821,21 +720,17 @@ class RabbitMq implements IAfRabbitMq {
|
|
|
821
720
|
});
|
|
822
721
|
}
|
|
823
722
|
|
|
824
|
-
// Used by the microservices to publish messages to the exchange
|
|
825
|
-
// TODO: [QUORUM-PHASE-2] Change the implementation to the new one
|
|
826
723
|
async publish(exchange: string, content: any, customHeaders?: any) : Promise<boolean> {
|
|
827
724
|
return wrapSetImmediate(async () => {
|
|
828
725
|
RabbitMq.validateName('exchange', exchange);
|
|
829
|
-
const channel: ChannelWrapper = await this.
|
|
830
|
-
await this.
|
|
726
|
+
const channel: ChannelWrapper = await this.assertChannel();
|
|
727
|
+
await this.assertExchange(exchange);
|
|
831
728
|
await channel.publish(exchange, '',
|
|
832
729
|
Buffer.from(JSON.stringify(content)),
|
|
833
730
|
RabbitMq.getPublishOptions(customHeaders));
|
|
834
731
|
});
|
|
835
732
|
}
|
|
836
733
|
|
|
837
|
-
// Used by the microservices to send messages to the queue
|
|
838
|
-
// TODO: [QUORUM-PHASE-2] Change the implementation to the new one
|
|
839
734
|
async sendToQueue(
|
|
840
735
|
queue: string,
|
|
841
736
|
content: any,
|
|
@@ -843,7 +738,7 @@ class RabbitMq implements IAfRabbitMq {
|
|
|
843
738
|
customHeaders?: any,
|
|
844
739
|
): Promise<boolean | undefined> {
|
|
845
740
|
try {
|
|
846
|
-
await this.
|
|
741
|
+
await this.assertChannel();
|
|
847
742
|
} catch (e) {
|
|
848
743
|
logger.error(`rabbit sendToQueue: failed to send assert channel when sending to queue ${queue}`, { e });
|
|
849
744
|
throw e;
|
|
@@ -851,14 +746,14 @@ class RabbitMq implements IAfRabbitMq {
|
|
|
851
746
|
|
|
852
747
|
try {
|
|
853
748
|
RabbitMq.validateName('queue', queue);
|
|
854
|
-
await this.
|
|
749
|
+
await this.assertQueue(queue, options);
|
|
855
750
|
} catch (e) {
|
|
856
751
|
logger.error(`rabbit sendToQueue: failed to assert queue ${queue}`, { e });
|
|
857
752
|
throw e;
|
|
858
753
|
}
|
|
859
754
|
|
|
860
755
|
try {
|
|
861
|
-
const res = await this.
|
|
756
|
+
const res = await this.channel?.sendToQueue(queue,
|
|
862
757
|
Buffer.from(JSON.stringify(content)),
|
|
863
758
|
RabbitMq.getPublishOptions(customHeaders));
|
|
864
759
|
debug(`rabbit: sending to queue ${queue}`, { res });
|
|
@@ -870,38 +765,67 @@ class RabbitMq implements IAfRabbitMq {
|
|
|
870
765
|
}
|
|
871
766
|
}
|
|
872
767
|
|
|
873
|
-
// TODO: [QUORUM-PHASE-2] After changing the publish from old to new change from the old to the new implementation
|
|
874
768
|
async isConnected() : Promise<boolean> {
|
|
875
|
-
const connection = await this.getConnectionOld();
|
|
876
|
-
const isConnected = connection.isConnected();
|
|
877
|
-
if (!isConnected) {
|
|
878
|
-
logger.error('rabbit: isConnected - false');
|
|
879
|
-
return false;
|
|
880
|
-
}
|
|
881
|
-
const channel: any = await this.assertChannelOld();
|
|
882
769
|
try {
|
|
883
|
-
await
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
770
|
+
const connection = await this.getConnection();
|
|
771
|
+
if (!connection.isConnected()) {
|
|
772
|
+
logger.error('rabbit: isConnected - false');
|
|
773
|
+
return false;
|
|
774
|
+
}
|
|
775
|
+
const channel: any = await this.assertChannel();
|
|
776
|
+
const timeoutMs = 5000;
|
|
777
|
+
const retries = 2;
|
|
778
|
+
const chunkSize = 10;
|
|
779
|
+
const processChunks = async (consumers: AfConsumer[]): Promise<boolean> => {
|
|
780
|
+
for (let i = 0; i < consumers.length; i += chunkSize) {
|
|
781
|
+
const chunk = consumers.slice(i, i + chunkSize);
|
|
782
|
+
// eslint-disable-next-line no-await-in-loop
|
|
783
|
+
const chunkResults = await Promise.all(
|
|
784
|
+
chunk.map((c: AfConsumer) => withRetry(() => channel.checkQueue(c.queue), retries, timeoutMs).catch((err) => {
|
|
785
|
+
logger.error(`rabbit: Error in isConnected (checkQueue) - ${err.message}`);
|
|
786
|
+
return null;
|
|
787
|
+
})),
|
|
788
|
+
);
|
|
789
|
+
if (chunkResults.some((res) => res === null)) {
|
|
790
|
+
return false;
|
|
791
|
+
}
|
|
792
|
+
}
|
|
793
|
+
return true;
|
|
794
|
+
};
|
|
795
|
+
const waitForConnectResult = await withRetry(
|
|
796
|
+
() => channel.waitForConnect(),
|
|
797
|
+
retries,
|
|
798
|
+
timeoutMs,
|
|
799
|
+
).catch((err) => {
|
|
800
|
+
logger.error(`rabbit: Error in isConnected (waitForConnect) - ${err.message}`);
|
|
801
|
+
return null;
|
|
802
|
+
});
|
|
803
|
+
|
|
804
|
+
if (waitForConnectResult === null) {
|
|
805
|
+
logger.error('rabbit: isConnected - false due to failed waitForConnect');
|
|
806
|
+
return false;
|
|
807
|
+
}
|
|
808
|
+
const allConsumersConnected = await processChunks(this.consumers);
|
|
809
|
+
if (!allConsumersConnected) {
|
|
810
|
+
logger.error('rabbit: isConnected - false due to failed consumer checks');
|
|
811
|
+
return false;
|
|
812
|
+
}
|
|
813
|
+
|
|
814
|
+
logger.info('rabbit: isConnected - true');
|
|
815
|
+
return true;
|
|
887
816
|
} catch (e) {
|
|
888
|
-
logger.error(
|
|
817
|
+
logger.error(`rabbit: isConnected - Exception occurred: ${e}`);
|
|
889
818
|
return false;
|
|
890
819
|
}
|
|
891
|
-
logger.info('rabbit: isConnected - true');
|
|
892
|
-
return true;
|
|
893
820
|
}
|
|
894
821
|
|
|
895
822
|
async gracefulShutdown(signal: string) : Promise<void> {
|
|
896
|
-
|
|
897
|
-
const tagsNumber = this.consumersTags.length + this.oldConsumersTags.length;
|
|
823
|
+
const tagsNumber = this.consumersTags.length;
|
|
898
824
|
logger.info(`rabbit: [gracefully-shutdown] received ${signal}! canceling #${tagsNumber} tags...`);
|
|
899
825
|
const cancelTagPromises = this.consumersTags.map(([channel, tag]) => channel.cancel(tag));
|
|
900
|
-
const cancelTagPromisesOld = this.oldConsumersTags.map(([channel, tag]) => channel.cancel(tag));
|
|
901
826
|
// Clean the array to avoid race
|
|
902
827
|
this.consumersTags = [];
|
|
903
|
-
|
|
904
|
-
const results = await Promise.allSettled([...cancelTagPromises, ...cancelTagPromisesOld]);
|
|
828
|
+
const results = await Promise.allSettled(cancelTagPromises);
|
|
905
829
|
const rejected = results.filter((p) => p.status === 'rejected');
|
|
906
830
|
if (rejected.length > 0) {
|
|
907
831
|
logger.warn(`rabbit: [gracefully-shutdown] #${rejected.length}/${tagsNumber} tags failed to cancel: ${rejected}`);
|
|
@@ -909,336 +833,6 @@ class RabbitMq implements IAfRabbitMq {
|
|
|
909
833
|
logger.info('rabbit: [gracefully-shutdown] all tags successfully canceled.');
|
|
910
834
|
}
|
|
911
835
|
}
|
|
912
|
-
|
|
913
|
-
private async consumeFromRabbitOld(queue: string, callback: CallbackFunction, options?: ConsumeOptions): Promise<any> {
|
|
914
|
-
const optionsWithDefaults = { ...DEFAULT_OPTIONS, ...options };
|
|
915
|
-
RabbitMq.validateName('queue', queue);
|
|
916
|
-
this.saveConsumerOld(queue, callback, options);
|
|
917
|
-
const uniqueId = randomUUID();
|
|
918
|
-
const {
|
|
919
|
-
limit, deadMessageTtl, useConsumeWithLock, lockTimeout, auditContext, enableRabbitTrace,
|
|
920
|
-
} = optionsWithDefaults;
|
|
921
|
-
if (useConsumeWithLock) {
|
|
922
|
-
if (!this.redisLock) {
|
|
923
|
-
throw new RabbitError('Usage of consumeWithLock requires RedisInstance');
|
|
924
|
-
}
|
|
925
|
-
logger.info(`rabbit: Consuming with lock from queue ${queue} with lockTimeout: ${lockTimeout}ms`);
|
|
926
|
-
}
|
|
927
|
-
const channel = await this.getNewChannelOld({});
|
|
928
|
-
return channel.addSetup(async (confirmChannel: ConfirmChannel) => {
|
|
929
|
-
const q = await this.assertQueueOld(queue, optionsWithDefaults);
|
|
930
|
-
await confirmChannel.prefetch(limit, false);
|
|
931
|
-
const { consumerTag } = await confirmChannel.consume(
|
|
932
|
-
queue,
|
|
933
|
-
async (msg: ConsumeMessageOrNull) => {
|
|
934
|
-
if (!msg) {
|
|
935
|
-
return null;
|
|
936
|
-
}
|
|
937
|
-
|
|
938
|
-
const traceId = msg.properties.headers[TRACING_HEADER];
|
|
939
|
-
const userId = msg.properties.headers[USER_TRACING_HEADER];
|
|
940
|
-
const automationId = msg.properties.headers[AUTOMATION_ID_HEADER];
|
|
941
|
-
const parsedMessage = RabbitMq.parseMsg(msg);
|
|
942
|
-
const releaseLock = await this.lockRedisIfNeeded(parsedMessage, optionsWithDefaults);
|
|
943
|
-
const trace = newTrace(traceTypes.RABBIT);
|
|
944
|
-
// setting also outbreak trace as part of legacy code
|
|
945
|
-
const outbreakTrace = outbreak.newTrace(traceTypes.RABBIT);
|
|
946
|
-
// enableRabbitTrace is a flag to protect from different flows that doesn't work with permission
|
|
947
|
-
// and we don't want to fail the flow because of it
|
|
948
|
-
if (userId && enableRabbitTrace) {
|
|
949
|
-
try {
|
|
950
|
-
await Promise.all([
|
|
951
|
-
createOrSetRabbitTrace(trace, userId),
|
|
952
|
-
createOrSetRabbitTrace(outbreakTrace, userId),
|
|
953
|
-
]);
|
|
954
|
-
} catch (e) {
|
|
955
|
-
logger.error('rabbit: failed to setRabbitTrace', { userId, e });
|
|
956
|
-
return this.nack(confirmChannel, queue, optionsWithDefaults, { messageTtl: deadMessageTtl }, msg, releaseLock)(msg);
|
|
957
|
-
}
|
|
958
|
-
}
|
|
959
|
-
|
|
960
|
-
if (traceId) {
|
|
961
|
-
(trace as any)?.context?.set(TRACING_HEADER, traceId);
|
|
962
|
-
(outbreakTrace as any)?.context.set(TRACING_HEADER, traceId);
|
|
963
|
-
}
|
|
964
|
-
|
|
965
|
-
if (auditContext) {
|
|
966
|
-
await auditContext(queue, {
|
|
967
|
-
userId,
|
|
968
|
-
automationId,
|
|
969
|
-
});
|
|
970
|
-
}
|
|
971
|
-
const shouldConsume = await this.shouldConsumeMessageByTimestamp(parsedMessage);
|
|
972
|
-
if (!shouldConsume) {
|
|
973
|
-
await this.unlockRedisIfNeeded(releaseLock);
|
|
974
|
-
return this.ack(confirmChannel, msg)(msg);
|
|
975
|
-
}
|
|
976
|
-
|
|
977
|
-
let messageAcked = false;
|
|
978
|
-
// setting the localAck function to be used in the callback
|
|
979
|
-
|
|
980
|
-
const localAck = async () => {
|
|
981
|
-
if (messageAcked) {
|
|
982
|
-
return;
|
|
983
|
-
}
|
|
984
|
-
messageAcked = true;
|
|
985
|
-
return this.ack(confirmChannel, msg, true, releaseLock)(msg);
|
|
986
|
-
};
|
|
987
|
-
|
|
988
|
-
const localNack = async (_: ConsumeMessageOrNull, nackOptions: NackOptions = {}) => {
|
|
989
|
-
if (messageAcked) {
|
|
990
|
-
return;
|
|
991
|
-
}
|
|
992
|
-
debug('rabbit localNack', { messageAcked, uniqueId, deliveryTag: msg.fields.deliveryTag });
|
|
993
|
-
messageAcked = true;
|
|
994
|
-
return this.nack(confirmChannel, queue, optionsWithDefaults, { messageTtl: deadMessageTtl }, msg, releaseLock)(msg, nackOptions);
|
|
995
|
-
};
|
|
996
|
-
|
|
997
|
-
try {
|
|
998
|
-
await callback(
|
|
999
|
-
parsedMessage,
|
|
1000
|
-
localAck,
|
|
1001
|
-
localNack,
|
|
1002
|
-
);
|
|
1003
|
-
} catch (e) {
|
|
1004
|
-
await localNack(msg);
|
|
1005
|
-
}
|
|
1006
|
-
}, CONSUMER_DEFAULT_OPTIONS,
|
|
1007
|
-
);
|
|
1008
|
-
if (!consumerTag) {
|
|
1009
|
-
logger.error(`rabbit: failed to consume from queue ${queue}`);
|
|
1010
|
-
} else {
|
|
1011
|
-
logger.info(`rabbit: adding tag ${consumerTag} to the array.`);
|
|
1012
|
-
this.oldConsumersTags.push([confirmChannel, consumerTag]);
|
|
1013
|
-
}
|
|
1014
|
-
});
|
|
1015
|
-
}
|
|
1016
|
-
|
|
1017
|
-
// TODO: [QUORUM-PHASE-3] Delete all the function under this line (getNewChannelOld, getConnectionOld, assertQueueOld, setupQueueOld, saveConsumerOld)
|
|
1018
|
-
async getNewChannelOld({ name = rand().toString(), onClose = null, options = {} }: newChannelOpts = {}) {
|
|
1019
|
-
let connection!: AmqpConnectionManager;
|
|
1020
|
-
try {
|
|
1021
|
-
connection = await this.getConnectionOld();
|
|
1022
|
-
} catch (e) {
|
|
1023
|
-
logger.error(`rabbit: error on get connection for new channel ${name} `, { e });
|
|
1024
|
-
throw e;
|
|
1025
|
-
}
|
|
1026
|
-
const channel = connection.createChannel({ ...options });
|
|
1027
|
-
once(channel, 'close').then((args) => {
|
|
1028
|
-
logger.error(`rabbit: channel ${name} closed`);
|
|
1029
|
-
onClose?.(args);
|
|
1030
|
-
});
|
|
1031
|
-
try {
|
|
1032
|
-
await once(channel, 'connect');
|
|
1033
|
-
debug(`rabbit: channel ${name} CONNECTED`);
|
|
1034
|
-
return channel;
|
|
1035
|
-
} catch (err) {
|
|
1036
|
-
logger.error(`rabbit: channel error ${name} error`, { err });
|
|
1037
|
-
throw err;
|
|
1038
|
-
}
|
|
1039
|
-
}
|
|
1040
|
-
|
|
1041
|
-
async getConnectionOld() {
|
|
1042
|
-
return new Promise<AmqpConnectionManager>(async (resolve, reject) => {
|
|
1043
|
-
if (this.oldBlockReconnect) {
|
|
1044
|
-
debug('rabbit: block reconnect');
|
|
1045
|
-
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
|
1046
|
-
// @ts-ignore
|
|
1047
|
-
return resolve();
|
|
1048
|
-
}
|
|
1049
|
-
if (this.oldConnection !== null) {
|
|
1050
|
-
if (this.options?.disableReconnect || this.oldConnection?.isConnected()) {
|
|
1051
|
-
debug('rabbit: connection - is connected');
|
|
1052
|
-
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
|
1053
|
-
// @ts-ignore
|
|
1054
|
-
return resolve(this.oldConnection);
|
|
1055
|
-
}
|
|
1056
|
-
debug('rabbit: connection - reconnecting');
|
|
1057
|
-
}
|
|
1058
|
-
if (this.oldCreatingConnection) {
|
|
1059
|
-
debug('rabbit: creating connection emi');
|
|
1060
|
-
this.oldEm.once(CONNECTION_CREATED_CONST, resolve);
|
|
1061
|
-
this.oldEm.once(CONNECTION_FAILED_CONST, reject);
|
|
1062
|
-
return;
|
|
1063
|
-
}
|
|
1064
|
-
this.oldCreatingConnection = true;
|
|
1065
|
-
let isResolved = false;
|
|
1066
|
-
|
|
1067
|
-
// It is import to use it as a function and not as a variable
|
|
1068
|
-
// because of k8s changes the env variables
|
|
1069
|
-
// and we want to use the new values
|
|
1070
|
-
const findServers = () => {
|
|
1071
|
-
const userName = process.env.RABBITMQ_USERNAME || 'guest';
|
|
1072
|
-
const password = process.env.RABBITMQ_PASSWORD || 'guest';
|
|
1073
|
-
const host = this.options?.rabbitHost || process.env.RABBITMQ_SERVICE_HOST || 'localhost';
|
|
1074
|
-
|
|
1075
|
-
debug('rabbit: creating connection', { host, userName, HEARTBEAT });
|
|
1076
|
-
|
|
1077
|
-
return [`amqp://${userName}:${password}@${host}?heartbeat=${HEARTBEAT}`];
|
|
1078
|
-
};
|
|
1079
|
-
|
|
1080
|
-
const defaultUrls = findServers();
|
|
1081
|
-
const connection: AmqpConnectionManager = await connect(defaultUrls, {
|
|
1082
|
-
findServers,
|
|
1083
|
-
});
|
|
1084
|
-
|
|
1085
|
-
this.oldConnection = connection;
|
|
1086
|
-
this.oldConnection.on('error', (err) => {
|
|
1087
|
-
logger.error('rabbit: connection error', { err });
|
|
1088
|
-
if (!isResolved) {
|
|
1089
|
-
isResolved = true;
|
|
1090
|
-
reject(err);
|
|
1091
|
-
this.oldEm.emit(CONNECTION_FAILED_CONST, err);
|
|
1092
|
-
}
|
|
1093
|
-
});
|
|
1094
|
-
|
|
1095
|
-
this.oldConnection.on('connectFailed', (err) => {
|
|
1096
|
-
this.oldConsumersTags = [];
|
|
1097
|
-
logger.error('rabbit: connection connectFailed', { err });
|
|
1098
|
-
if (!isResolved) {
|
|
1099
|
-
isResolved = true;
|
|
1100
|
-
reject(err);
|
|
1101
|
-
this.oldEm.emit(CONNECTION_FAILED_CONST, err);
|
|
1102
|
-
}
|
|
1103
|
-
});
|
|
1104
|
-
|
|
1105
|
-
this.oldConnection.on('disconnect', ({ err }) => {
|
|
1106
|
-
this.oldConsumersTags = [];
|
|
1107
|
-
debug('rabbit: connection closed');
|
|
1108
|
-
if (this.options?.disableReconnect) {
|
|
1109
|
-
logger.error(`${this.DISCONNECT_MSG}${err && ` - ${err}`}`);
|
|
1110
|
-
this.oldBlockReconnect = true;
|
|
1111
|
-
} else {
|
|
1112
|
-
logger.error(`${this.RECONNECT_MSG}${err && ` - ${err}`}`);
|
|
1113
|
-
}
|
|
1114
|
-
});
|
|
1115
|
-
|
|
1116
|
-
this.oldConnection.once('connect', async () => {
|
|
1117
|
-
debug('rabbit: connection established');
|
|
1118
|
-
this.oldCreatingConnection = false;
|
|
1119
|
-
this.oldEm.emit(CONNECTION_CREATED_CONST, connection);
|
|
1120
|
-
isResolved = true;
|
|
1121
|
-
resolve(connection);
|
|
1122
|
-
});
|
|
1123
|
-
});
|
|
1124
|
-
}
|
|
1125
|
-
|
|
1126
|
-
private saveConsumerOld(queue: string, callback: CallbackFunction, options: ConsumeOptions | undefined) {
|
|
1127
|
-
const isConsumerExist :boolean = this.oldConsumers.some((consumer) => consumer.queue === queue);
|
|
1128
|
-
if (!isConsumerExist) {
|
|
1129
|
-
logger.info(`rabbit: consumer: ${queue} saved in consumer array`);
|
|
1130
|
-
this.oldConsumers.push({
|
|
1131
|
-
queue,
|
|
1132
|
-
callback,
|
|
1133
|
-
options,
|
|
1134
|
-
});
|
|
1135
|
-
}
|
|
1136
|
-
}
|
|
1137
|
-
|
|
1138
|
-
async assertQueueOld(queueName: string, options?: Options.AssertQueue): Promise<any> {
|
|
1139
|
-
RabbitMq.validateName('queue', queueName);
|
|
1140
|
-
if (this.oldQueues[queueName]) {
|
|
1141
|
-
delete this.oldQueueSetupPromises[queueName];
|
|
1142
|
-
return this.oldQueues[queueName];
|
|
1143
|
-
}
|
|
1144
|
-
|
|
1145
|
-
if (this.oldQueueSetupPromises[queueName]) {
|
|
1146
|
-
return this.oldQueueSetupPromises[queueName];
|
|
1147
|
-
}
|
|
1148
|
-
|
|
1149
|
-
this.oldQueueSetupPromises[queueName] = this.setupQueueOld(queueName, options);
|
|
1150
|
-
return this.oldQueueSetupPromises[queueName];
|
|
1151
|
-
}
|
|
1152
|
-
|
|
1153
|
-
async setupQueueOld(queueName: string, options?: Options.AssertQueue): Promise<Replies.AssertQueue> {
|
|
1154
|
-
let queue: Replies.AssertQueue;
|
|
1155
|
-
const shouldUseQuorum = RabbitMq.shouldUseQuorum(queueName);
|
|
1156
|
-
const localeOptions = {
|
|
1157
|
-
...options,
|
|
1158
|
-
durable: true,
|
|
1159
|
-
arguments: {
|
|
1160
|
-
...options?.arguments,
|
|
1161
|
-
'x-consumer-timeout': 1000 * 60 * 60 * 24,
|
|
1162
|
-
'x-queue-type': shouldUseQuorum ? 'quorum' : 'classic',
|
|
1163
|
-
},
|
|
1164
|
-
};
|
|
1165
|
-
try {
|
|
1166
|
-
const channel: ChannelWrapper = await this.assertChannelOld();
|
|
1167
|
-
debug('assertQueue->channel.addSetup', { queueName });
|
|
1168
|
-
await channel.addSetup((setupChannel: ConfirmChannel) => setupChannel.assertQueue(queueName, localeOptions));
|
|
1169
|
-
debug('assertQueue->channel.assertQueue', { queueName });
|
|
1170
|
-
queue = await channel.assertQueue(queueName, localeOptions);
|
|
1171
|
-
} catch (e) {
|
|
1172
|
-
logger.error('rabbit: assertQueue error', { queueName, options, error: e });
|
|
1173
|
-
if (!this.options?.dontRetryAssert) {
|
|
1174
|
-
debug('retrying assertQueue', { queueName });
|
|
1175
|
-
const channel = await this.assertChannelOld({ force: true });
|
|
1176
|
-
await this.deleteQueueOld(queueName);
|
|
1177
|
-
|
|
1178
|
-
debug('retrying assertQueue->channel.addSetup', { queueName });
|
|
1179
|
-
await channel.addSetup((setupChannel: ConfirmChannel) => setupChannel.assertQueue(queueName, localeOptions));
|
|
1180
|
-
debug('retrying assertQueue->channel.assertQueue', { queueName });
|
|
1181
|
-
queue = await channel.assertQueue(queueName, localeOptions);
|
|
1182
|
-
} else {
|
|
1183
|
-
throw e;
|
|
1184
|
-
}
|
|
1185
|
-
}
|
|
1186
|
-
|
|
1187
|
-
this.oldQueues[queueName] = queue;
|
|
1188
|
-
return queue;
|
|
1189
|
-
}
|
|
1190
|
-
|
|
1191
|
-
async assertChannelOld({ force = false } : assertChannelOpts = {}): Promise<ChannelWrapper> {
|
|
1192
|
-
if (!this.oldPublishChannelSetupPromise) {
|
|
1193
|
-
this.oldPublishChannelSetupPromise = new Promise<ChannelWrapper>(async (resolve, reject) => {
|
|
1194
|
-
if (this.oldChannel && !force) {
|
|
1195
|
-
return resolve(this.oldChannel);
|
|
1196
|
-
}
|
|
1197
|
-
|
|
1198
|
-
try {
|
|
1199
|
-
const channel = await this.getNewChannelOld({});
|
|
1200
|
-
channel.on('error', (err) => {
|
|
1201
|
-
logger.error('rabbit: channel error', { err });
|
|
1202
|
-
});
|
|
1203
|
-
this.oldChannel = channel;
|
|
1204
|
-
resolve(channel);
|
|
1205
|
-
} catch (e) {
|
|
1206
|
-
reject(e);
|
|
1207
|
-
}
|
|
1208
|
-
});
|
|
1209
|
-
}
|
|
1210
|
-
return this.oldPublishChannelSetupPromise;
|
|
1211
|
-
}
|
|
1212
|
-
|
|
1213
|
-
private async deleteQueueOld(queue: string) {
|
|
1214
|
-
RabbitMq.validateName('queue', queue);
|
|
1215
|
-
const channel: ChannelWrapper = await this.assertChannelOld();
|
|
1216
|
-
logger.info('rabbit: deleting queue', { queue });
|
|
1217
|
-
const deleteQueueRes = await channel.deleteQueue(queue);
|
|
1218
|
-
debug('queue deleted', deleteQueueRes);
|
|
1219
|
-
return deleteQueueRes;
|
|
1220
|
-
}
|
|
1221
|
-
|
|
1222
|
-
async assertExchangeOld(exchangeName: string, options?: any) {
|
|
1223
|
-
const channel: ChannelWrapper = await this.assertChannelOld();
|
|
1224
|
-
|
|
1225
|
-
if (this.oldExchanges[exchangeName]) {
|
|
1226
|
-
delete this.oldAssertExchangePromises[exchangeName];
|
|
1227
|
-
return this.oldExchanges[exchangeName];
|
|
1228
|
-
}
|
|
1229
|
-
|
|
1230
|
-
if (this.oldAssertExchangePromises[exchangeName]) {
|
|
1231
|
-
return this.oldAssertExchangePromises[exchangeName];
|
|
1232
|
-
}
|
|
1233
|
-
|
|
1234
|
-
this.oldAssertExchangePromises[exchangeName] = assertExchangeFanout(channel, exchangeName);
|
|
1235
|
-
this.oldExchanges[exchangeName] = await this.oldAssertExchangePromises[exchangeName];
|
|
1236
|
-
return this.oldExchanges[exchangeName];
|
|
1237
|
-
}
|
|
1238
836
|
}
|
|
1239
837
|
|
|
1240
838
|
export default RabbitMq;
|
|
1241
|
-
|
|
1242
|
-
export {
|
|
1243
|
-
sendCeleryTaskViaHttp,
|
|
1244
|
-
} from './lib/celery';
|