@autofleet/rabbit 3.3.22-connection-test-beta → 3.4.0-0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.nvmrc +1 -1
- package/dist/index.d.ts +29 -0
- package/dist/index.js +459 -94
- package/dist/lib/celery.d.ts +9 -0
- package/dist/lib/celery.js +54 -0
- package/dist/lib/types.d.ts +1 -0
- package/package.json +6 -3
- package/src/index.ts +542 -107
- package/src/lib/celery.ts +89 -0
- package/src/lib/types.ts +2 -0
package/src/index.ts
CHANGED
|
@@ -41,6 +41,7 @@ import {
|
|
|
41
41
|
QueueSetupPromisesDictionary,
|
|
42
42
|
AssertExchangePromisesDictionary,
|
|
43
43
|
} from './lib/types';
|
|
44
|
+
import { SendTaskOptions, TaskData } from './lib/celery';
|
|
44
45
|
|
|
45
46
|
// const debug = nodeDebug('af-rabbitmq')
|
|
46
47
|
const debug = logger.debug.bind(logger);
|
|
@@ -100,44 +101,6 @@ type AfConsumer = {
|
|
|
100
101
|
}
|
|
101
102
|
|
|
102
103
|
const HEARTBEAT = '60';
|
|
103
|
-
const withTimeout = async<T>(promise: Promise<T>, timeoutMs: number): Promise<T> => new Promise((resolve, reject) => {
|
|
104
|
-
const timer = setTimeout(() => {
|
|
105
|
-
reject(new Error(`Promise timed out after ${timeoutMs} ms`));
|
|
106
|
-
}, timeoutMs);
|
|
107
|
-
|
|
108
|
-
promise
|
|
109
|
-
.then((value) => {
|
|
110
|
-
clearTimeout(timer);
|
|
111
|
-
resolve(value);
|
|
112
|
-
})
|
|
113
|
-
.catch((err) => {
|
|
114
|
-
clearTimeout(timer);
|
|
115
|
-
reject(err);
|
|
116
|
-
});
|
|
117
|
-
});
|
|
118
|
-
|
|
119
|
-
const withRetry = async <T>(
|
|
120
|
-
operation: () => Promise<T>,
|
|
121
|
-
retries: number,
|
|
122
|
-
timeoutMs: number,
|
|
123
|
-
): Promise<T> => new Promise(async (resolve, reject) => {
|
|
124
|
-
let lastError: any;
|
|
125
|
-
|
|
126
|
-
for (let attempt = 1; attempt <= retries; attempt += 1) {
|
|
127
|
-
try {
|
|
128
|
-
// eslint-disable-next-line no-await-in-loop
|
|
129
|
-
const result = await withTimeout(operation(), timeoutMs);
|
|
130
|
-
return resolve(result);
|
|
131
|
-
} catch (err) {
|
|
132
|
-
lastError = err;
|
|
133
|
-
logger.error(`Attempt ${attempt} failed: ${err}`);
|
|
134
|
-
if (attempt < retries) {
|
|
135
|
-
logger.info(`Retrying (${attempt}/${retries})...`);
|
|
136
|
-
}
|
|
137
|
-
}
|
|
138
|
-
}
|
|
139
|
-
reject(lastError);
|
|
140
|
-
});
|
|
141
104
|
|
|
142
105
|
class RabbitMq implements IAfRabbitMq {
|
|
143
106
|
static parseMsg(msg: any) : any {
|
|
@@ -212,7 +175,36 @@ class RabbitMq implements IAfRabbitMq {
|
|
|
212
175
|
|
|
213
176
|
private consumers: Array<AfConsumer> = [];
|
|
214
177
|
|
|
215
|
-
|
|
178
|
+
private doesVHostExist = false;
|
|
179
|
+
|
|
180
|
+
private vhost = 'quorum-vhost';
|
|
181
|
+
|
|
182
|
+
// TODO:[QUORUM-PHASE-3] Delete the old properties that we use for the old consumers and publishers
|
|
183
|
+
oldChannel: ChannelWrapper | null;
|
|
184
|
+
|
|
185
|
+
oldPublishChannelSetupPromise: Promise<ChannelWrapper> | null;
|
|
186
|
+
|
|
187
|
+
oldBlockReconnect: boolean | null | undefined
|
|
188
|
+
|
|
189
|
+
oldConnection: AmqpConnectionManager | null | undefined
|
|
190
|
+
|
|
191
|
+
oldEm: EventEmitter;
|
|
192
|
+
|
|
193
|
+
oldCreatingConnection: boolean;
|
|
194
|
+
|
|
195
|
+
oldExchanges: ExchangesCache;
|
|
196
|
+
|
|
197
|
+
oldQueues: QueuesCache;
|
|
198
|
+
|
|
199
|
+
oldQueueSetupPromises: QueueSetupPromisesDictionary;
|
|
200
|
+
|
|
201
|
+
oldAssertExchangePromises: AssertExchangePromisesDictionary;
|
|
202
|
+
|
|
203
|
+
oldConsumersTags: Array<[ConfirmChannel, string]>;
|
|
204
|
+
|
|
205
|
+
private oldConsumers: Array<AfConsumer> = [];
|
|
206
|
+
|
|
207
|
+
constructor(options: AfRabbitOptions = {}, redisConfig?: RedisConfig) {
|
|
216
208
|
this.em = new EventEmitter();
|
|
217
209
|
this.channel = null;
|
|
218
210
|
this.publishChannelSetupPromise = null;
|
|
@@ -224,6 +216,7 @@ class RabbitMq implements IAfRabbitMq {
|
|
|
224
216
|
this.assertExchangePromises = {};
|
|
225
217
|
this.consumers = [];
|
|
226
218
|
this.options = options;
|
|
219
|
+
|
|
227
220
|
this.redisClient = redisConfig && getRedisInstance(redisConfig);
|
|
228
221
|
if (this.redisClient) {
|
|
229
222
|
this.redisLock = promisify(RedisLock(this.redisClient)) as RedisLockType;
|
|
@@ -238,8 +231,74 @@ class RabbitMq implements IAfRabbitMq {
|
|
|
238
231
|
await this.gracefulShutdown('SIGINT');
|
|
239
232
|
});
|
|
240
233
|
}
|
|
234
|
+
|
|
235
|
+
// TODO: [QUORUM-PHASE-3] Delete the old properties that we use for the old consumers and publishers
|
|
236
|
+
this.oldEm = new EventEmitter();
|
|
237
|
+
this.oldChannel = null;
|
|
238
|
+
this.oldPublishChannelSetupPromise = null;
|
|
239
|
+
this.oldConnection = null;
|
|
240
|
+
this.oldCreatingConnection = false;
|
|
241
|
+
this.oldExchanges = {};
|
|
242
|
+
this.oldQueues = {};
|
|
243
|
+
this.oldQueueSetupPromises = {};
|
|
244
|
+
this.oldAssertExchangePromises = {};
|
|
245
|
+
this.oldConsumers = [];
|
|
246
|
+
this.oldConsumersTags = [];
|
|
241
247
|
}
|
|
242
248
|
|
|
249
|
+
private assertVHost = async () => {
|
|
250
|
+
if (this.doesVHostExist) {
|
|
251
|
+
return;
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
const username = process.env.RABBITMQ_USERNAME || 'guest';
|
|
255
|
+
const password = process.env.RABBITMQ_PASSWORD || 'guest';
|
|
256
|
+
const credentials = Buffer.from(`${username}:${password}`).toString('base64');
|
|
257
|
+
const headers = {
|
|
258
|
+
Authorization: `Basic ${credentials}`,
|
|
259
|
+
'Content-Type': 'application/json',
|
|
260
|
+
};
|
|
261
|
+
|
|
262
|
+
const rabbitHost = `http://${(this.options?.rabbitHost || process.env.RABBITMQ_SERVICE_HOST || 'localhost').split(':')[0]}:15672`;
|
|
263
|
+
|
|
264
|
+
const url = `${rabbitHost}/api/vhosts/${encodeURIComponent(this.vhost)}`;
|
|
265
|
+
|
|
266
|
+
try {
|
|
267
|
+
const response = await fetch(url, {
|
|
268
|
+
method: 'GET',
|
|
269
|
+
headers,
|
|
270
|
+
});
|
|
271
|
+
|
|
272
|
+
if (response.status === 200) {
|
|
273
|
+
this.doesVHostExist = true;
|
|
274
|
+
logger.info('Vhost exists', { vhost: this.vhost });
|
|
275
|
+
return;
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
if (response.status !== 404) {
|
|
279
|
+
logger.error('Failed to check vhost', { response });
|
|
280
|
+
throw new RabbitError('Failed to check vhost');
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
const createResponse = await fetch(url, {
|
|
284
|
+
method: 'PUT',
|
|
285
|
+
headers,
|
|
286
|
+
body: JSON.stringify({ default_queue_type: 'quorum' }),
|
|
287
|
+
});
|
|
288
|
+
|
|
289
|
+
if (!createResponse.ok) {
|
|
290
|
+
logger.error('Failed to create vhost', { response: createResponse });
|
|
291
|
+
throw new RabbitError('Failed to create vhost');
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
this.doesVHostExist = true;
|
|
295
|
+
logger.info('Vhost created', { vhost: this.vhost });
|
|
296
|
+
} catch (error) {
|
|
297
|
+
logger.error('Failed to check or create vhost', { error });
|
|
298
|
+
throw error;
|
|
299
|
+
}
|
|
300
|
+
};
|
|
301
|
+
|
|
243
302
|
private shouldConsumeMessageByTimestamp = async (msg: ConsumeMessageOrNull) => {
|
|
244
303
|
if (msg) {
|
|
245
304
|
const { properties: { headers } } = msg;
|
|
@@ -350,8 +409,7 @@ class RabbitMq implements IAfRabbitMq {
|
|
|
350
409
|
const host = this.options?.rabbitHost || process.env.RABBITMQ_SERVICE_HOST || 'localhost';
|
|
351
410
|
|
|
352
411
|
debug('rabbit: creating connection', { host, userName, HEARTBEAT });
|
|
353
|
-
|
|
354
|
-
return [`amqp://${userName}:${password}@${host}?heartbeat=${HEARTBEAT}`];
|
|
412
|
+
return [`amqp://${userName}:${password}@${host}/${this.vhost}?heartbeat=${HEARTBEAT}`];
|
|
355
413
|
};
|
|
356
414
|
|
|
357
415
|
const defaultUrls = findServers();
|
|
@@ -371,7 +429,7 @@ class RabbitMq implements IAfRabbitMq {
|
|
|
371
429
|
|
|
372
430
|
this.connection.on('connectFailed', (err) => {
|
|
373
431
|
this.consumersTags = [];
|
|
374
|
-
logger.error('rabbit: connection connectFailed', { err });
|
|
432
|
+
logger.error('rabbit: connection connectFailed', { err, advice: 'Check if the vhost exist', vhost: this.vhost });
|
|
375
433
|
if (!isResolved) {
|
|
376
434
|
isResolved = true;
|
|
377
435
|
reject(err);
|
|
@@ -463,11 +521,12 @@ class RabbitMq implements IAfRabbitMq {
|
|
|
463
521
|
return this.exchanges[exchangeName];
|
|
464
522
|
}
|
|
465
523
|
|
|
524
|
+
// TODO: [QUORUM-PHASE-2] Change the implementation to the new one
|
|
466
525
|
async getQueueLength(queue: string) {
|
|
467
526
|
RabbitMq.validateName('queue', queue);
|
|
468
|
-
const { channel } = this;
|
|
527
|
+
const { oldChannel: channel } = this;
|
|
469
528
|
if (!channel) {
|
|
470
|
-
throw new
|
|
529
|
+
throw new RabbitError('channel is not defined');
|
|
471
530
|
}
|
|
472
531
|
debug('rabbit: getting queue length', { queue, connected: this.connection?.isConnected() });
|
|
473
532
|
return channel?.checkQueue(queue);
|
|
@@ -482,28 +541,30 @@ class RabbitMq implements IAfRabbitMq {
|
|
|
482
541
|
return deleteQueueRes;
|
|
483
542
|
}
|
|
484
543
|
|
|
544
|
+
// TODO: [QUORUM-PHASE-2] Change the implementation to the new one
|
|
485
545
|
async bindQueue(queue: string, exchange: string) {
|
|
486
|
-
const channel: ChannelWrapper = await this.
|
|
546
|
+
const channel: ChannelWrapper = await this.assertChannelOld();
|
|
487
547
|
await channel.addSetup((setupChannel: ConfirmChannel) => setupChannel.bindQueue(queue, exchange, ''));
|
|
488
548
|
return channel.bindQueue(queue, exchange, '');
|
|
489
549
|
}
|
|
490
550
|
|
|
491
551
|
async setupQueue(queueName: string, options?: Options.AssertQueue): Promise<Replies.AssertQueue> {
|
|
492
552
|
let queue: Replies.AssertQueue;
|
|
493
|
-
const shouldUseQuorum = RabbitMq.shouldUseQuorum(queueName);
|
|
494
553
|
const localeOptions = {
|
|
495
554
|
...options,
|
|
496
555
|
durable: true,
|
|
497
556
|
arguments: {
|
|
498
557
|
...options?.arguments,
|
|
499
558
|
'x-consumer-timeout': 1000 * 60 * 60 * 24,
|
|
500
|
-
'x-queue-type':
|
|
559
|
+
'x-queue-type': 'quorum',
|
|
501
560
|
},
|
|
502
561
|
};
|
|
503
562
|
try {
|
|
504
563
|
const channel: ChannelWrapper = await this.assertChannel();
|
|
505
564
|
debug('assertQueue->channel.addSetup', { queueName });
|
|
506
|
-
await channel.addSetup((setupChannel: ConfirmChannel) =>
|
|
565
|
+
await channel.addSetup(async (setupChannel: ConfirmChannel) => {
|
|
566
|
+
await setupChannel.assertQueue(queueName, localeOptions);
|
|
567
|
+
});
|
|
507
568
|
debug('assertQueue->channel.assertQueue', { queueName });
|
|
508
569
|
queue = await channel.assertQueue(queueName, localeOptions);
|
|
509
570
|
} catch (e) {
|
|
@@ -526,6 +587,7 @@ class RabbitMq implements IAfRabbitMq {
|
|
|
526
587
|
return queue;
|
|
527
588
|
}
|
|
528
589
|
|
|
590
|
+
// TODO: [QUORUM-PHASE-3] Can be deleted after deleting the old consumers and publishers because all the queues are created us quorum queues
|
|
529
591
|
static shouldUseQuorum(queueName: string): boolean {
|
|
530
592
|
const envQuorumQueuesWhitelist = process.env.QUORUM_QUEUES_WHITELIST;
|
|
531
593
|
|
|
@@ -568,10 +630,27 @@ class RabbitMq implements IAfRabbitMq {
|
|
|
568
630
|
}
|
|
569
631
|
}
|
|
570
632
|
|
|
633
|
+
// Used by the microservices to consume messages from the queue
|
|
571
634
|
async consume(queue: string, callback: CallbackFunction, options?: ConsumeOptions): Promise<any> {
|
|
635
|
+
// TODO: [QUORUM-PHASE-3] Use only the implementation of consumeNew and delete consumeNew and consumeOld
|
|
636
|
+
if (options?.isQuorumQueue !== false) {
|
|
637
|
+
await this.assertVHost();
|
|
638
|
+
await this.consumeNew(queue, callback, options);
|
|
639
|
+
}
|
|
640
|
+
|
|
641
|
+
await this.consumeOld(queue, callback, options);
|
|
642
|
+
}
|
|
643
|
+
|
|
644
|
+
// TODO: [QUORUM-PHASE-3] Delete consumeNew we do not use it anymore
|
|
645
|
+
async consumeNew(queue: string, callback: CallbackFunction, options?: ConsumeOptions): Promise<any> {
|
|
572
646
|
await this.consumeFromRabbit(queue, callback, options);
|
|
573
647
|
}
|
|
574
648
|
|
|
649
|
+
// TODO: [QUORUM-PHASE-3] Delete consumeOld we do not use it anymore
|
|
650
|
+
async consumeOld(queue: string, callback: CallbackFunction, options?: ConsumeOptions): Promise<any> {
|
|
651
|
+
await this.consumeFromRabbitOld(queue, callback, options);
|
|
652
|
+
}
|
|
653
|
+
|
|
575
654
|
private async lockRedisIfNeeded(msg: any, options: any) {
|
|
576
655
|
const { properties: { headers } } = msg;
|
|
577
656
|
const timestamp = headers?.creationTimestamp;
|
|
@@ -602,7 +681,7 @@ class RabbitMq implements IAfRabbitMq {
|
|
|
602
681
|
} = optionsWithDefaults;
|
|
603
682
|
if (useConsumeWithLock) {
|
|
604
683
|
if (!this.redisLock) {
|
|
605
|
-
throw new
|
|
684
|
+
throw new RabbitError('Usage of consumeWithLock requires RedisInstance');
|
|
606
685
|
}
|
|
607
686
|
logger.info(`rabbit: Consuming with lock from queue ${queue} with lockTimeout: ${lockTimeout}ms`);
|
|
608
687
|
}
|
|
@@ -696,22 +775,45 @@ class RabbitMq implements IAfRabbitMq {
|
|
|
696
775
|
});
|
|
697
776
|
}
|
|
698
777
|
|
|
778
|
+
// Used by the microservices to consume messages from the exchange
|
|
699
779
|
async consumeFromExchange(queue: string, exchange: string, callback: CallbackFunction, options?: ConsumeOptions) : Promise<any> {
|
|
700
780
|
const optionsWithDefaults = { ...DEFAULT_OPTIONS, ...options };
|
|
701
781
|
RabbitMq.validateName('exchange', exchange);
|
|
702
782
|
RabbitMq.validateName('queue', queue);
|
|
703
783
|
const { limit, deadMessageTtl } = optionsWithDefaults;
|
|
704
|
-
|
|
705
|
-
|
|
784
|
+
// TODO: [QUORUM-PHASE-3] Delete the if statement after all the queues are created as quorum queues
|
|
785
|
+
if (options?.isQuorumQueue !== false) {
|
|
786
|
+
await this.assertVHost();
|
|
787
|
+
await this.saveConsumer(queue, callback, options);
|
|
788
|
+
const channel: ChannelWrapper = await this.getNewChannel({ name: `consume-exchange-${exchange}-queue-${queue}` });
|
|
789
|
+
|
|
790
|
+
await channel.addSetup(async (c: ConfirmChannel) => {
|
|
791
|
+
const assertExchange = await assertExchangeFanout(c, exchange);
|
|
792
|
+
await c.assertQueue(queue);
|
|
793
|
+
this.exchanges[exchange] = assertExchange;
|
|
794
|
+
await c.prefetch(limit, false);
|
|
795
|
+
return Promise.all([
|
|
796
|
+
c.bindQueue(queue, exchange, ''),
|
|
797
|
+
this.consumeNew(
|
|
798
|
+
queue,
|
|
799
|
+
callback,
|
|
800
|
+
options,
|
|
801
|
+
),
|
|
802
|
+
]);
|
|
803
|
+
});
|
|
804
|
+
}
|
|
706
805
|
|
|
707
|
-
|
|
806
|
+
// TODO: [QUORUM-PHASE-3] Delete the old implementation
|
|
807
|
+
await this.saveConsumerOld(queue, callback, options);
|
|
808
|
+
const channelOld: ChannelWrapper = await this.getNewChannelOld({ name: `consume-exchange-${exchange}-queue-${queue}-old` });
|
|
809
|
+
await channelOld.addSetup(async (c: ConfirmChannel) => {
|
|
708
810
|
const assertExchange = await assertExchangeFanout(c, exchange);
|
|
709
811
|
await c.assertQueue(queue);
|
|
710
|
-
this.
|
|
711
|
-
await c.prefetch(limit,
|
|
812
|
+
this.oldExchanges[exchange] = assertExchange;
|
|
813
|
+
await c.prefetch(limit, false);
|
|
712
814
|
return Promise.all([
|
|
713
815
|
c.bindQueue(queue, exchange, ''),
|
|
714
|
-
this.
|
|
816
|
+
this.consumeOld(
|
|
715
817
|
queue,
|
|
716
818
|
callback,
|
|
717
819
|
options,
|
|
@@ -720,17 +822,21 @@ class RabbitMq implements IAfRabbitMq {
|
|
|
720
822
|
});
|
|
721
823
|
}
|
|
722
824
|
|
|
825
|
+
// Used by the microservices to publish messages to the exchange
|
|
826
|
+
// TODO: [QUORUM-PHASE-2] Change the implementation to the new one
|
|
723
827
|
async publish(exchange: string, content: any, customHeaders?: any) : Promise<boolean> {
|
|
724
828
|
return wrapSetImmediate(async () => {
|
|
725
829
|
RabbitMq.validateName('exchange', exchange);
|
|
726
|
-
const channel: ChannelWrapper = await this.
|
|
727
|
-
await this.
|
|
830
|
+
const channel: ChannelWrapper = await this.assertChannelOld();
|
|
831
|
+
await this.assertExchangeOld(exchange);
|
|
728
832
|
await channel.publish(exchange, '',
|
|
729
833
|
Buffer.from(JSON.stringify(content)),
|
|
730
834
|
RabbitMq.getPublishOptions(customHeaders));
|
|
731
835
|
});
|
|
732
836
|
}
|
|
733
837
|
|
|
838
|
+
// Used by the microservices to send messages to the queue
|
|
839
|
+
// TODO: [QUORUM-PHASE-2] Change the implementation to the new one
|
|
734
840
|
async sendToQueue(
|
|
735
841
|
queue: string,
|
|
736
842
|
content: any,
|
|
@@ -738,7 +844,7 @@ class RabbitMq implements IAfRabbitMq {
|
|
|
738
844
|
customHeaders?: any,
|
|
739
845
|
): Promise<boolean | undefined> {
|
|
740
846
|
try {
|
|
741
|
-
await this.
|
|
847
|
+
await this.assertChannelOld();
|
|
742
848
|
} catch (e) {
|
|
743
849
|
logger.error(`rabbit sendToQueue: failed to send assert channel when sending to queue ${queue}`, { e });
|
|
744
850
|
throw e;
|
|
@@ -746,14 +852,14 @@ class RabbitMq implements IAfRabbitMq {
|
|
|
746
852
|
|
|
747
853
|
try {
|
|
748
854
|
RabbitMq.validateName('queue', queue);
|
|
749
|
-
await this.
|
|
855
|
+
await this.assertQueueOld(queue, options);
|
|
750
856
|
} catch (e) {
|
|
751
857
|
logger.error(`rabbit sendToQueue: failed to assert queue ${queue}`, { e });
|
|
752
858
|
throw e;
|
|
753
859
|
}
|
|
754
860
|
|
|
755
861
|
try {
|
|
756
|
-
const res = await this.
|
|
862
|
+
const res = await this.oldChannel?.sendToQueue(queue,
|
|
757
863
|
Buffer.from(JSON.stringify(content)),
|
|
758
864
|
RabbitMq.getPublishOptions(customHeaders));
|
|
759
865
|
debug(`rabbit: sending to queue ${queue}`, { res });
|
|
@@ -765,67 +871,66 @@ class RabbitMq implements IAfRabbitMq {
|
|
|
765
871
|
}
|
|
766
872
|
}
|
|
767
873
|
|
|
768
|
-
async
|
|
874
|
+
async sendToCeleryQueue(
|
|
875
|
+
data: TaskData,
|
|
876
|
+
taskName: string,
|
|
877
|
+
queueName: string,
|
|
878
|
+
): Promise<boolean | undefined> {
|
|
769
879
|
try {
|
|
770
|
-
const
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
const retries = 2;
|
|
778
|
-
const chunkSize = 10;
|
|
779
|
-
const processChunks = async (consumers: AfConsumer[]): Promise<boolean> => {
|
|
780
|
-
for (let i = 0; i < consumers.length; i += chunkSize) {
|
|
781
|
-
const chunk = consumers.slice(i, i + chunkSize);
|
|
782
|
-
// eslint-disable-next-line no-await-in-loop
|
|
783
|
-
const chunkResults = await Promise.all(
|
|
784
|
-
chunk.map((c: AfConsumer) => withRetry(() => channel.checkQueue(c.queue), retries, timeoutMs).catch((err) => {
|
|
785
|
-
logger.error(`rabbit: Error in isConnected (checkQueue) - ${err.message}`);
|
|
786
|
-
return null;
|
|
787
|
-
})),
|
|
788
|
-
);
|
|
789
|
-
if (chunkResults.some((res) => res === null)) {
|
|
790
|
-
return false;
|
|
791
|
-
}
|
|
792
|
-
}
|
|
793
|
-
return true;
|
|
880
|
+
const taskPayload = {
|
|
881
|
+
task: taskName,
|
|
882
|
+
id: randomUUID(),
|
|
883
|
+
args: [data],
|
|
884
|
+
kwargs: {},
|
|
885
|
+
retries: 0,
|
|
886
|
+
eta: null,
|
|
794
887
|
};
|
|
795
|
-
const waitForConnectResult = await withRetry(
|
|
796
|
-
() => channel.waitForConnect(),
|
|
797
|
-
retries,
|
|
798
|
-
timeoutMs,
|
|
799
|
-
).catch((err) => {
|
|
800
|
-
logger.error(`rabbit: Error in isConnected (waitForConnect) - ${err.message}`);
|
|
801
|
-
return null;
|
|
802
|
-
});
|
|
803
888
|
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
}
|
|
808
|
-
const allConsumersConnected = await processChunks(this.consumers);
|
|
809
|
-
if (!allConsumersConnected) {
|
|
810
|
-
logger.error('rabbit: isConnected - false due to failed consumer checks');
|
|
811
|
-
return false;
|
|
812
|
-
}
|
|
889
|
+
const headers = {
|
|
890
|
+
'content-type': 'application/json',
|
|
891
|
+
delivery_mode: 2,
|
|
892
|
+
};
|
|
813
893
|
|
|
814
|
-
|
|
815
|
-
return true;
|
|
894
|
+
return await this.sendToQueue(queueName, taskPayload, undefined, headers);
|
|
816
895
|
} catch (e) {
|
|
817
|
-
|
|
896
|
+
const isConnected = await this.isConnected();
|
|
897
|
+
logger.error(`rabbit sendToQueue: failed to send to queue ${queueName}, isConnected: ${isConnected}`, { e });
|
|
898
|
+
throw e;
|
|
899
|
+
}
|
|
900
|
+
}
|
|
901
|
+
|
|
902
|
+
// TODO: [QUORUM-PHASE-2] After changing the publish from old to new change from the old to the new implementation
|
|
903
|
+
async isConnected() : Promise<boolean> {
|
|
904
|
+
const connection = await this.getConnectionOld();
|
|
905
|
+
const isConnected = connection.isConnected();
|
|
906
|
+
if (!isConnected) {
|
|
907
|
+
logger.error('rabbit: isConnected - false');
|
|
908
|
+
return false;
|
|
909
|
+
}
|
|
910
|
+
const channel: any = await this.assertChannelOld();
|
|
911
|
+
try {
|
|
912
|
+
await Promise.all([
|
|
913
|
+
channel.waitForConnect(),
|
|
914
|
+
...this.oldConsumers.map((c: AfConsumer) => channel.checkQueue(c.queue)),
|
|
915
|
+
]);
|
|
916
|
+
} catch (e) {
|
|
917
|
+
logger.error('rabbit: isConnected - false');
|
|
818
918
|
return false;
|
|
819
919
|
}
|
|
920
|
+
logger.info('rabbit: isConnected - true');
|
|
921
|
+
return true;
|
|
820
922
|
}
|
|
821
923
|
|
|
822
924
|
async gracefulShutdown(signal: string) : Promise<void> {
|
|
823
|
-
|
|
925
|
+
// TODO: [QUORUM-PHASE-2] After changing the publish from old to new change from the old to the new implementation
|
|
926
|
+
const tagsNumber = this.consumersTags.length + this.oldConsumersTags.length;
|
|
824
927
|
logger.info(`rabbit: [gracefully-shutdown] received ${signal}! canceling #${tagsNumber} tags...`);
|
|
825
928
|
const cancelTagPromises = this.consumersTags.map(([channel, tag]) => channel.cancel(tag));
|
|
929
|
+
const cancelTagPromisesOld = this.oldConsumersTags.map(([channel, tag]) => channel.cancel(tag));
|
|
826
930
|
// Clean the array to avoid race
|
|
827
931
|
this.consumersTags = [];
|
|
828
|
-
|
|
932
|
+
this.oldConsumersTags = [];
|
|
933
|
+
const results = await Promise.allSettled([...cancelTagPromises, ...cancelTagPromisesOld]);
|
|
829
934
|
const rejected = results.filter((p) => p.status === 'rejected');
|
|
830
935
|
if (rejected.length > 0) {
|
|
831
936
|
logger.warn(`rabbit: [gracefully-shutdown] #${rejected.length}/${tagsNumber} tags failed to cancel: ${rejected}`);
|
|
@@ -833,6 +938,336 @@ class RabbitMq implements IAfRabbitMq {
|
|
|
833
938
|
logger.info('rabbit: [gracefully-shutdown] all tags successfully canceled.');
|
|
834
939
|
}
|
|
835
940
|
}
|
|
941
|
+
|
|
942
|
+
private async consumeFromRabbitOld(queue: string, callback: CallbackFunction, options?: ConsumeOptions): Promise<any> {
|
|
943
|
+
const optionsWithDefaults = { ...DEFAULT_OPTIONS, ...options };
|
|
944
|
+
RabbitMq.validateName('queue', queue);
|
|
945
|
+
this.saveConsumerOld(queue, callback, options);
|
|
946
|
+
const uniqueId = randomUUID();
|
|
947
|
+
const {
|
|
948
|
+
limit, deadMessageTtl, useConsumeWithLock, lockTimeout, auditContext, enableRabbitTrace,
|
|
949
|
+
} = optionsWithDefaults;
|
|
950
|
+
if (useConsumeWithLock) {
|
|
951
|
+
if (!this.redisLock) {
|
|
952
|
+
throw new RabbitError('Usage of consumeWithLock requires RedisInstance');
|
|
953
|
+
}
|
|
954
|
+
logger.info(`rabbit: Consuming with lock from queue ${queue} with lockTimeout: ${lockTimeout}ms`);
|
|
955
|
+
}
|
|
956
|
+
const channel = await this.getNewChannelOld({});
|
|
957
|
+
return channel.addSetup(async (confirmChannel: ConfirmChannel) => {
|
|
958
|
+
const q = await this.assertQueueOld(queue, optionsWithDefaults);
|
|
959
|
+
await confirmChannel.prefetch(limit, false);
|
|
960
|
+
const { consumerTag } = await confirmChannel.consume(
|
|
961
|
+
queue,
|
|
962
|
+
async (msg: ConsumeMessageOrNull) => {
|
|
963
|
+
if (!msg) {
|
|
964
|
+
return null;
|
|
965
|
+
}
|
|
966
|
+
|
|
967
|
+
const traceId = msg.properties.headers[TRACING_HEADER];
|
|
968
|
+
const userId = msg.properties.headers[USER_TRACING_HEADER];
|
|
969
|
+
const automationId = msg.properties.headers[AUTOMATION_ID_HEADER];
|
|
970
|
+
const parsedMessage = RabbitMq.parseMsg(msg);
|
|
971
|
+
const releaseLock = await this.lockRedisIfNeeded(parsedMessage, optionsWithDefaults);
|
|
972
|
+
const trace = newTrace(traceTypes.RABBIT);
|
|
973
|
+
// setting also outbreak trace as part of legacy code
|
|
974
|
+
const outbreakTrace = outbreak.newTrace(traceTypes.RABBIT);
|
|
975
|
+
// enableRabbitTrace is a flag to protect from different flows that doesn't work with permission
|
|
976
|
+
// and we don't want to fail the flow because of it
|
|
977
|
+
if (userId && enableRabbitTrace) {
|
|
978
|
+
try {
|
|
979
|
+
await Promise.all([
|
|
980
|
+
createOrSetRabbitTrace(trace, userId),
|
|
981
|
+
createOrSetRabbitTrace(outbreakTrace, userId),
|
|
982
|
+
]);
|
|
983
|
+
} catch (e) {
|
|
984
|
+
logger.error('rabbit: failed to setRabbitTrace', { userId, e });
|
|
985
|
+
return this.nack(confirmChannel, queue, optionsWithDefaults, { messageTtl: deadMessageTtl }, msg, releaseLock)(msg);
|
|
986
|
+
}
|
|
987
|
+
}
|
|
988
|
+
|
|
989
|
+
if (traceId) {
|
|
990
|
+
(trace as any)?.context?.set(TRACING_HEADER, traceId);
|
|
991
|
+
(outbreakTrace as any)?.context.set(TRACING_HEADER, traceId);
|
|
992
|
+
}
|
|
993
|
+
|
|
994
|
+
if (auditContext) {
|
|
995
|
+
await auditContext(queue, {
|
|
996
|
+
userId,
|
|
997
|
+
automationId,
|
|
998
|
+
});
|
|
999
|
+
}
|
|
1000
|
+
const shouldConsume = await this.shouldConsumeMessageByTimestamp(parsedMessage);
|
|
1001
|
+
if (!shouldConsume) {
|
|
1002
|
+
await this.unlockRedisIfNeeded(releaseLock);
|
|
1003
|
+
return this.ack(confirmChannel, msg)(msg);
|
|
1004
|
+
}
|
|
1005
|
+
|
|
1006
|
+
let messageAcked = false;
|
|
1007
|
+
// setting the localAck function to be used in the callback
|
|
1008
|
+
|
|
1009
|
+
const localAck = async () => {
|
|
1010
|
+
if (messageAcked) {
|
|
1011
|
+
return;
|
|
1012
|
+
}
|
|
1013
|
+
messageAcked = true;
|
|
1014
|
+
return this.ack(confirmChannel, msg, true, releaseLock)(msg);
|
|
1015
|
+
};
|
|
1016
|
+
|
|
1017
|
+
const localNack = async (_: ConsumeMessageOrNull, nackOptions: NackOptions = {}) => {
|
|
1018
|
+
if (messageAcked) {
|
|
1019
|
+
return;
|
|
1020
|
+
}
|
|
1021
|
+
debug('rabbit localNack', { messageAcked, uniqueId, deliveryTag: msg.fields.deliveryTag });
|
|
1022
|
+
messageAcked = true;
|
|
1023
|
+
return this.nack(confirmChannel, queue, optionsWithDefaults, { messageTtl: deadMessageTtl }, msg, releaseLock)(msg, nackOptions);
|
|
1024
|
+
};
|
|
1025
|
+
|
|
1026
|
+
try {
|
|
1027
|
+
await callback(
|
|
1028
|
+
parsedMessage,
|
|
1029
|
+
localAck,
|
|
1030
|
+
localNack,
|
|
1031
|
+
);
|
|
1032
|
+
} catch (e) {
|
|
1033
|
+
await localNack(msg);
|
|
1034
|
+
}
|
|
1035
|
+
}, CONSUMER_DEFAULT_OPTIONS,
|
|
1036
|
+
);
|
|
1037
|
+
if (!consumerTag) {
|
|
1038
|
+
logger.error(`rabbit: failed to consume from queue ${queue}`);
|
|
1039
|
+
} else {
|
|
1040
|
+
logger.info(`rabbit: adding tag ${consumerTag} to the array.`);
|
|
1041
|
+
this.oldConsumersTags.push([confirmChannel, consumerTag]);
|
|
1042
|
+
}
|
|
1043
|
+
});
|
|
1044
|
+
}
|
|
1045
|
+
|
|
1046
|
+
// TODO: [QUORUM-PHASE-3] Delete all the function under this line (getNewChannelOld, getConnectionOld, assertQueueOld, setupQueueOld, saveConsumerOld)
|
|
1047
|
+
async getNewChannelOld({ name = rand().toString(), onClose = null, options = {} }: newChannelOpts = {}) {
|
|
1048
|
+
let connection!: AmqpConnectionManager;
|
|
1049
|
+
try {
|
|
1050
|
+
connection = await this.getConnectionOld();
|
|
1051
|
+
} catch (e) {
|
|
1052
|
+
logger.error(`rabbit: error on get connection for new channel ${name} `, { e });
|
|
1053
|
+
throw e;
|
|
1054
|
+
}
|
|
1055
|
+
const channel = connection.createChannel({ ...options });
|
|
1056
|
+
once(channel, 'close').then((args) => {
|
|
1057
|
+
logger.error(`rabbit: channel ${name} closed`);
|
|
1058
|
+
onClose?.(args);
|
|
1059
|
+
});
|
|
1060
|
+
try {
|
|
1061
|
+
await once(channel, 'connect');
|
|
1062
|
+
debug(`rabbit: channel ${name} CONNECTED`);
|
|
1063
|
+
return channel;
|
|
1064
|
+
} catch (err) {
|
|
1065
|
+
logger.error(`rabbit: channel error ${name} error`, { err });
|
|
1066
|
+
throw err;
|
|
1067
|
+
}
|
|
1068
|
+
}
|
|
1069
|
+
|
|
1070
|
+
async getConnectionOld() {
|
|
1071
|
+
return new Promise<AmqpConnectionManager>(async (resolve, reject) => {
|
|
1072
|
+
if (this.oldBlockReconnect) {
|
|
1073
|
+
debug('rabbit: block reconnect');
|
|
1074
|
+
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
|
1075
|
+
// @ts-ignore
|
|
1076
|
+
return resolve();
|
|
1077
|
+
}
|
|
1078
|
+
if (this.oldConnection !== null) {
|
|
1079
|
+
if (this.options?.disableReconnect || this.oldConnection?.isConnected()) {
|
|
1080
|
+
debug('rabbit: connection - is connected');
|
|
1081
|
+
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
|
1082
|
+
// @ts-ignore
|
|
1083
|
+
return resolve(this.oldConnection);
|
|
1084
|
+
}
|
|
1085
|
+
debug('rabbit: connection - reconnecting');
|
|
1086
|
+
}
|
|
1087
|
+
if (this.oldCreatingConnection) {
|
|
1088
|
+
debug('rabbit: creating connection emi');
|
|
1089
|
+
this.oldEm.once(CONNECTION_CREATED_CONST, resolve);
|
|
1090
|
+
this.oldEm.once(CONNECTION_FAILED_CONST, reject);
|
|
1091
|
+
return;
|
|
1092
|
+
}
|
|
1093
|
+
this.oldCreatingConnection = true;
|
|
1094
|
+
let isResolved = false;
|
|
1095
|
+
|
|
1096
|
+
// It is import to use it as a function and not as a variable
|
|
1097
|
+
// because of k8s changes the env variables
|
|
1098
|
+
// and we want to use the new values
|
|
1099
|
+
const findServers = () => {
|
|
1100
|
+
const userName = process.env.RABBITMQ_USERNAME || 'guest';
|
|
1101
|
+
const password = process.env.RABBITMQ_PASSWORD || 'guest';
|
|
1102
|
+
const host = this.options?.rabbitHost || process.env.RABBITMQ_SERVICE_HOST || 'localhost';
|
|
1103
|
+
|
|
1104
|
+
debug('rabbit: creating connection', { host, userName, HEARTBEAT });
|
|
1105
|
+
|
|
1106
|
+
return [`amqp://${userName}:${password}@${host}?heartbeat=${HEARTBEAT}`];
|
|
1107
|
+
};
|
|
1108
|
+
|
|
1109
|
+
const defaultUrls = findServers();
|
|
1110
|
+
const connection: AmqpConnectionManager = await connect(defaultUrls, {
|
|
1111
|
+
findServers,
|
|
1112
|
+
});
|
|
1113
|
+
|
|
1114
|
+
this.oldConnection = connection;
|
|
1115
|
+
this.oldConnection.on('error', (err) => {
|
|
1116
|
+
logger.error('rabbit: connection error', { err });
|
|
1117
|
+
if (!isResolved) {
|
|
1118
|
+
isResolved = true;
|
|
1119
|
+
reject(err);
|
|
1120
|
+
this.oldEm.emit(CONNECTION_FAILED_CONST, err);
|
|
1121
|
+
}
|
|
1122
|
+
});
|
|
1123
|
+
|
|
1124
|
+
this.oldConnection.on('connectFailed', (err) => {
|
|
1125
|
+
this.oldConsumersTags = [];
|
|
1126
|
+
logger.error('rabbit: connection connectFailed', { err });
|
|
1127
|
+
if (!isResolved) {
|
|
1128
|
+
isResolved = true;
|
|
1129
|
+
reject(err);
|
|
1130
|
+
this.oldEm.emit(CONNECTION_FAILED_CONST, err);
|
|
1131
|
+
}
|
|
1132
|
+
});
|
|
1133
|
+
|
|
1134
|
+
this.oldConnection.on('disconnect', ({ err }) => {
|
|
1135
|
+
this.oldConsumersTags = [];
|
|
1136
|
+
debug('rabbit: connection closed');
|
|
1137
|
+
if (this.options?.disableReconnect) {
|
|
1138
|
+
logger.error(`${this.DISCONNECT_MSG}${err && ` - ${err}`}`);
|
|
1139
|
+
this.oldBlockReconnect = true;
|
|
1140
|
+
} else {
|
|
1141
|
+
logger.error(`${this.RECONNECT_MSG}${err && ` - ${err}`}`);
|
|
1142
|
+
}
|
|
1143
|
+
});
|
|
1144
|
+
|
|
1145
|
+
this.oldConnection.once('connect', async () => {
|
|
1146
|
+
debug('rabbit: connection established');
|
|
1147
|
+
this.oldCreatingConnection = false;
|
|
1148
|
+
this.oldEm.emit(CONNECTION_CREATED_CONST, connection);
|
|
1149
|
+
isResolved = true;
|
|
1150
|
+
resolve(connection);
|
|
1151
|
+
});
|
|
1152
|
+
});
|
|
1153
|
+
}
|
|
1154
|
+
|
|
1155
|
+
private saveConsumerOld(queue: string, callback: CallbackFunction, options: ConsumeOptions | undefined) {
|
|
1156
|
+
const isConsumerExist :boolean = this.oldConsumers.some((consumer) => consumer.queue === queue);
|
|
1157
|
+
if (!isConsumerExist) {
|
|
1158
|
+
logger.info(`rabbit: consumer: ${queue} saved in consumer array`);
|
|
1159
|
+
this.oldConsumers.push({
|
|
1160
|
+
queue,
|
|
1161
|
+
callback,
|
|
1162
|
+
options,
|
|
1163
|
+
});
|
|
1164
|
+
}
|
|
1165
|
+
}
|
|
1166
|
+
|
|
1167
|
+
async assertQueueOld(queueName: string, options?: Options.AssertQueue): Promise<any> {
|
|
1168
|
+
RabbitMq.validateName('queue', queueName);
|
|
1169
|
+
if (this.oldQueues[queueName]) {
|
|
1170
|
+
delete this.oldQueueSetupPromises[queueName];
|
|
1171
|
+
return this.oldQueues[queueName];
|
|
1172
|
+
}
|
|
1173
|
+
|
|
1174
|
+
if (this.oldQueueSetupPromises[queueName]) {
|
|
1175
|
+
return this.oldQueueSetupPromises[queueName];
|
|
1176
|
+
}
|
|
1177
|
+
|
|
1178
|
+
this.oldQueueSetupPromises[queueName] = this.setupQueueOld(queueName, options);
|
|
1179
|
+
return this.oldQueueSetupPromises[queueName];
|
|
1180
|
+
}
|
|
1181
|
+
|
|
1182
|
+
async setupQueueOld(queueName: string, options?: Options.AssertQueue): Promise<Replies.AssertQueue> {
|
|
1183
|
+
let queue: Replies.AssertQueue;
|
|
1184
|
+
const shouldUseQuorum = RabbitMq.shouldUseQuorum(queueName);
|
|
1185
|
+
const localeOptions = {
|
|
1186
|
+
...options,
|
|
1187
|
+
durable: true,
|
|
1188
|
+
arguments: {
|
|
1189
|
+
...options?.arguments,
|
|
1190
|
+
'x-consumer-timeout': 1000 * 60 * 60 * 24,
|
|
1191
|
+
'x-queue-type': shouldUseQuorum ? 'quorum' : 'classic',
|
|
1192
|
+
},
|
|
1193
|
+
};
|
|
1194
|
+
try {
|
|
1195
|
+
const channel: ChannelWrapper = await this.assertChannelOld();
|
|
1196
|
+
debug('assertQueue->channel.addSetup', { queueName });
|
|
1197
|
+
await channel.addSetup((setupChannel: ConfirmChannel) => setupChannel.assertQueue(queueName, localeOptions));
|
|
1198
|
+
debug('assertQueue->channel.assertQueue', { queueName });
|
|
1199
|
+
queue = await channel.assertQueue(queueName, localeOptions);
|
|
1200
|
+
} catch (e) {
|
|
1201
|
+
logger.error('rabbit: assertQueue error', { queueName, options, error: e });
|
|
1202
|
+
if (!this.options?.dontRetryAssert) {
|
|
1203
|
+
debug('retrying assertQueue', { queueName });
|
|
1204
|
+
const channel = await this.assertChannelOld({ force: true });
|
|
1205
|
+
await this.deleteQueueOld(queueName);
|
|
1206
|
+
|
|
1207
|
+
debug('retrying assertQueue->channel.addSetup', { queueName });
|
|
1208
|
+
await channel.addSetup((setupChannel: ConfirmChannel) => setupChannel.assertQueue(queueName, localeOptions));
|
|
1209
|
+
debug('retrying assertQueue->channel.assertQueue', { queueName });
|
|
1210
|
+
queue = await channel.assertQueue(queueName, localeOptions);
|
|
1211
|
+
} else {
|
|
1212
|
+
throw e;
|
|
1213
|
+
}
|
|
1214
|
+
}
|
|
1215
|
+
|
|
1216
|
+
this.oldQueues[queueName] = queue;
|
|
1217
|
+
return queue;
|
|
1218
|
+
}
|
|
1219
|
+
|
|
1220
|
+
async assertChannelOld({ force = false } : assertChannelOpts = {}): Promise<ChannelWrapper> {
|
|
1221
|
+
if (!this.oldPublishChannelSetupPromise) {
|
|
1222
|
+
this.oldPublishChannelSetupPromise = new Promise<ChannelWrapper>(async (resolve, reject) => {
|
|
1223
|
+
if (this.oldChannel && !force) {
|
|
1224
|
+
return resolve(this.oldChannel);
|
|
1225
|
+
}
|
|
1226
|
+
|
|
1227
|
+
try {
|
|
1228
|
+
const channel = await this.getNewChannelOld({});
|
|
1229
|
+
channel.on('error', (err) => {
|
|
1230
|
+
logger.error('rabbit: channel error', { err });
|
|
1231
|
+
});
|
|
1232
|
+
this.oldChannel = channel;
|
|
1233
|
+
resolve(channel);
|
|
1234
|
+
} catch (e) {
|
|
1235
|
+
reject(e);
|
|
1236
|
+
}
|
|
1237
|
+
});
|
|
1238
|
+
}
|
|
1239
|
+
return this.oldPublishChannelSetupPromise;
|
|
1240
|
+
}
|
|
1241
|
+
|
|
1242
|
+
private async deleteQueueOld(queue: string) {
|
|
1243
|
+
RabbitMq.validateName('queue', queue);
|
|
1244
|
+
const channel: ChannelWrapper = await this.assertChannelOld();
|
|
1245
|
+
logger.info('rabbit: deleting queue', { queue });
|
|
1246
|
+
const deleteQueueRes = await channel.deleteQueue(queue);
|
|
1247
|
+
debug('queue deleted', deleteQueueRes);
|
|
1248
|
+
return deleteQueueRes;
|
|
1249
|
+
}
|
|
1250
|
+
|
|
1251
|
+
async assertExchangeOld(exchangeName: string, options?: any) {
|
|
1252
|
+
const channel: ChannelWrapper = await this.assertChannelOld();
|
|
1253
|
+
|
|
1254
|
+
if (this.oldExchanges[exchangeName]) {
|
|
1255
|
+
delete this.oldAssertExchangePromises[exchangeName];
|
|
1256
|
+
return this.oldExchanges[exchangeName];
|
|
1257
|
+
}
|
|
1258
|
+
|
|
1259
|
+
if (this.oldAssertExchangePromises[exchangeName]) {
|
|
1260
|
+
return this.oldAssertExchangePromises[exchangeName];
|
|
1261
|
+
}
|
|
1262
|
+
|
|
1263
|
+
this.oldAssertExchangePromises[exchangeName] = assertExchangeFanout(channel, exchangeName);
|
|
1264
|
+
this.oldExchanges[exchangeName] = await this.oldAssertExchangePromises[exchangeName];
|
|
1265
|
+
return this.oldExchanges[exchangeName];
|
|
1266
|
+
}
|
|
836
1267
|
}
|
|
837
1268
|
|
|
838
1269
|
export default RabbitMq;
|
|
1270
|
+
|
|
1271
|
+
export {
|
|
1272
|
+
sendCeleryTaskViaHttp,
|
|
1273
|
+
} from './lib/celery';
|