@autofleet/rabbit 2.5.4 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.nvmrc +1 -0
- package/dist/index.d.ts +35 -32
- package/dist/index.js +164 -92
- package/dist/lib/consts.d.ts +13 -0
- package/dist/lib/consts.js +16 -0
- package/dist/lib/types.d.ts +35 -0
- package/dist/lib/types.js +11 -0
- package/dist/lib/utils.d.ts +5 -0
- package/dist/lib/utils.js +16 -0
- package/package.json +1 -1
- package/src/index.ts +220 -112
- package/src/lib/consts.ts +13 -0
- package/src/lib/types.ts +51 -0
- package/src/lib/utils.ts +15 -0
- /package/dist/{rabbitError.d.ts → lib/rabbitError.d.ts} +0 -0
- /package/dist/{rabbitError.js → lib/rabbitError.js} +0 -0
- /package/dist/{redis.d.ts → lib/redis.d.ts} +0 -0
- /package/dist/{redis.js → lib/redis.js} +0 -0
- /package/src/{rabbitError.ts → lib/rabbitError.ts} +0 -0
- /package/src/{redis.ts → lib/redis.ts} +0 -0
package/.nvmrc
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
v16
|
package/dist/index.d.ts
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
|
-
import { ConfirmChannel, Options, ConsumeMessage } from 'amqplib';
|
|
3
|
-
import { AmqpConnectionManager, ChannelWrapper } from 'amqp-connection-manager';
|
|
4
2
|
import { EventEmitter } from 'events';
|
|
5
|
-
import {
|
|
3
|
+
import { AmqpConnectionManager, ChannelWrapper } from 'amqp-connection-manager';
|
|
4
|
+
import { ConfirmChannel, ConsumeMessage, Options, Replies } from 'amqplib';
|
|
5
|
+
import { RedisConfig } from './lib/redis';
|
|
6
|
+
import { TRACING_HEADER } from './lib/consts';
|
|
7
|
+
import { CallbackFunction, ConsumeMessageOrNull, ConsumeOptions, CustomMessageHeaders, QueuesCache, RedisLockType, ExchangesCache } from './lib/types';
|
|
6
8
|
export interface IAfRabbitMq {
|
|
7
9
|
ack: any;
|
|
8
10
|
nack: any;
|
|
@@ -15,29 +17,27 @@ export interface IAfRabbitMq {
|
|
|
15
17
|
sendToQueue: any;
|
|
16
18
|
redisClient?: any;
|
|
17
19
|
}
|
|
18
|
-
interface ExchangesCache {
|
|
19
|
-
[key: string]: any;
|
|
20
|
-
}
|
|
21
|
-
interface QueuesCache {
|
|
22
|
-
[key: string]: any;
|
|
23
|
-
}
|
|
24
|
-
declare type CustomMessageHeaders = {
|
|
25
|
-
redisTimestampValidationKey?: string;
|
|
26
|
-
};
|
|
27
|
-
declare type RedisLockType = (args0?: string, arg1?: number) => Promise<any>;
|
|
28
|
-
declare type ConsumeMessageOrNull = ConsumeMessage | null;
|
|
29
|
-
interface ConsumeOptions {
|
|
30
|
-
retries?: number;
|
|
31
|
-
deadMessageTtl?: number;
|
|
32
|
-
limit?: number;
|
|
33
|
-
lockTimeout?: number;
|
|
34
|
-
useConsumeWithLock?: boolean;
|
|
35
|
-
}
|
|
36
|
-
declare type CallbackFunction = (msg: ConsumeMessage, ack: Function, nack: Function) => Promise<any>;
|
|
37
20
|
export interface AfRabbitOptions {
|
|
38
|
-
disableReconnect
|
|
21
|
+
disableReconnect?: boolean;
|
|
22
|
+
/**
|
|
23
|
+
* When you want your own grace-full shutdown, set this to true.
|
|
24
|
+
* @default false
|
|
25
|
+
*/
|
|
26
|
+
dontGracefulShutdown?: boolean;
|
|
27
|
+
/**
|
|
28
|
+
* dont retry on creation error
|
|
29
|
+
* @default false
|
|
30
|
+
*/
|
|
31
|
+
dontRetryAssert?: boolean;
|
|
39
32
|
}
|
|
40
|
-
declare
|
|
33
|
+
declare type newChannelOpts = {
|
|
34
|
+
name?: string;
|
|
35
|
+
onClose?: null | ((args: any | null) => void);
|
|
36
|
+
};
|
|
37
|
+
declare type assertChannelOpts = {
|
|
38
|
+
channelName?: string;
|
|
39
|
+
force?: boolean;
|
|
40
|
+
};
|
|
41
41
|
declare class RabbitMq implements IAfRabbitMq {
|
|
42
42
|
static parseMsg(msg: any): any;
|
|
43
43
|
static validateName(type: string, name: string): void;
|
|
@@ -49,12 +49,11 @@ declare class RabbitMq implements IAfRabbitMq {
|
|
|
49
49
|
creationTimestamp: number;
|
|
50
50
|
};
|
|
51
51
|
};
|
|
52
|
-
PUBLISH_TIMEOUT: number;
|
|
53
|
-
PUBLISH_ERROR_MSG: string;
|
|
54
52
|
DISCONNECT_MSG: string;
|
|
55
|
-
|
|
53
|
+
RECONNECT_MSG: string;
|
|
56
54
|
channel: ChannelWrapper | null;
|
|
57
|
-
|
|
55
|
+
blockReconnect: boolean | null | undefined;
|
|
56
|
+
connection: AmqpConnectionManager | null | undefined;
|
|
58
57
|
em: EventEmitter;
|
|
59
58
|
creatingConnection: boolean;
|
|
60
59
|
exchanges: ExchangesCache;
|
|
@@ -64,24 +63,28 @@ declare class RabbitMq implements IAfRabbitMq {
|
|
|
64
63
|
redisLock?: RedisLockType;
|
|
65
64
|
/** Array of consumers tags used for canceling consumption */
|
|
66
65
|
consumersTags: Array<[ConfirmChannel, string]>;
|
|
66
|
+
private consumers;
|
|
67
67
|
constructor(options?: AfRabbitOptions, redisConfig?: RedisConfig);
|
|
68
68
|
private shouldConsumeMessageByTimestamp;
|
|
69
69
|
ack: (channel: ChannelWrapper, msg: ConsumeMessageOrNull, shouldUpdateRedisTimestamp?: boolean, releaseLock?: null) => (userMsg: ConsumeMessage) => Promise<any>;
|
|
70
70
|
nack: (channel: ChannelWrapper, queue: string, options: any, deadQueueOptions: Options.AssertQueue, msg: ConsumeMessageOrNull, releaseLock: any) => (userMsg: ConsumeMessageOrNull, { skipRetry, }?: any) => Promise<any>;
|
|
71
71
|
getConnection(): Promise<AmqpConnectionManager>;
|
|
72
|
-
getNewChannel(): Promise<ChannelWrapper>;
|
|
73
|
-
assertChannel(): Promise<ChannelWrapper>;
|
|
72
|
+
getNewChannel({ name, onClose }?: newChannelOpts): Promise<ChannelWrapper>;
|
|
73
|
+
assertChannel({ force }?: assertChannelOpts): Promise<ChannelWrapper>;
|
|
74
74
|
assertExchange(exchangeName: string, options?: any): Promise<any>;
|
|
75
|
-
getQueueLength(queue: string): Promise<
|
|
75
|
+
getQueueLength(queue: string): Promise<Replies.AssertQueue>;
|
|
76
|
+
private deleteQueue;
|
|
76
77
|
bindQueue(queue: string, exchange: string): Promise<void>;
|
|
77
78
|
assertQueue(queueName: string, options?: Options.AssertQueue): Promise<any>;
|
|
79
|
+
private saveConsumer;
|
|
80
|
+
private loadConsumers;
|
|
78
81
|
consume(queue: string, callback: CallbackFunction, options?: ConsumeOptions): Promise<any>;
|
|
79
82
|
private lockRedisIfNeeded;
|
|
80
83
|
private unlockRedisIfNeeded;
|
|
81
84
|
private consumeFromRabbit;
|
|
82
85
|
consumeFromExchange(queue: string, exchange: string, callback: CallbackFunction, options?: ConsumeOptions): Promise<any>;
|
|
83
86
|
publish(exchange: string, content: any, customHeaders?: any): Promise<boolean>;
|
|
84
|
-
sendToQueue(queue: string, content: any, options?: any, customHeaders?: any): Promise<boolean>;
|
|
87
|
+
sendToQueue(queue: string, content: any, options?: any, customHeaders?: any, isBlocking?: boolean): Promise<boolean | undefined>;
|
|
85
88
|
isConnected(): Promise<boolean>;
|
|
86
89
|
gracefulShutdown(signal: string): Promise<void>;
|
|
87
90
|
}
|
package/dist/index.js
CHANGED
|
@@ -1,50 +1,34 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
/* eslint-disable no-empty,consistent-return,no-async-promise-executor,@typescript-eslint/no-unused-vars */
|
|
2
3
|
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
4
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
5
|
};
|
|
5
6
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
|
|
7
|
-
|
|
7
|
+
const events_1 = require("events");
|
|
8
|
+
const util_1 = require("util");
|
|
8
9
|
const moment_1 = __importDefault(require("moment"));
|
|
9
10
|
const redis_lock_1 = __importDefault(require("redis-lock"));
|
|
10
11
|
const amqp_connection_manager_1 = require("amqp-connection-manager");
|
|
11
|
-
const events_1 = require("events");
|
|
12
|
-
const util_1 = require("util");
|
|
13
12
|
const outbreak_1 = require("@autofleet/outbreak");
|
|
14
13
|
const logger_1 = __importDefault(require("./logger"));
|
|
15
|
-
const rabbitError_1 = __importDefault(require("./rabbitError"));
|
|
16
|
-
const redis_1 = __importDefault(require("./redis"));
|
|
17
|
-
const
|
|
18
|
-
const
|
|
19
|
-
const
|
|
20
|
-
const
|
|
21
|
-
const
|
|
22
|
-
const
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
lockTimeout: DEFAULT_LOCK_TIMEOUT,
|
|
27
|
-
useConsumeWithLock: DEFAULT_USE_CONSUME_WITH_LOCK,
|
|
28
|
-
};
|
|
29
|
-
const assertExchangeFanout = async (c, exchangeName) => c.assertExchange(exchangeName, 'fanout');
|
|
30
|
-
const connectionCreatedConst = 'connectionCreated';
|
|
31
|
-
const wrapSetImmediate = (callback) => new Promise(async (resolve, reject) => {
|
|
32
|
-
setImmediate(async () => {
|
|
33
|
-
try {
|
|
34
|
-
const value = await callback();
|
|
35
|
-
resolve(value);
|
|
36
|
-
}
|
|
37
|
-
catch (error) {
|
|
38
|
-
reject(error);
|
|
39
|
-
}
|
|
40
|
-
});
|
|
41
|
-
});
|
|
14
|
+
const rabbitError_1 = __importDefault(require("./lib/rabbitError"));
|
|
15
|
+
const redis_1 = __importDefault(require("./lib/redis"));
|
|
16
|
+
const utils_1 = require("./lib/utils");
|
|
17
|
+
const consts_1 = require("./lib/consts");
|
|
18
|
+
const types_1 = require("./lib/types");
|
|
19
|
+
// const debug = nodeDebug('af-rabbitmq')
|
|
20
|
+
const debug = logger_1.default.info;
|
|
21
|
+
const USERNAME = process.env.RABBITMQ_USERNAME || 'guest';
|
|
22
|
+
const PASSWORD = process.env.RABBITMQ_PASSWORD || 'guest';
|
|
23
|
+
const HOST = process.env.RABBITMQ_SERVICE_HOST || 'localhost';
|
|
24
|
+
const HEARTBEAT = '60';
|
|
42
25
|
class RabbitMq {
|
|
43
26
|
constructor(options, redisConfig) {
|
|
44
|
-
|
|
45
|
-
|
|
27
|
+
// PUBLISH_TIMEOUT = Number(process.env.RABBITMQ_PUBLISH_TIMEOUT) || 60000;
|
|
28
|
+
// PUBLISH_ERROR_MSG = `rabbit: publish timeout(${this.PUBLISH_TIMEOUT}ms) has pass, exchange: `;
|
|
46
29
|
this.DISCONNECT_MSG = 'rabbit: connection disconnect';
|
|
47
|
-
this.
|
|
30
|
+
this.RECONNECT_MSG = 'rabbit: connection disconnect - reconnecting';
|
|
31
|
+
this.consumers = [];
|
|
48
32
|
this.shouldConsumeMessageByTimestamp = async (msg) => {
|
|
49
33
|
if (msg) {
|
|
50
34
|
const { properties: { headers } } = msg;
|
|
@@ -73,12 +57,12 @@ class RabbitMq {
|
|
|
73
57
|
await this.unlockRedisIfNeeded(releaseLock);
|
|
74
58
|
if (channel && msg) {
|
|
75
59
|
if (!skipRetry
|
|
76
|
-
&& (!msg.properties.headers[RETRY_HEADER]
|
|
77
|
-
|| parseInt(msg.properties.headers[RETRY_HEADER], 10) < options.retries)) {
|
|
60
|
+
&& (!msg.properties.headers[consts_1.RETRY_HEADER]
|
|
61
|
+
|| parseInt(msg.properties.headers[consts_1.RETRY_HEADER], 10) < options.retries)) {
|
|
78
62
|
await this.sendToQueue(queue, RabbitMq.parseMsg(msg).content, options, {
|
|
79
63
|
...msg.properties.headers,
|
|
80
|
-
[RETRY_HEADER]: msg.properties.headers[RETRY_HEADER]
|
|
81
|
-
? msg.properties.headers[RETRY_HEADER] + 1
|
|
64
|
+
[consts_1.RETRY_HEADER]: msg.properties.headers[consts_1.RETRY_HEADER]
|
|
65
|
+
? msg.properties.headers[consts_1.RETRY_HEADER] + 1
|
|
82
66
|
: 1,
|
|
83
67
|
});
|
|
84
68
|
}
|
|
@@ -86,8 +70,8 @@ class RabbitMq {
|
|
|
86
70
|
const deadQueue = `${queue}-dead`;
|
|
87
71
|
await this.sendToQueue(deadQueue, RabbitMq.parseMsg(msg).content, deadQueueOptions, {
|
|
88
72
|
...msg.properties.headers,
|
|
89
|
-
[RETRY_HEADER]: msg.properties.headers[RETRY_HEADER]
|
|
90
|
-
? msg.properties.headers[RETRY_HEADER] + 1
|
|
73
|
+
[consts_1.RETRY_HEADER]: msg.properties.headers[consts_1.RETRY_HEADER]
|
|
74
|
+
? msg.properties.headers[consts_1.RETRY_HEADER] + 1
|
|
91
75
|
: 1,
|
|
92
76
|
});
|
|
93
77
|
}
|
|
@@ -113,12 +97,14 @@ class RabbitMq {
|
|
|
113
97
|
}
|
|
114
98
|
this.consumersTags = [];
|
|
115
99
|
logger_1.default.info(`rabbit: [gracefully-shutdown] adding gracefully shutdown for process.pid ${process.pid}`);
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
100
|
+
if (!this.options?.dontGracefulShutdown) {
|
|
101
|
+
process.on('SIGTERM', async () => {
|
|
102
|
+
await this.gracefulShutdown('SIGTERM');
|
|
103
|
+
});
|
|
104
|
+
process.on('SIGINT', async () => {
|
|
105
|
+
await this.gracefulShutdown('SIGINT');
|
|
106
|
+
});
|
|
107
|
+
}
|
|
122
108
|
}
|
|
123
109
|
static parseMsg(msg) {
|
|
124
110
|
let { content } = msg;
|
|
@@ -144,57 +130,91 @@ class RabbitMq {
|
|
|
144
130
|
headers: {
|
|
145
131
|
creationTimestamp: moment_1.default().valueOf(),
|
|
146
132
|
...customHeaders,
|
|
147
|
-
[TRACING_HEADER]: trace?.context?.get(TRACING_HEADER),
|
|
133
|
+
[consts_1.TRACING_HEADER]: trace?.context?.get(consts_1.TRACING_HEADER),
|
|
148
134
|
},
|
|
149
135
|
};
|
|
150
136
|
}
|
|
151
137
|
async getConnection() {
|
|
152
138
|
return new Promise(async (resolve) => {
|
|
139
|
+
if (this.blockReconnect) {
|
|
140
|
+
debug('rabbit: block reconnect');
|
|
141
|
+
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
|
142
|
+
// @ts-ignore
|
|
143
|
+
return resolve();
|
|
144
|
+
}
|
|
153
145
|
if (this.connection !== null) {
|
|
154
|
-
|
|
146
|
+
if (this.options?.disableReconnect || this.connection?.isConnected()) {
|
|
147
|
+
debug('rabbit: connection - is connected');
|
|
148
|
+
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
|
149
|
+
// @ts-ignore
|
|
150
|
+
return resolve(this.connection);
|
|
151
|
+
}
|
|
152
|
+
debug('rabbit: connection - reconnecting');
|
|
155
153
|
}
|
|
156
154
|
if (this.creatingConnection) {
|
|
157
|
-
|
|
155
|
+
debug('rabbit: creating connection emi');
|
|
156
|
+
this.em.once(consts_1.CONNECTION_CREATED_CONST, resolve);
|
|
158
157
|
return;
|
|
159
158
|
}
|
|
160
159
|
this.creatingConnection = true;
|
|
161
|
-
|
|
162
|
-
const
|
|
163
|
-
const password = process.env.RABBITMQ_PASSWORD || 'guest';
|
|
164
|
-
const host = process.env.RABBITMQ_SERVICE_HOST || '';
|
|
165
|
-
const connection = await amqp_connection_manager_1.connect([`amqp://${username}:${password}@${host}?heartbeat=60`]);
|
|
160
|
+
debug('rabbit: creating connection', { HOST, USERNAME, HEARTBEAT });
|
|
161
|
+
const connection = await amqp_connection_manager_1.connect([`amqp://${USERNAME}:${PASSWORD}@${HOST}?heartbeat=${HEARTBEAT}}`]);
|
|
166
162
|
this.connection = connection;
|
|
167
163
|
this.connection.on('disconnect', ({ err }) => {
|
|
164
|
+
debug('rabbit: connection closed');
|
|
168
165
|
this.exchanges = {};
|
|
169
166
|
this.queues = {};
|
|
167
|
+
this.connection = null;
|
|
168
|
+
this.channel = null;
|
|
170
169
|
if (this.options?.disableReconnect) {
|
|
171
170
|
logger_1.default.error(`${this.DISCONNECT_MSG}${err && ` - ${err}`}`);
|
|
171
|
+
this.blockReconnect = true;
|
|
172
172
|
}
|
|
173
173
|
else {
|
|
174
|
-
logger_1.default.error(`${this.
|
|
175
|
-
this.connection = null;
|
|
174
|
+
logger_1.default.error(`${this.RECONNECT_MSG}${err && ` - ${err}`}`);
|
|
176
175
|
}
|
|
177
176
|
});
|
|
178
|
-
this.
|
|
179
|
-
|
|
180
|
-
|
|
177
|
+
this.connection.once('connect', async () => {
|
|
178
|
+
debug('rabbit: connection established');
|
|
179
|
+
await this.loadConsumers();
|
|
180
|
+
this.creatingConnection = false;
|
|
181
|
+
this.em.emit(consts_1.CONNECTION_CREATED_CONST, connection);
|
|
182
|
+
resolve(connection);
|
|
183
|
+
});
|
|
181
184
|
});
|
|
182
185
|
}
|
|
183
|
-
async getNewChannel() {
|
|
184
|
-
|
|
185
|
-
|
|
186
|
+
async getNewChannel({ name = utils_1.rand().toString(), onClose = null } = {}) {
|
|
187
|
+
return new Promise(async (resolve, reject) => {
|
|
188
|
+
const connection = await this.getConnection();
|
|
189
|
+
const channel = connection.createChannel({});
|
|
190
|
+
channel.on('error', (err) => {
|
|
191
|
+
logger_1.default.error(`rabbit: channel ${name} error`, { err });
|
|
192
|
+
});
|
|
193
|
+
channel.on('close', (...args) => {
|
|
194
|
+
logger_1.default.error(`rabbit: channel ${name} closed`, { args });
|
|
195
|
+
if (onClose) {
|
|
196
|
+
onClose(args);
|
|
197
|
+
}
|
|
198
|
+
});
|
|
199
|
+
channel.once('connect', () => {
|
|
200
|
+
debug(`rabbit: channel ${name} CONNECTED`);
|
|
201
|
+
resolve(channel);
|
|
202
|
+
});
|
|
203
|
+
});
|
|
186
204
|
}
|
|
187
|
-
async assertChannel() {
|
|
205
|
+
async assertChannel({ force = false } = {}) {
|
|
188
206
|
return new Promise(async (resolve, reject) => {
|
|
189
|
-
if (this.channel) {
|
|
207
|
+
if (this.channel && !force) {
|
|
190
208
|
return resolve(this.channel);
|
|
191
209
|
}
|
|
192
210
|
try {
|
|
193
|
-
const
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
211
|
+
const channel = await this.getNewChannel({
|
|
212
|
+
onClose: () => {
|
|
213
|
+
this.channel = null;
|
|
214
|
+
},
|
|
215
|
+
});
|
|
216
|
+
this.channel = channel;
|
|
217
|
+
resolve(channel);
|
|
198
218
|
}
|
|
199
219
|
catch (e) {
|
|
200
220
|
reject(e);
|
|
@@ -206,42 +226,88 @@ class RabbitMq {
|
|
|
206
226
|
if (this.exchanges[exchangeName]) {
|
|
207
227
|
return this.exchanges[exchangeName];
|
|
208
228
|
}
|
|
209
|
-
const exchange = await assertExchangeFanout(channel, exchangeName);
|
|
229
|
+
const exchange = await utils_1.assertExchangeFanout(channel, exchangeName);
|
|
210
230
|
this.exchanges[exchangeName] = exchange;
|
|
211
231
|
return exchange;
|
|
212
232
|
}
|
|
213
233
|
async getQueueLength(queue) {
|
|
214
234
|
RabbitMq.validateName('queue', queue);
|
|
215
235
|
const channel = await this.assertChannel();
|
|
216
|
-
|
|
236
|
+
debug('rabbit: getting queue length', { queue, connected: this.connection?.isConnected() });
|
|
217
237
|
return channel.checkQueue(queue);
|
|
218
238
|
}
|
|
239
|
+
async deleteQueue(queue) {
|
|
240
|
+
return new Promise(async (resolve, reject) => {
|
|
241
|
+
RabbitMq.validateName('queue', queue);
|
|
242
|
+
const channel = await this.assertChannel();
|
|
243
|
+
logger_1.default.info('rabbit: deleting queue', { queue });
|
|
244
|
+
const deleteQueueRes = await channel.deleteQueue(queue);
|
|
245
|
+
debug('queue deleted', deleteQueueRes);
|
|
246
|
+
resolve(deleteQueueRes);
|
|
247
|
+
});
|
|
248
|
+
}
|
|
219
249
|
async bindQueue(queue, exchange) {
|
|
220
250
|
const channel = await this.assertChannel();
|
|
221
251
|
await channel.addSetup((setupChannel) => setupChannel.bindQueue(queue, exchange, ''));
|
|
222
|
-
// @ts-ignore
|
|
223
252
|
return channel.bindQueue(queue, exchange, '');
|
|
224
253
|
}
|
|
225
254
|
async assertQueue(queueName, options) {
|
|
255
|
+
let queue = null;
|
|
226
256
|
RabbitMq.validateName('queue', queueName);
|
|
227
|
-
const channel = await this.assertChannel();
|
|
228
257
|
if (this.queues[queueName]) {
|
|
229
258
|
return this.queues[queueName];
|
|
230
259
|
}
|
|
231
|
-
|
|
232
|
-
|
|
260
|
+
try {
|
|
261
|
+
const channel = await this.assertChannel();
|
|
262
|
+
debug('assertQueue->channel.addSetup', { queueName });
|
|
263
|
+
await channel.addSetup((setupChannel) => setupChannel.assertQueue(queueName, options));
|
|
264
|
+
debug('assertQueue->channel.assertQueue', { queueName });
|
|
265
|
+
queue = await channel.assertQueue(queueName, options);
|
|
266
|
+
}
|
|
267
|
+
catch (e) {
|
|
268
|
+
logger_1.default.error('rabbit: assertQueue error', { queueName, options, error: e });
|
|
269
|
+
if (!this.options?.dontRetryAssert) {
|
|
270
|
+
debug('retrying assertQueue', { queueName });
|
|
271
|
+
const channel = await this.assertChannel({ force: true });
|
|
272
|
+
await this.deleteQueue(queueName);
|
|
273
|
+
debug('1assertQueue->channel.addSetup', { queueName });
|
|
274
|
+
await channel.addSetup((setupChannel) => setupChannel.assertQueue(queueName, options));
|
|
275
|
+
debug('1assertQueue->channel.assertQueue', { queueName });
|
|
276
|
+
queue = await channel.assertQueue(queueName, options);
|
|
277
|
+
}
|
|
278
|
+
else {
|
|
279
|
+
throw e;
|
|
280
|
+
}
|
|
281
|
+
}
|
|
233
282
|
this.queues[queueName] = queueName;
|
|
234
283
|
return queue;
|
|
235
284
|
}
|
|
285
|
+
saveConsumer(queue, callback, options) {
|
|
286
|
+
this.consumers.push({
|
|
287
|
+
queue,
|
|
288
|
+
callback,
|
|
289
|
+
options,
|
|
290
|
+
});
|
|
291
|
+
}
|
|
292
|
+
async loadConsumers() {
|
|
293
|
+
debug('rabbit: loading consumers', { consumers: this.consumers.length });
|
|
294
|
+
if (this.consumers.length > 0) {
|
|
295
|
+
await Promise.all(this.consumers.map((consumer) => this
|
|
296
|
+
.consumeFromRabbit(consumer.queue, consumer.callback, consumer.options)));
|
|
297
|
+
}
|
|
298
|
+
}
|
|
236
299
|
async consume(queue, callback, options) {
|
|
237
|
-
|
|
300
|
+
if (this.connection && !this.creatingConnection) {
|
|
301
|
+
this.consumeFromRabbit(queue, callback, options);
|
|
302
|
+
}
|
|
303
|
+
return this.saveConsumer(queue, callback, options);
|
|
238
304
|
}
|
|
239
305
|
async lockRedisIfNeeded(msg, options) {
|
|
240
306
|
const { properties: { headers } } = msg;
|
|
241
307
|
const timestamp = headers?.creationTimestamp;
|
|
242
308
|
let releaseLock = null;
|
|
243
309
|
if (options.useConsumeWithLock && timestamp && headers?.redisTimestampValidationKey && this.redisLock) {
|
|
244
|
-
releaseLock = await this.redisLock(headers.redisTimestampValidationKey, options?.lockTimeout || DEFAULT_LOCK_TIMEOUT);
|
|
310
|
+
releaseLock = await this.redisLock(headers.redisTimestampValidationKey, options?.lockTimeout || consts_1.DEFAULT_LOCK_TIMEOUT);
|
|
245
311
|
}
|
|
246
312
|
return releaseLock;
|
|
247
313
|
}
|
|
@@ -251,7 +317,7 @@ class RabbitMq {
|
|
|
251
317
|
}
|
|
252
318
|
}
|
|
253
319
|
async consumeFromRabbit(queue, callback, options) {
|
|
254
|
-
const optionsWithDefaults = { ...
|
|
320
|
+
const optionsWithDefaults = { ...consts_1.DEFAULT_OPTIONS, ...options };
|
|
255
321
|
RabbitMq.validateName('queue', queue);
|
|
256
322
|
const { limit, deadMessageTtl, useConsumeWithLock, lockTimeout, } = optionsWithDefaults;
|
|
257
323
|
if (useConsumeWithLock) {
|
|
@@ -260,18 +326,18 @@ class RabbitMq {
|
|
|
260
326
|
}
|
|
261
327
|
logger_1.default.info(`rabbit: Consuming with lock from queue ${queue} with lockTimeout: ${lockTimeout}ms`);
|
|
262
328
|
}
|
|
263
|
-
const channel = await this.getNewChannel();
|
|
329
|
+
const channel = await this.getNewChannel({ name: `consume-queue-${queue}` });
|
|
264
330
|
return channel.addSetup(async (c) => {
|
|
265
|
-
await c.assertQueue(queue);
|
|
331
|
+
await c.assertQueue(queue, options ? { messageTtl: options.messageTtl } : undefined);
|
|
266
332
|
await c.prefetch(limit, true);
|
|
267
333
|
const { consumerTag } = await c.consume(queue, async (msg) => {
|
|
268
334
|
if (!msg) {
|
|
269
335
|
return null;
|
|
270
336
|
}
|
|
271
|
-
const traceId = msg.properties.headers[TRACING_HEADER];
|
|
337
|
+
const traceId = msg.properties.headers[consts_1.TRACING_HEADER];
|
|
272
338
|
if (traceId) {
|
|
273
339
|
const trace = outbreak_1.newTrace(outbreak_1.traceTypes.RABBIT);
|
|
274
|
-
trace?.context?.set(TRACING_HEADER, traceId);
|
|
340
|
+
trace?.context?.set(consts_1.TRACING_HEADER, traceId);
|
|
275
341
|
}
|
|
276
342
|
const parsedMessage = RabbitMq.parseMsg(msg);
|
|
277
343
|
const releaseLock = await this.lockRedisIfNeeded(parsedMessage, optionsWithDefaults);
|
|
@@ -286,7 +352,7 @@ class RabbitMq {
|
|
|
286
352
|
catch (e) {
|
|
287
353
|
await this.nack(channel, queue, optionsWithDefaults, { messageTtl: deadMessageTtl }, msg, releaseLock)(msg);
|
|
288
354
|
}
|
|
289
|
-
});
|
|
355
|
+
}, types_1.CONSUMER_DEFAULT_OPTIONS);
|
|
290
356
|
if (!consumerTag) {
|
|
291
357
|
logger_1.default.error(`rabbit: failed to consume from queue ${queue}`);
|
|
292
358
|
}
|
|
@@ -297,13 +363,13 @@ class RabbitMq {
|
|
|
297
363
|
});
|
|
298
364
|
}
|
|
299
365
|
async consumeFromExchange(queue, exchange, callback, options) {
|
|
300
|
-
const optionsWithDefaults = { ...
|
|
366
|
+
const optionsWithDefaults = { ...consts_1.DEFAULT_OPTIONS, ...options };
|
|
301
367
|
RabbitMq.validateName('exchange', exchange);
|
|
302
368
|
RabbitMq.validateName('queue', queue);
|
|
303
369
|
const { limit, deadMessageTtl } = optionsWithDefaults;
|
|
304
|
-
const channel = await this.getNewChannel();
|
|
370
|
+
const channel = await this.getNewChannel({ name: `consume-exchange-${exchange}-queue-${queue}` });
|
|
305
371
|
return channel.addSetup(async (c) => {
|
|
306
|
-
const assertExchange = await assertExchangeFanout(c, exchange);
|
|
372
|
+
const assertExchange = await utils_1.assertExchangeFanout(c, exchange);
|
|
307
373
|
await c.assertQueue(queue);
|
|
308
374
|
this.exchanges[exchange] = assertExchange;
|
|
309
375
|
await c.prefetch(limit, true);
|
|
@@ -314,20 +380,26 @@ class RabbitMq {
|
|
|
314
380
|
});
|
|
315
381
|
}
|
|
316
382
|
async publish(exchange, content, customHeaders) {
|
|
317
|
-
return wrapSetImmediate(async () => {
|
|
383
|
+
return utils_1.wrapSetImmediate(async () => {
|
|
318
384
|
RabbitMq.validateName('exchange', exchange);
|
|
319
385
|
const channel = await this.assertChannel();
|
|
320
386
|
await this.assertExchange(exchange);
|
|
321
387
|
await channel.publish(exchange, '', Buffer.from(JSON.stringify(content)), RabbitMq.getPublishOptions(customHeaders));
|
|
322
388
|
});
|
|
323
389
|
}
|
|
324
|
-
async sendToQueue(queue, content, options, customHeaders) {
|
|
325
|
-
|
|
390
|
+
async sendToQueue(queue, content, options, customHeaders, isBlocking) {
|
|
391
|
+
const callback = async () => {
|
|
326
392
|
RabbitMq.validateName('queue', queue);
|
|
327
|
-
|
|
393
|
+
await this.assertChannel();
|
|
328
394
|
await this.assertQueue(queue, options);
|
|
329
|
-
|
|
330
|
-
|
|
395
|
+
const res = await this.channel?.sendToQueue(queue, Buffer.from(JSON.stringify(content)), RabbitMq.getPublishOptions(customHeaders));
|
|
396
|
+
debug(`rabbit: sending to queue ${queue}`, { res });
|
|
397
|
+
return res;
|
|
398
|
+
};
|
|
399
|
+
if (isBlocking) {
|
|
400
|
+
return callback();
|
|
401
|
+
}
|
|
402
|
+
return utils_1.wrapSetImmediate(callback);
|
|
331
403
|
}
|
|
332
404
|
async isConnected() {
|
|
333
405
|
const connection = await this.getConnection();
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
export declare const DEFAULT_DEAD_TTL_TWO_DAYS: number;
|
|
2
|
+
export declare const DEFAULT_LOCK_TIMEOUT: number;
|
|
3
|
+
export declare const RETRY_HEADER = "x-retry-count";
|
|
4
|
+
export declare const TRACING_HEADER = "x-trace-id";
|
|
5
|
+
export declare const DEFAULT_USE_CONSUME_WITH_LOCK = false;
|
|
6
|
+
export declare const CONNECTION_CREATED_CONST = "connectionCreated";
|
|
7
|
+
export declare const DEFAULT_OPTIONS: {
|
|
8
|
+
limit: number;
|
|
9
|
+
retries: number;
|
|
10
|
+
deadMessageTtl: number;
|
|
11
|
+
lockTimeout: number;
|
|
12
|
+
useConsumeWithLock: boolean;
|
|
13
|
+
};
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.DEFAULT_OPTIONS = exports.CONNECTION_CREATED_CONST = exports.DEFAULT_USE_CONSUME_WITH_LOCK = exports.TRACING_HEADER = exports.RETRY_HEADER = exports.DEFAULT_LOCK_TIMEOUT = exports.DEFAULT_DEAD_TTL_TWO_DAYS = void 0;
|
|
4
|
+
exports.DEFAULT_DEAD_TTL_TWO_DAYS = 60000 * 60 * 12;
|
|
5
|
+
exports.DEFAULT_LOCK_TIMEOUT = 1000 * 5;
|
|
6
|
+
exports.RETRY_HEADER = 'x-retry-count';
|
|
7
|
+
exports.TRACING_HEADER = 'x-trace-id';
|
|
8
|
+
exports.DEFAULT_USE_CONSUME_WITH_LOCK = false;
|
|
9
|
+
exports.CONNECTION_CREATED_CONST = 'connectionCreated';
|
|
10
|
+
exports.DEFAULT_OPTIONS = {
|
|
11
|
+
limit: 1,
|
|
12
|
+
retries: 1,
|
|
13
|
+
deadMessageTtl: exports.DEFAULT_DEAD_TTL_TWO_DAYS,
|
|
14
|
+
lockTimeout: exports.DEFAULT_LOCK_TIMEOUT,
|
|
15
|
+
useConsumeWithLock: exports.DEFAULT_USE_CONSUME_WITH_LOCK,
|
|
16
|
+
};
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { ConsumeMessage, Options } from 'amqplib';
|
|
2
|
+
export interface ExchangesCache {
|
|
3
|
+
[key: string]: any;
|
|
4
|
+
}
|
|
5
|
+
export interface QueuesCache {
|
|
6
|
+
[key: string]: any;
|
|
7
|
+
}
|
|
8
|
+
export declare type CustomMessageHeaders = {
|
|
9
|
+
redisTimestampValidationKey?: string;
|
|
10
|
+
};
|
|
11
|
+
export declare type RedisLockType = (args0?: string, arg1?: number) => Promise<any>;
|
|
12
|
+
export declare type ConsumeMessageOrNull = ConsumeMessage | null;
|
|
13
|
+
export interface ConsumeOptions {
|
|
14
|
+
retries?: number;
|
|
15
|
+
deadMessageTtl?: number;
|
|
16
|
+
messageTtl?: number;
|
|
17
|
+
limit?: number;
|
|
18
|
+
lockTimeout?: number;
|
|
19
|
+
useConsumeWithLock?: boolean;
|
|
20
|
+
}
|
|
21
|
+
export declare type CallbackFunction = (msg: ConsumeMessage, ack: any, nack: any) => Promise<any>;
|
|
22
|
+
export declare type newChannelOpts = {
|
|
23
|
+
name?: string;
|
|
24
|
+
onClose?: null | ((args: any | null) => void);
|
|
25
|
+
};
|
|
26
|
+
export declare type assertChannelOpts = {
|
|
27
|
+
channelName?: string;
|
|
28
|
+
force?: boolean;
|
|
29
|
+
};
|
|
30
|
+
export declare type AfConsumer = {
|
|
31
|
+
queue: string;
|
|
32
|
+
callback: CallbackFunction;
|
|
33
|
+
options: ConsumeOptions | undefined;
|
|
34
|
+
};
|
|
35
|
+
export declare const CONSUMER_DEFAULT_OPTIONS: Options.Consume;
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.CONSUMER_DEFAULT_OPTIONS = void 0;
|
|
4
|
+
const HA_PROMOTE_ON_FAILURE = 'ha-promote-on-failure';
|
|
5
|
+
const HA_PROMOTE_ON_SHUTDOWN = 'ha-promote-on-shutdown';
|
|
6
|
+
exports.CONSUMER_DEFAULT_OPTIONS = {
|
|
7
|
+
arguments: {
|
|
8
|
+
[HA_PROMOTE_ON_FAILURE]: 'always',
|
|
9
|
+
[HA_PROMOTE_ON_SHUTDOWN]: 'always',
|
|
10
|
+
},
|
|
11
|
+
};
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import { ChannelWrapper } from 'amqp-connection-manager';
|
|
2
|
+
import { ConfirmChannel } from 'amqplib';
|
|
3
|
+
export declare const assertExchangeFanout: (c: ChannelWrapper | ConfirmChannel, exchangeName: string) => Promise<import("amqplib").Replies.AssertExchange>;
|
|
4
|
+
export declare const wrapSetImmediate: (callback: () => any) => Promise<any>;
|
|
5
|
+
export declare const rand: () => number;
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.rand = exports.wrapSetImmediate = exports.assertExchangeFanout = void 0;
|
|
4
|
+
exports.assertExchangeFanout = async (c, exchangeName) => c.assertExchange(exchangeName, 'fanout');
|
|
5
|
+
exports.wrapSetImmediate = (callback) => new Promise((resolve, reject) => {
|
|
6
|
+
setImmediate(async () => {
|
|
7
|
+
try {
|
|
8
|
+
const value = await callback();
|
|
9
|
+
resolve(value);
|
|
10
|
+
}
|
|
11
|
+
catch (error) {
|
|
12
|
+
reject(error);
|
|
13
|
+
}
|
|
14
|
+
});
|
|
15
|
+
});
|
|
16
|
+
exports.rand = () => Math.floor(Math.random() * 100000);
|
package/package.json
CHANGED
package/src/index.ts
CHANGED
|
@@ -1,15 +1,37 @@
|
|
|
1
|
-
/* eslint-disable
|
|
2
|
-
|
|
1
|
+
/* eslint-disable no-empty,consistent-return,no-async-promise-executor,@typescript-eslint/no-unused-vars */
|
|
2
|
+
|
|
3
|
+
import { EventEmitter } from 'events';
|
|
4
|
+
import { promisify } from 'util';
|
|
3
5
|
import moment from 'moment';
|
|
4
6
|
import RedisLock from 'redis-lock';
|
|
5
|
-
import { ConfirmChannel, Options, ConsumeMessage } from 'amqplib';
|
|
6
7
|
import { AmqpConnectionManager, ChannelWrapper, connect } from 'amqp-connection-manager';
|
|
7
|
-
import {
|
|
8
|
-
|
|
9
|
-
|
|
8
|
+
import {
|
|
9
|
+
ConfirmChannel, ConsumeMessage, Options, Replies,
|
|
10
|
+
} from 'amqplib';
|
|
11
|
+
import { getCurrentContext, newTrace, traceTypes } from '@autofleet/outbreak';
|
|
10
12
|
import logger from './logger';
|
|
11
|
-
import RabbitError from './rabbitError';
|
|
12
|
-
import getRedisInstance, { RedisConfig } from './redis';
|
|
13
|
+
import RabbitError from './lib/rabbitError';
|
|
14
|
+
import getRedisInstance, { RedisConfig } from './lib/redis';
|
|
15
|
+
import { assertExchangeFanout, rand, wrapSetImmediate } from './lib/utils';
|
|
16
|
+
import {
|
|
17
|
+
CONNECTION_CREATED_CONST,
|
|
18
|
+
DEFAULT_LOCK_TIMEOUT,
|
|
19
|
+
DEFAULT_OPTIONS,
|
|
20
|
+
RETRY_HEADER,
|
|
21
|
+
TRACING_HEADER,
|
|
22
|
+
} from './lib/consts';
|
|
23
|
+
import {
|
|
24
|
+
CallbackFunction,
|
|
25
|
+
ConsumeMessageOrNull,
|
|
26
|
+
ConsumeOptions,
|
|
27
|
+
CustomMessageHeaders,
|
|
28
|
+
QueuesCache,
|
|
29
|
+
RedisLockType,
|
|
30
|
+
ExchangesCache, CONSUMER_DEFAULT_OPTIONS,
|
|
31
|
+
} from './lib/types';
|
|
32
|
+
|
|
33
|
+
// const debug = nodeDebug('af-rabbitmq')
|
|
34
|
+
const debug = logger.info;
|
|
13
35
|
|
|
14
36
|
export interface IAfRabbitMq {
|
|
15
37
|
ack: any;
|
|
@@ -24,62 +46,45 @@ export interface IAfRabbitMq {
|
|
|
24
46
|
redisClient?: any;
|
|
25
47
|
}
|
|
26
48
|
|
|
27
|
-
interface
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
49
|
+
export interface AfRabbitOptions {
|
|
50
|
+
disableReconnect?: boolean;
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* When you want your own grace-full shutdown, set this to true.
|
|
54
|
+
* @default false
|
|
55
|
+
*/
|
|
56
|
+
dontGracefulShutdown?: boolean;
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* dont retry on creation error
|
|
60
|
+
* @default false
|
|
61
|
+
*/
|
|
62
|
+
dontRetryAssert?: boolean;
|
|
36
63
|
}
|
|
37
64
|
|
|
38
|
-
type
|
|
65
|
+
type newChannelOpts = {
|
|
66
|
+
name?: string;
|
|
67
|
+
onClose?: null | ((args: any | null) => void);
|
|
68
|
+
};
|
|
39
69
|
|
|
40
|
-
type
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
deadMessageTtl?: number;
|
|
44
|
-
limit?: number;
|
|
45
|
-
lockTimeout?: number;
|
|
46
|
-
useConsumeWithLock?: boolean;
|
|
70
|
+
type assertChannelOpts = {
|
|
71
|
+
channelName?: string;
|
|
72
|
+
force?: boolean;
|
|
47
73
|
}
|
|
48
74
|
|
|
49
|
-
type
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
75
|
+
type AfConsumer = {
|
|
76
|
+
queue: string;
|
|
77
|
+
callback: CallbackFunction;
|
|
78
|
+
options: ConsumeOptions | undefined;
|
|
53
79
|
}
|
|
54
80
|
|
|
55
|
-
const
|
|
56
|
-
const DEFAULT_LOCK_TIMEOUT = 1000 * 5;
|
|
57
|
-
const RETRY_HEADER = 'x-retry-count';
|
|
58
|
-
const TRACING_HEADER = 'x-trace-id';
|
|
59
|
-
const DEFAULT_USE_CONSUME_WITH_LOCK = false;
|
|
60
|
-
|
|
61
|
-
const defaultOptions = {
|
|
62
|
-
limit: 1,
|
|
63
|
-
retries: 1,
|
|
64
|
-
deadMessageTtl: DEFAULT_DEAD_TTL_TWO_DAYS,
|
|
65
|
-
lockTimeout: DEFAULT_LOCK_TIMEOUT,
|
|
66
|
-
useConsumeWithLock: DEFAULT_USE_CONSUME_WITH_LOCK,
|
|
67
|
-
};
|
|
81
|
+
const USERNAME: string = process.env.RABBITMQ_USERNAME || 'guest';
|
|
68
82
|
|
|
69
|
-
const
|
|
83
|
+
const PASSWORD: string = process.env.RABBITMQ_PASSWORD || 'guest';
|
|
70
84
|
|
|
71
|
-
const
|
|
85
|
+
const HOST: string = process.env.RABBITMQ_SERVICE_HOST || 'localhost';
|
|
72
86
|
|
|
73
|
-
const
|
|
74
|
-
setImmediate(async () => {
|
|
75
|
-
try {
|
|
76
|
-
const value = await callback();
|
|
77
|
-
resolve(value);
|
|
78
|
-
} catch (error) {
|
|
79
|
-
reject(error);
|
|
80
|
-
}
|
|
81
|
-
});
|
|
82
|
-
});
|
|
87
|
+
const HEARTBEAT = '60';
|
|
83
88
|
|
|
84
89
|
class RabbitMq implements IAfRabbitMq {
|
|
85
90
|
static parseMsg(msg: any) : any {
|
|
@@ -114,17 +119,19 @@ class RabbitMq implements IAfRabbitMq {
|
|
|
114
119
|
};
|
|
115
120
|
}
|
|
116
121
|
|
|
117
|
-
PUBLISH_TIMEOUT = Number(process.env.RABBITMQ_PUBLISH_TIMEOUT) || 60000;
|
|
122
|
+
// PUBLISH_TIMEOUT = Number(process.env.RABBITMQ_PUBLISH_TIMEOUT) || 60000;
|
|
118
123
|
|
|
119
|
-
PUBLISH_ERROR_MSG = `rabbit: publish timeout(${this.PUBLISH_TIMEOUT}ms) has pass, exchange: `;
|
|
124
|
+
// PUBLISH_ERROR_MSG = `rabbit: publish timeout(${this.PUBLISH_TIMEOUT}ms) has pass, exchange: `;
|
|
120
125
|
|
|
121
126
|
DISCONNECT_MSG = 'rabbit: connection disconnect';
|
|
122
127
|
|
|
123
|
-
|
|
128
|
+
RECONNECT_MSG = 'rabbit: connection disconnect - reconnecting';
|
|
124
129
|
|
|
125
130
|
channel: ChannelWrapper | null;
|
|
126
131
|
|
|
127
|
-
|
|
132
|
+
blockReconnect: boolean | null | undefined
|
|
133
|
+
|
|
134
|
+
connection: AmqpConnectionManager | null | undefined
|
|
128
135
|
|
|
129
136
|
em: EventEmitter;
|
|
130
137
|
|
|
@@ -143,6 +150,8 @@ class RabbitMq implements IAfRabbitMq {
|
|
|
143
150
|
/** Array of consumers tags used for canceling consumption */
|
|
144
151
|
consumersTags: Array<[ConfirmChannel, string]>;
|
|
145
152
|
|
|
153
|
+
private consumers: Array<AfConsumer> = [];
|
|
154
|
+
|
|
146
155
|
constructor(options?: AfRabbitOptions, redisConfig?: RedisConfig) {
|
|
147
156
|
this.em = new EventEmitter();
|
|
148
157
|
this.channel = null;
|
|
@@ -157,12 +166,14 @@ class RabbitMq implements IAfRabbitMq {
|
|
|
157
166
|
}
|
|
158
167
|
this.consumersTags = [];
|
|
159
168
|
logger.info(`rabbit: [gracefully-shutdown] adding gracefully shutdown for process.pid ${process.pid}`);
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
169
|
+
if (!this.options?.dontGracefulShutdown) {
|
|
170
|
+
process.on('SIGTERM', async () => {
|
|
171
|
+
await this.gracefulShutdown('SIGTERM');
|
|
172
|
+
});
|
|
173
|
+
process.on('SIGINT', async () => {
|
|
174
|
+
await this.gracefulShutdown('SIGINT');
|
|
175
|
+
});
|
|
176
|
+
}
|
|
166
177
|
}
|
|
167
178
|
|
|
168
179
|
private shouldConsumeMessageByTimestamp = async (msg: ConsumeMessageOrNull) => {
|
|
@@ -241,54 +252,88 @@ class RabbitMq implements IAfRabbitMq {
|
|
|
241
252
|
|
|
242
253
|
async getConnection() {
|
|
243
254
|
return new Promise<AmqpConnectionManager>(async (resolve) => {
|
|
255
|
+
if (this.blockReconnect) {
|
|
256
|
+
debug('rabbit: block reconnect');
|
|
257
|
+
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
|
258
|
+
// @ts-ignore
|
|
259
|
+
return resolve();
|
|
260
|
+
}
|
|
244
261
|
if (this.connection !== null) {
|
|
245
|
-
|
|
262
|
+
if (this.options?.disableReconnect || this.connection?.isConnected()) {
|
|
263
|
+
debug('rabbit: connection - is connected');
|
|
264
|
+
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
|
265
|
+
// @ts-ignore
|
|
266
|
+
return resolve(this.connection);
|
|
267
|
+
}
|
|
268
|
+
debug('rabbit: connection - reconnecting');
|
|
246
269
|
}
|
|
247
270
|
if (this.creatingConnection) {
|
|
248
|
-
|
|
271
|
+
debug('rabbit: creating connection emi');
|
|
272
|
+
this.em.once(CONNECTION_CREATED_CONST, resolve);
|
|
249
273
|
return;
|
|
250
274
|
}
|
|
251
275
|
this.creatingConnection = true;
|
|
252
|
-
|
|
253
|
-
const
|
|
254
|
-
const password: string = process.env.RABBITMQ_PASSWORD || 'guest';
|
|
255
|
-
const host: string = process.env.RABBITMQ_SERVICE_HOST || '';
|
|
256
|
-
const connection: AmqpConnectionManager = await connect([`amqp://${username}:${password}@${host}?heartbeat=60`]);
|
|
276
|
+
debug('rabbit: creating connection', { HOST, USERNAME, HEARTBEAT });
|
|
277
|
+
const connection: AmqpConnectionManager = await connect([`amqp://${USERNAME}:${PASSWORD}@${HOST}?heartbeat=${HEARTBEAT}}`]);
|
|
257
278
|
this.connection = connection;
|
|
258
|
-
this.connection.on('disconnect',
|
|
259
|
-
(
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
279
|
+
this.connection.on('disconnect', ({ err }) => {
|
|
280
|
+
debug('rabbit: connection closed');
|
|
281
|
+
this.exchanges = {};
|
|
282
|
+
this.queues = {};
|
|
283
|
+
this.connection = null;
|
|
284
|
+
this.channel = null;
|
|
285
|
+
if (this.options?.disableReconnect) {
|
|
286
|
+
logger.error(`${this.DISCONNECT_MSG}${err && ` - ${err}`}`);
|
|
287
|
+
this.blockReconnect = true;
|
|
288
|
+
} else {
|
|
289
|
+
logger.error(`${this.RECONNECT_MSG}${err && ` - ${err}`}`);
|
|
290
|
+
}
|
|
291
|
+
});
|
|
292
|
+
this.connection.once('connect', async () => {
|
|
293
|
+
debug('rabbit: connection established');
|
|
294
|
+
await this.loadConsumers();
|
|
295
|
+
this.creatingConnection = false;
|
|
296
|
+
this.em.emit(CONNECTION_CREATED_CONST, connection);
|
|
297
|
+
resolve(connection);
|
|
298
|
+
});
|
|
272
299
|
});
|
|
273
300
|
}
|
|
274
301
|
|
|
275
|
-
async getNewChannel() {
|
|
276
|
-
|
|
277
|
-
|
|
302
|
+
async getNewChannel({ name = rand().toString(), onClose = null }: newChannelOpts = {}) {
|
|
303
|
+
return new Promise<ChannelWrapper>(async (resolve, reject) => {
|
|
304
|
+
const connection: AmqpConnectionManager = await this.getConnection();
|
|
305
|
+
const channel = connection.createChannel({});
|
|
306
|
+
|
|
307
|
+
channel.on('error', (err) => {
|
|
308
|
+
logger.error(`rabbit: channel ${name} error`, { err });
|
|
309
|
+
});
|
|
310
|
+
channel.on('close', (...args) => {
|
|
311
|
+
logger.error(`rabbit: channel ${name} closed`, { args });
|
|
312
|
+
if (onClose) {
|
|
313
|
+
onClose(args);
|
|
314
|
+
}
|
|
315
|
+
});
|
|
316
|
+
channel.once('connect', () => {
|
|
317
|
+
debug(`rabbit: channel ${name} CONNECTED`);
|
|
318
|
+
resolve(channel);
|
|
319
|
+
});
|
|
320
|
+
});
|
|
278
321
|
}
|
|
279
322
|
|
|
280
|
-
async assertChannel() {
|
|
323
|
+
async assertChannel({ force = false } : assertChannelOpts = {}): Promise<ChannelWrapper> {
|
|
281
324
|
return new Promise<ChannelWrapper>(async (resolve, reject) => {
|
|
282
|
-
if (this.channel) {
|
|
325
|
+
if (this.channel && !force) {
|
|
283
326
|
return resolve(this.channel);
|
|
284
327
|
}
|
|
285
328
|
|
|
286
329
|
try {
|
|
287
|
-
const
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
330
|
+
const channel = await this.getNewChannel({
|
|
331
|
+
onClose: () => {
|
|
332
|
+
this.channel = null;
|
|
333
|
+
},
|
|
334
|
+
});
|
|
335
|
+
this.channel = channel;
|
|
336
|
+
resolve(channel);
|
|
292
337
|
} catch (e) {
|
|
293
338
|
reject(e);
|
|
294
339
|
}
|
|
@@ -308,31 +353,82 @@ class RabbitMq implements IAfRabbitMq {
|
|
|
308
353
|
async getQueueLength(queue: string) {
|
|
309
354
|
RabbitMq.validateName('queue', queue);
|
|
310
355
|
const channel: ChannelWrapper = await this.assertChannel();
|
|
311
|
-
|
|
356
|
+
debug('rabbit: getting queue length', { queue, connected: this.connection?.isConnected() });
|
|
312
357
|
return channel.checkQueue(queue);
|
|
313
358
|
}
|
|
314
359
|
|
|
360
|
+
private async deleteQueue(queue: string) {
|
|
361
|
+
return new Promise(async (resolve, reject) => {
|
|
362
|
+
RabbitMq.validateName('queue', queue);
|
|
363
|
+
const channel: ChannelWrapper = await this.assertChannel();
|
|
364
|
+
logger.info('rabbit: deleting queue', { queue });
|
|
365
|
+
const deleteQueueRes = await channel.deleteQueue(queue);
|
|
366
|
+
debug('queue deleted', deleteQueueRes);
|
|
367
|
+
resolve(deleteQueueRes);
|
|
368
|
+
});
|
|
369
|
+
}
|
|
370
|
+
|
|
315
371
|
async bindQueue(queue: string, exchange: string) {
|
|
316
372
|
const channel: ChannelWrapper = await this.assertChannel();
|
|
317
373
|
await channel.addSetup((setupChannel: ConfirmChannel) => setupChannel.bindQueue(queue, exchange, ''));
|
|
318
|
-
// @ts-ignore
|
|
319
374
|
return channel.bindQueue(queue, exchange, '');
|
|
320
375
|
}
|
|
321
376
|
|
|
322
377
|
async assertQueue(queueName: string, options?: Options.AssertQueue) {
|
|
378
|
+
let queue: Replies.AssertQueue | null = null;
|
|
323
379
|
RabbitMq.validateName('queue', queueName);
|
|
324
|
-
const channel: ChannelWrapper = await this.assertChannel();
|
|
325
380
|
if (this.queues[queueName]) {
|
|
326
381
|
return this.queues[queueName];
|
|
327
382
|
}
|
|
328
|
-
|
|
329
|
-
|
|
383
|
+
try {
|
|
384
|
+
const channel: ChannelWrapper = await this.assertChannel();
|
|
385
|
+
debug('assertQueue->channel.addSetup', { queueName });
|
|
386
|
+
await channel.addSetup((setupChannel: ConfirmChannel) => setupChannel.assertQueue(queueName, options));
|
|
387
|
+
debug('assertQueue->channel.assertQueue', { queueName });
|
|
388
|
+
queue = await channel.assertQueue(queueName, options);
|
|
389
|
+
} catch (e) {
|
|
390
|
+
logger.error('rabbit: assertQueue error', { queueName, options, error: e });
|
|
391
|
+
if (!this.options?.dontRetryAssert) {
|
|
392
|
+
debug('retrying assertQueue', { queueName });
|
|
393
|
+
const channel = await this.assertChannel({ force: true });
|
|
394
|
+
await this.deleteQueue(queueName);
|
|
395
|
+
|
|
396
|
+
debug('1assertQueue->channel.addSetup', { queueName });
|
|
397
|
+
await channel.addSetup((setupChannel: ConfirmChannel) => setupChannel.assertQueue(queueName, options));
|
|
398
|
+
debug('1assertQueue->channel.assertQueue', { queueName });
|
|
399
|
+
queue = await channel.assertQueue(queueName, options);
|
|
400
|
+
} else {
|
|
401
|
+
throw e;
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
|
|
330
405
|
this.queues[queueName] = queueName;
|
|
331
406
|
return queue;
|
|
332
407
|
}
|
|
333
408
|
|
|
409
|
+
private saveConsumer(queue: string, callback: CallbackFunction, options: ConsumeOptions | undefined) {
|
|
410
|
+
this.consumers.push({
|
|
411
|
+
queue,
|
|
412
|
+
callback,
|
|
413
|
+
options,
|
|
414
|
+
});
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
private async loadConsumers() {
|
|
418
|
+
debug('rabbit: loading consumers', { consumers: this.consumers.length });
|
|
419
|
+
if (this.consumers.length > 0) {
|
|
420
|
+
await Promise.all(
|
|
421
|
+
this.consumers.map((consumer) => this
|
|
422
|
+
.consumeFromRabbit(consumer.queue, consumer.callback, consumer.options)),
|
|
423
|
+
);
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
|
|
334
427
|
async consume(queue: string, callback: CallbackFunction, options?: ConsumeOptions): Promise<any> {
|
|
335
|
-
|
|
428
|
+
if (this.connection && !this.creatingConnection) {
|
|
429
|
+
this.consumeFromRabbit(queue, callback, options);
|
|
430
|
+
}
|
|
431
|
+
return this.saveConsumer(queue, callback, options);
|
|
336
432
|
}
|
|
337
433
|
|
|
338
434
|
private async lockRedisIfNeeded(msg: any, options: any) {
|
|
@@ -356,7 +452,7 @@ class RabbitMq implements IAfRabbitMq {
|
|
|
356
452
|
}
|
|
357
453
|
|
|
358
454
|
private async consumeFromRabbit(queue: string, callback: CallbackFunction, options?: ConsumeOptions): Promise<any> {
|
|
359
|
-
const optionsWithDefaults = { ...
|
|
455
|
+
const optionsWithDefaults = { ...DEFAULT_OPTIONS, ...options };
|
|
360
456
|
RabbitMq.validateName('queue', queue);
|
|
361
457
|
const {
|
|
362
458
|
limit, deadMessageTtl, useConsumeWithLock, lockTimeout,
|
|
@@ -367,9 +463,9 @@ class RabbitMq implements IAfRabbitMq {
|
|
|
367
463
|
}
|
|
368
464
|
logger.info(`rabbit: Consuming with lock from queue ${queue} with lockTimeout: ${lockTimeout}ms`);
|
|
369
465
|
}
|
|
370
|
-
const channel: ChannelWrapper = await this.getNewChannel();
|
|
466
|
+
const channel: ChannelWrapper = await this.getNewChannel({ name: `consume-queue-${queue}` });
|
|
371
467
|
return channel.addSetup(async (c: ConfirmChannel) => {
|
|
372
|
-
await c.assertQueue(queue);
|
|
468
|
+
await c.assertQueue(queue, options ? { messageTtl: options.messageTtl } : undefined);
|
|
373
469
|
await c.prefetch(limit, true);
|
|
374
470
|
const { consumerTag } = await c.consume(
|
|
375
471
|
queue,
|
|
@@ -400,7 +496,7 @@ class RabbitMq implements IAfRabbitMq {
|
|
|
400
496
|
} catch (e) {
|
|
401
497
|
await this.nack(channel, queue, optionsWithDefaults, { messageTtl: deadMessageTtl }, msg, releaseLock)(msg);
|
|
402
498
|
}
|
|
403
|
-
},
|
|
499
|
+
}, CONSUMER_DEFAULT_OPTIONS,
|
|
404
500
|
);
|
|
405
501
|
if (!consumerTag) {
|
|
406
502
|
logger.error(`rabbit: failed to consume from queue ${queue}`);
|
|
@@ -412,11 +508,11 @@ class RabbitMq implements IAfRabbitMq {
|
|
|
412
508
|
}
|
|
413
509
|
|
|
414
510
|
async consumeFromExchange(queue: string, exchange: string, callback: CallbackFunction, options?: ConsumeOptions) : Promise<any> {
|
|
415
|
-
const optionsWithDefaults = { ...
|
|
511
|
+
const optionsWithDefaults = { ...DEFAULT_OPTIONS, ...options };
|
|
416
512
|
RabbitMq.validateName('exchange', exchange);
|
|
417
513
|
RabbitMq.validateName('queue', queue);
|
|
418
514
|
const { limit, deadMessageTtl } = optionsWithDefaults;
|
|
419
|
-
const channel: ChannelWrapper = await this.getNewChannel();
|
|
515
|
+
const channel: ChannelWrapper = await this.getNewChannel({ name: `consume-exchange-${exchange}-queue-${queue}` });
|
|
420
516
|
|
|
421
517
|
return channel.addSetup(async (c: ConfirmChannel) => {
|
|
422
518
|
const assertExchange = await assertExchangeFanout(c, exchange);
|
|
@@ -445,15 +541,27 @@ class RabbitMq implements IAfRabbitMq {
|
|
|
445
541
|
});
|
|
446
542
|
}
|
|
447
543
|
|
|
448
|
-
async sendToQueue(
|
|
449
|
-
|
|
544
|
+
async sendToQueue(
|
|
545
|
+
queue: string,
|
|
546
|
+
content: any,
|
|
547
|
+
options?: any,
|
|
548
|
+
customHeaders?: any,
|
|
549
|
+
isBlocking?: boolean,
|
|
550
|
+
): Promise<boolean | undefined> {
|
|
551
|
+
const callback = async (): Promise<boolean | undefined> => {
|
|
450
552
|
RabbitMq.validateName('queue', queue);
|
|
451
|
-
|
|
553
|
+
await this.assertChannel();
|
|
452
554
|
await this.assertQueue(queue, options);
|
|
453
|
-
|
|
555
|
+
const res = await this.channel?.sendToQueue(queue,
|
|
454
556
|
Buffer.from(JSON.stringify(content)),
|
|
455
557
|
RabbitMq.getPublishOptions(customHeaders));
|
|
456
|
-
|
|
558
|
+
debug(`rabbit: sending to queue ${queue}`, { res });
|
|
559
|
+
return res;
|
|
560
|
+
};
|
|
561
|
+
if (isBlocking) {
|
|
562
|
+
return callback();
|
|
563
|
+
}
|
|
564
|
+
return wrapSetImmediate(callback);
|
|
457
565
|
}
|
|
458
566
|
|
|
459
567
|
async isConnected() : Promise<boolean> {
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
export const DEFAULT_DEAD_TTL_TWO_DAYS = 60000 * 60 * 12;
|
|
2
|
+
export const DEFAULT_LOCK_TIMEOUT = 1000 * 5;
|
|
3
|
+
export const RETRY_HEADER = 'x-retry-count';
|
|
4
|
+
export const TRACING_HEADER = 'x-trace-id';
|
|
5
|
+
export const DEFAULT_USE_CONSUME_WITH_LOCK = false;
|
|
6
|
+
export const CONNECTION_CREATED_CONST = 'connectionCreated';
|
|
7
|
+
export const DEFAULT_OPTIONS = {
|
|
8
|
+
limit: 1,
|
|
9
|
+
retries: 1,
|
|
10
|
+
deadMessageTtl: DEFAULT_DEAD_TTL_TWO_DAYS,
|
|
11
|
+
lockTimeout: DEFAULT_LOCK_TIMEOUT,
|
|
12
|
+
useConsumeWithLock: DEFAULT_USE_CONSUME_WITH_LOCK,
|
|
13
|
+
};
|
package/src/lib/types.ts
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { ConsumeMessage, Options } from 'amqplib';
|
|
2
|
+
|
|
3
|
+
export interface ExchangesCache {
|
|
4
|
+
[key: string]: any
|
|
5
|
+
}
|
|
6
|
+
|
|
7
|
+
export interface QueuesCache {
|
|
8
|
+
[key: string]: any
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export type CustomMessageHeaders = {
|
|
12
|
+
redisTimestampValidationKey?: string;
|
|
13
|
+
}
|
|
14
|
+
export type RedisLockType = (args0?: string, arg1?: number) => Promise<any>
|
|
15
|
+
export type ConsumeMessageOrNull = ConsumeMessage | null;
|
|
16
|
+
|
|
17
|
+
export interface ConsumeOptions {
|
|
18
|
+
retries?: number;
|
|
19
|
+
deadMessageTtl?: number;
|
|
20
|
+
messageTtl?: number;
|
|
21
|
+
limit?: number;
|
|
22
|
+
lockTimeout?: number;
|
|
23
|
+
useConsumeWithLock?: boolean;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export type CallbackFunction = (msg: ConsumeMessage, ack: any, nack: any) => Promise<any>
|
|
27
|
+
|
|
28
|
+
export type newChannelOpts = {
|
|
29
|
+
name?: string;
|
|
30
|
+
onClose?: null | ((args: any | null) => void);
|
|
31
|
+
};
|
|
32
|
+
export type assertChannelOpts = {
|
|
33
|
+
channelName?: string;
|
|
34
|
+
force?: boolean;
|
|
35
|
+
}
|
|
36
|
+
export type AfConsumer = {
|
|
37
|
+
queue: string;
|
|
38
|
+
callback: CallbackFunction;
|
|
39
|
+
options: ConsumeOptions | undefined;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
const HA_PROMOTE_ON_FAILURE = 'ha-promote-on-failure';
|
|
43
|
+
|
|
44
|
+
const HA_PROMOTE_ON_SHUTDOWN = 'ha-promote-on-shutdown';
|
|
45
|
+
|
|
46
|
+
export const CONSUMER_DEFAULT_OPTIONS: Options.Consume = {
|
|
47
|
+
arguments: {
|
|
48
|
+
[HA_PROMOTE_ON_FAILURE]: 'always',
|
|
49
|
+
[HA_PROMOTE_ON_SHUTDOWN]: 'always',
|
|
50
|
+
},
|
|
51
|
+
};
|
package/src/lib/utils.ts
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { ChannelWrapper } from 'amqp-connection-manager';
|
|
2
|
+
import { ConfirmChannel } from 'amqplib';
|
|
3
|
+
|
|
4
|
+
export const assertExchangeFanout = async (c: ChannelWrapper | ConfirmChannel, exchangeName: string) => c.assertExchange(exchangeName, 'fanout');
|
|
5
|
+
export const wrapSetImmediate = (callback: () => any) => new Promise<any>((resolve, reject) => {
|
|
6
|
+
setImmediate(async () => {
|
|
7
|
+
try {
|
|
8
|
+
const value = await callback();
|
|
9
|
+
resolve(value);
|
|
10
|
+
} catch (error) {
|
|
11
|
+
reject(error);
|
|
12
|
+
}
|
|
13
|
+
});
|
|
14
|
+
});
|
|
15
|
+
export const rand = () => Math.floor(Math.random() * 100000);
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|