@yolo-croket-dev/core 1.10.1 → 1.10.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/functions/rmq-diagnostics.function.d.ts +12 -0
- package/functions/rmq-diagnostics.function.js +21 -1
- package/interceptors/logging-service.interceptor.js +2 -0
- package/interceptors/rpc.interceptor.d.ts +1 -0
- package/interceptors/rpc.interceptor.js +45 -23
- package/modules/graceful-shutdown/graceful-shutdown.module.js +1 -2
- package/modules/graceful-shutdown/graceful-shutdown.rmq.d.ts +9 -2
- package/modules/graceful-shutdown/graceful-shutdown.rmq.js +36 -6
- package/modules/graceful-shutdown/graceful-shutdown.service.d.ts +8 -0
- package/modules/graceful-shutdown/graceful-shutdown.service.js +11 -2
- package/package.json +1 -1
- package/tests/rmq-consumer-diagnostics.test.cjs +89 -1
- package/tests/rmq-shutdown-requeue.test.cjs +191 -0
package/CHANGELOG.md
CHANGED
|
@@ -67,6 +67,18 @@ export declare function registerRmqObservation(owner: object, observer: RmqConne
|
|
|
67
67
|
* @process 관찰 대상이 아니면 undefined를 반환한다.
|
|
68
68
|
*/
|
|
69
69
|
export declare function getRmqDiagnosticSnapshot(owner: object): Snapshot | undefined;
|
|
70
|
+
/**
|
|
71
|
+
* @author jun
|
|
72
|
+
* @description 현재 메시지의 핸들러 오류 상세를 LoggingServiceInterceptor가 기록할 수 있음을 표시한다.
|
|
73
|
+
* @process 메시지 객체만 약하게 참조하여 동시 요청과 재전달 메시지를 구분한다.
|
|
74
|
+
*/
|
|
75
|
+
export declare function markRmqHandlerLogging(message: unknown): void;
|
|
76
|
+
/**
|
|
77
|
+
* @author jun
|
|
78
|
+
* @description 현재 메시지에 핸들러 오류 상세 로거가 있는지 확인한다.
|
|
79
|
+
* @process RPCInterceptor의 중복 상세 기록 여부를 결정한다.
|
|
80
|
+
*/
|
|
81
|
+
export declare function hasRmqHandlerLogging(message: unknown): boolean;
|
|
70
82
|
/**
|
|
71
83
|
* @author jun
|
|
72
84
|
* @description 이미 기록한 오류 상세를 표시한다.
|
|
@@ -26,13 +26,14 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
26
26
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
27
27
|
};
|
|
28
28
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
29
|
-
exports.createDiagnosticRmqClient = exports.getRecordedRmqDiagnosticError = exports.recordRmqDiagnosticError = exports.getRmqDiagnosticSnapshot = exports.registerRmqObservation = exports.withRmqConnectionName = exports.createRmqObserver = exports.RmqConnectionObserver = exports.isRmqDiagnosticsEnabled = void 0;
|
|
29
|
+
exports.createDiagnosticRmqClient = exports.getRecordedRmqDiagnosticError = exports.recordRmqDiagnosticError = exports.hasRmqHandlerLogging = exports.markRmqHandlerLogging = exports.getRmqDiagnosticSnapshot = exports.registerRmqObservation = exports.withRmqConnectionName = exports.createRmqObserver = exports.RmqConnectionObserver = exports.isRmqDiagnosticsEnabled = void 0;
|
|
30
30
|
const node_crypto_1 = require("node:crypto");
|
|
31
31
|
const node_os_1 = __importDefault(require("node:os"));
|
|
32
32
|
const microservices_1 = require("@nestjs/microservices");
|
|
33
33
|
const msa_logger_1 = __importStar(require("@yolo-croket-dev/msa-logger"));
|
|
34
34
|
const observations = new WeakMap();
|
|
35
35
|
const recordedErrors = new WeakMap();
|
|
36
|
+
const handlerLoggingMessages = new WeakSet();
|
|
36
37
|
let logger;
|
|
37
38
|
/**
|
|
38
39
|
* @author jun
|
|
@@ -245,6 +246,25 @@ function getRmqDiagnosticSnapshot(owner) {
|
|
|
245
246
|
return (_a = observations.get(owner)) === null || _a === void 0 ? void 0 : _a.snapshot();
|
|
246
247
|
}
|
|
247
248
|
exports.getRmqDiagnosticSnapshot = getRmqDiagnosticSnapshot;
|
|
249
|
+
/**
|
|
250
|
+
* @author jun
|
|
251
|
+
* @description 현재 메시지의 핸들러 오류 상세를 LoggingServiceInterceptor가 기록할 수 있음을 표시한다.
|
|
252
|
+
* @process 메시지 객체만 약하게 참조하여 동시 요청과 재전달 메시지를 구분한다.
|
|
253
|
+
*/
|
|
254
|
+
function markRmqHandlerLogging(message) {
|
|
255
|
+
if (message !== null && typeof message === 'object')
|
|
256
|
+
handlerLoggingMessages.add(message);
|
|
257
|
+
}
|
|
258
|
+
exports.markRmqHandlerLogging = markRmqHandlerLogging;
|
|
259
|
+
/**
|
|
260
|
+
* @author jun
|
|
261
|
+
* @description 현재 메시지에 핸들러 오류 상세 로거가 있는지 확인한다.
|
|
262
|
+
* @process RPCInterceptor의 중복 상세 기록 여부를 결정한다.
|
|
263
|
+
*/
|
|
264
|
+
function hasRmqHandlerLogging(message) {
|
|
265
|
+
return message !== null && typeof message === 'object' && handlerLoggingMessages.has(message);
|
|
266
|
+
}
|
|
267
|
+
exports.hasRmqHandlerLogging = hasRmqHandlerLogging;
|
|
248
268
|
// Contains error metadata only, never per-request context: one RMQ error can reject concurrent calls.
|
|
249
269
|
/**
|
|
250
270
|
* @author jun
|
|
@@ -161,6 +161,8 @@ let LoggingServiceInterceptor = class LoggingServiceInterceptor {
|
|
|
161
161
|
loggingSpan.setTag('traceId', traceId);
|
|
162
162
|
loggingSpan.setTag('messagePattern', pattern);
|
|
163
163
|
}
|
|
164
|
+
if (connection)
|
|
165
|
+
(0, rmq_diagnostics_function_1.markRmqHandlerLogging)(originalMessage);
|
|
164
166
|
return next.handle()
|
|
165
167
|
.pipe((0, operators_1.tap)(() => {
|
|
166
168
|
const elapsed = Date.now() - now;
|
|
@@ -12,5 +12,6 @@ export declare class RPCInterceptor implements NestInterceptor {
|
|
|
12
12
|
/** 메시지가 재전송되었을때에, 재시도 Count가 명시되어 있지 않을때에 처리 할지 여부를 결정하는 함수 */
|
|
13
13
|
isRedeliveryMessage(replyTo: string, traceId: string): Promise<boolean>;
|
|
14
14
|
private updateMaxLengthArrayShorten;
|
|
15
|
+
private requeueMessageOnShutdown;
|
|
15
16
|
intercept(context: ExecutionContext, next: CallHandler): Promise<Observable<unknown>>;
|
|
16
17
|
}
|
|
@@ -148,6 +148,13 @@ let RPCInterceptor = class RPCInterceptor {
|
|
|
148
148
|
}
|
|
149
149
|
return obj;
|
|
150
150
|
}
|
|
151
|
+
async requeueMessageOnShutdown(channel, message) {
|
|
152
|
+
msaLogger.debug('RPCInterceptor isApplicationExit called', { traceId: 'IS_APPLICATION_EXIT' });
|
|
153
|
+
await channel.cancel(message.fields.consumerTag);
|
|
154
|
+
channel.nack(message, false, true);
|
|
155
|
+
// 기존 서버에서 응답을 완료하면 재전달된 요청의 실제 응답을 클라이언트가 기다릴 수 없다.
|
|
156
|
+
return new rxjs_1.Observable();
|
|
157
|
+
}
|
|
151
158
|
async intercept(context, next) {
|
|
152
159
|
var _a, _b, _c, _d, _e;
|
|
153
160
|
const options = this.reflector.get('rpcInterceptorScope', context.getHandler());
|
|
@@ -175,9 +182,7 @@ let RPCInterceptor = class RPCInterceptor {
|
|
|
175
182
|
let abortController = new AbortController();
|
|
176
183
|
/** consume 메시지 전부 cancel */
|
|
177
184
|
if (this.gracefulShutdownService.isApplicationExit) {
|
|
178
|
-
|
|
179
|
-
channel.cancel(message.fields.consumerTag);
|
|
180
|
-
return new rxjs_1.Observable();
|
|
185
|
+
return this.requeueMessageOnShutdown(channel, message);
|
|
181
186
|
}
|
|
182
187
|
if (!staticRetryCount && reDeliveryCount > 0) {
|
|
183
188
|
msaLogger.debug('RETRY_COUNT_ZERO_OR_NO_RETRY 발생 전, redelivery 발생', { traceId: ((_b = content.data.param) === null || _b === void 0 ? void 0 : _b.traceId) || 'RETRY_COUNT_ZERO_OR_NO_RETRY' }, {
|
|
@@ -189,6 +194,9 @@ let RPCInterceptor = class RPCInterceptor {
|
|
|
189
194
|
staticRetryCount,
|
|
190
195
|
});
|
|
191
196
|
const isRedeliveryMessage = await this.isRedeliveryMessage(message.properties.replyTo, execInterceptor.traceId);
|
|
197
|
+
if (this.gracefulShutdownService.isApplicationExit) {
|
|
198
|
+
return this.requeueMessageOnShutdown(channel, message);
|
|
199
|
+
}
|
|
192
200
|
if (!isRedeliveryMessage || reDeliveryCount > 5) {
|
|
193
201
|
msaLogger.debug('isRedeliveryMessage RETRY_COUNT_ZERO_OR_NO_RETRY false', { traceId: ((_c = content.data.param) === null || _c === void 0 ? void 0 : _c.traceId) || 'RETRY_COUNT_ZERO_OR_NO_RETRY' });
|
|
194
202
|
channel.sendToQueue(message.properties.replyTo, Buffer.from(JSON.stringify({
|
|
@@ -207,7 +215,7 @@ let RPCInterceptor = class RPCInterceptor {
|
|
|
207
215
|
msaLogger.debug('isRedeliveryMessage RETRY_COUNT_ZERO_OR_NO_RETRY true, 재처리하여 전달토록 하기', { traceId: ((_d = content.data.param) === null || _d === void 0 ? void 0 : _d.traceId) || 'RETRY_COUNT_ZERO_OR_NO_RETRY' });
|
|
208
216
|
}
|
|
209
217
|
const noAck = message.fields.routingKey.includes('ASYNC');
|
|
210
|
-
let localErrorProcessor = async (err) => {
|
|
218
|
+
let localErrorProcessor = async (err, preHandlerError = false) => {
|
|
211
219
|
var _a, _b;
|
|
212
220
|
const errMsg = err.response ? err.response : err.message;
|
|
213
221
|
const status = err.status ? err.status : 500;
|
|
@@ -280,7 +288,8 @@ let RPCInterceptor = class RPCInterceptor {
|
|
|
280
288
|
}
|
|
281
289
|
}
|
|
282
290
|
const sendMessage = Buffer.from(JSON.stringify(abstractAmqpHandlerData));
|
|
283
|
-
const
|
|
291
|
+
const delegatedDiagnostic = Boolean(startConnection && !preHandlerError && (0, rmq_diagnostics_function_1.hasRmqHandlerLogging)(message));
|
|
292
|
+
const diagnostic = startConnection && !delegatedDiagnostic
|
|
284
293
|
? (0, msa_logger_1.normalizeDiagnosticError)(err, isTimeoutError ? 'response_timeout' : 'service_error') : undefined;
|
|
285
294
|
const diagnosticInfo = diagnostic ? {
|
|
286
295
|
...diagnostic.diagnosticInfo,
|
|
@@ -293,26 +302,35 @@ let RPCInterceptor = class RPCInterceptor {
|
|
|
293
302
|
} : undefined;
|
|
294
303
|
if (!isAck || diagnostic) {
|
|
295
304
|
const currentContext = functions_1.AbortContext.getCurrentContext();
|
|
296
|
-
|
|
297
|
-
traceId: execInterceptor.traceId,
|
|
298
|
-
diagnosticInfo,
|
|
299
|
-
}, diagnostic ? {
|
|
300
|
-
error: diagnostic.details,
|
|
305
|
+
let logDetails = {
|
|
301
306
|
reDeliveryCount,
|
|
302
307
|
staticRetryCount,
|
|
303
308
|
currentContext: (_a = currentContext === null || currentContext === void 0 ? void 0 : currentContext.requestId) !== null && _a !== void 0 ? _a : null,
|
|
304
|
-
}
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
309
|
+
};
|
|
310
|
+
if (diagnostic) {
|
|
311
|
+
logDetails = {
|
|
312
|
+
...logDetails,
|
|
313
|
+
error: diagnostic.details,
|
|
314
|
+
};
|
|
315
|
+
}
|
|
316
|
+
else if (!delegatedDiagnostic) {
|
|
317
|
+
logDetails = {
|
|
318
|
+
error: {
|
|
319
|
+
message: err.message,
|
|
320
|
+
stack: err.stack,
|
|
321
|
+
name: err.name,
|
|
322
|
+
response: err.response,
|
|
323
|
+
reDeliveryCount,
|
|
324
|
+
staticRetryCount,
|
|
325
|
+
},
|
|
326
|
+
messagePattern: method,
|
|
327
|
+
currentContext: currentContext && currentContext.requestId ? currentContext.requestId : null,
|
|
328
|
+
};
|
|
329
|
+
}
|
|
330
|
+
msaLogger.warn(isAck ? 'RPCInterceptor 오류 처리' : '에러 응답 전송 처리', {
|
|
331
|
+
traceId: execInterceptor.traceId,
|
|
332
|
+
diagnosticInfo,
|
|
333
|
+
}, logDetails);
|
|
316
334
|
}
|
|
317
335
|
if (!isAck) {
|
|
318
336
|
channel.sendToQueue(message.properties.replyTo, sendMessage, { correlationId: message.properties.correlationId });
|
|
@@ -344,10 +362,14 @@ let RPCInterceptor = class RPCInterceptor {
|
|
|
344
362
|
}
|
|
345
363
|
}
|
|
346
364
|
catch (err) {
|
|
347
|
-
await localErrorProcessor(err);
|
|
365
|
+
await localErrorProcessor(err, true);
|
|
348
366
|
return new rxjs_1.Observable();
|
|
349
367
|
}
|
|
350
368
|
}
|
|
369
|
+
// 재전달 여부를 비동기로 확인하는 동안 종료가 시작될 수 있으므로 실행 직전에 다시 확인한다.
|
|
370
|
+
if (this.gracefulShutdownService.isApplicationExit) {
|
|
371
|
+
return this.requeueMessageOnShutdown(channel, message);
|
|
372
|
+
}
|
|
351
373
|
this.gracefulShutdownService.pushActiveInterceptors(execInterceptor);
|
|
352
374
|
// eslint-disable-next-line consistent-return
|
|
353
375
|
return functions_1.AbortContext.run(abortController.signal, execInterceptor.traceId, () => next
|
|
@@ -42,8 +42,7 @@ let GracefulShutdownModule = class GracefulShutdownModule {
|
|
|
42
42
|
this.gracefulShutdownService = gracefulShutdownService;
|
|
43
43
|
}
|
|
44
44
|
async onModuleDestroy() {
|
|
45
|
-
//
|
|
46
|
-
// 이렇게 하면 onModuleDestroy가 호출되기 전에 모든 이벤트가 처리됨
|
|
45
|
+
// provider의 onModuleDestroy에서 신규 수신을 중단한 뒤 기존 요청만 완료까지 기다린다.
|
|
47
46
|
await this.gracefulShutdownService.waitForActiveInterceptors();
|
|
48
47
|
msaLogger.info('GracefulShutdownService.onModuleDestroy: all interceptors finished, shutting down', { traceId: 'ON_MODULE_DESTROY' });
|
|
49
48
|
}
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { ServerRMQ } from '@nestjs/microservices';
|
|
2
2
|
import type { RmqOptions } from '@nestjs/microservices';
|
|
3
3
|
import type { Channel } from 'amqp-connection-manager';
|
|
4
|
-
import type { ConsumeMessage } from 'amqplib';
|
|
5
4
|
export type GracefulRmqOptions = RmqOptions['options'] & {
|
|
6
5
|
waitingEndingHandlersTimeoutMs?: number;
|
|
7
6
|
waitingEndingHandlersIntervalMs?: number;
|
|
@@ -17,12 +16,20 @@ export declare class GracefulServerRMQ extends ServerRMQ {
|
|
|
17
16
|
protected runningMessages: number;
|
|
18
17
|
protected closing: boolean;
|
|
19
18
|
protected consumerTag: string | null;
|
|
19
|
+
private stopConsumingPromise?;
|
|
20
20
|
protected readonly waitingEndingHandlersTimeoutMs: number;
|
|
21
21
|
protected readonly waitingEndingHandlersIntervalMs: number;
|
|
22
22
|
constructor(options: GracefulRmqOptions);
|
|
23
23
|
createClient(): any;
|
|
24
24
|
setupChannel(channel: Channel, callback: () => void): Promise<void>;
|
|
25
25
|
protected waitingHandlers(): Promise<void>;
|
|
26
|
-
handleMessage(message:
|
|
26
|
+
handleMessage(message: Parameters<ServerRMQ['handleMessage']>[0] | null, channel: Channel): Promise<void>;
|
|
27
|
+
/**
|
|
28
|
+
* @author jun
|
|
29
|
+
* @description 실행 중인 요청의 채널을 유지하면서 신규 메시지 수신을 중단한다.
|
|
30
|
+
* @process 종료 플래그 설정 → consumer 취소 → 이후 도착한 미실행 메시지는 재전달한다.
|
|
31
|
+
*/
|
|
32
|
+
stopConsuming(): Promise<void>;
|
|
33
|
+
private cancelConsumer;
|
|
27
34
|
close(): Promise<void>;
|
|
28
35
|
}
|
|
@@ -87,8 +87,15 @@ class GracefulServerRMQ extends microservices_1.ServerRMQ {
|
|
|
87
87
|
await channel.assertQueue(this.queue, this.queueOptions);
|
|
88
88
|
}
|
|
89
89
|
await channel.prefetch(this.options.prefetchCount || 0, this.options.isGlobalPrefetchCount);
|
|
90
|
+
if (this.closing)
|
|
91
|
+
return;
|
|
90
92
|
const { consumerTag } = await channel.consume(this.queue, (msg) => this.handleMessage(msg, channel), { noAck: false });
|
|
91
93
|
this.consumerTag = consumerTag;
|
|
94
|
+
if (this.closing) {
|
|
95
|
+
await channel.cancel(consumerTag);
|
|
96
|
+
this.consumerTag = null;
|
|
97
|
+
return;
|
|
98
|
+
}
|
|
92
99
|
callback();
|
|
93
100
|
}
|
|
94
101
|
async waitingHandlers() {
|
|
@@ -97,19 +104,42 @@ class GracefulServerRMQ extends microservices_1.ServerRMQ {
|
|
|
97
104
|
}
|
|
98
105
|
}
|
|
99
106
|
async handleMessage(message, channel) {
|
|
107
|
+
if (!message)
|
|
108
|
+
return;
|
|
109
|
+
if (this.closing) {
|
|
110
|
+
channel.nack(message, false, true);
|
|
111
|
+
return;
|
|
112
|
+
}
|
|
100
113
|
this.runningMessages += 1;
|
|
101
|
-
|
|
114
|
+
await super.handleMessage(message, channel).finally(() => {
|
|
102
115
|
this.runningMessages -= 1;
|
|
103
116
|
});
|
|
104
117
|
}
|
|
118
|
+
/**
|
|
119
|
+
* @author jun
|
|
120
|
+
* @description 실행 중인 요청의 채널을 유지하면서 신규 메시지 수신을 중단한다.
|
|
121
|
+
* @process 종료 플래그 설정 → consumer 취소 → 이후 도착한 미실행 메시지는 재전달한다.
|
|
122
|
+
*/
|
|
123
|
+
stopConsuming() {
|
|
124
|
+
this.closing = true;
|
|
125
|
+
if (!this.stopConsumingPromise)
|
|
126
|
+
this.stopConsumingPromise = this.cancelConsumer();
|
|
127
|
+
return this.stopConsumingPromise;
|
|
128
|
+
}
|
|
129
|
+
async cancelConsumer() {
|
|
130
|
+
const consumerTag = this.consumerTag;
|
|
131
|
+
if (this.channel) {
|
|
132
|
+
await this.channel.removeSetup(undefined, async (channel) => {
|
|
133
|
+
if (consumerTag)
|
|
134
|
+
await channel.cancel(consumerTag);
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
this.consumerTag = null;
|
|
138
|
+
}
|
|
105
139
|
async close() {
|
|
106
140
|
var _a;
|
|
107
|
-
this.closing = true;
|
|
108
141
|
try {
|
|
109
|
-
|
|
110
|
-
await this.channel.removeSetup(undefined, (channel) => channel.cancel(this.consumerTag));
|
|
111
|
-
}
|
|
112
|
-
this.consumerTag = null;
|
|
142
|
+
await this.stopConsuming();
|
|
113
143
|
await Promise.race([
|
|
114
144
|
this.waitingHandlers(),
|
|
115
145
|
this.waitingEndingHandlersTimeoutMs > 0
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { OnApplicationShutdown, OnModuleDestroy, OnApplicationBootstrap } from '@nestjs/common';
|
|
2
|
+
import type { GracefulServerRMQ } from './graceful-shutdown.rmq';
|
|
2
3
|
interface GracefulExecInterceptor {
|
|
3
4
|
method: string;
|
|
4
5
|
traceId?: string;
|
|
@@ -8,10 +9,17 @@ export declare class GracefulShutdownService implements OnApplicationShutdown, O
|
|
|
8
9
|
private isExecLocal?;
|
|
9
10
|
private activeInterceptors;
|
|
10
11
|
isApplicationExit: boolean;
|
|
12
|
+
private readonly rmqServers;
|
|
11
13
|
private readonly HEALTHY_FILE_PATH;
|
|
12
14
|
private shutdownSignal?;
|
|
13
15
|
private isShutdownInProgress;
|
|
14
16
|
onApplicationBootstrap(): void;
|
|
17
|
+
/**
|
|
18
|
+
* @author jun
|
|
19
|
+
* @description 종료 시 신규 수신을 중단할 RMQ 서버를 등록한다.
|
|
20
|
+
* @process bootstrap에서 등록한 서버를 요청 drain보다 먼저 중단한다.
|
|
21
|
+
*/
|
|
22
|
+
registerRmqServer(server: GracefulServerRMQ): void;
|
|
15
23
|
pushActiveInterceptors(execInterceptor: GracefulExecInterceptor): void;
|
|
16
24
|
popActiveInterceptors(execInterceptor: GracefulExecInterceptor): void;
|
|
17
25
|
getActiveInterceptorsCount(): number;
|
|
@@ -38,6 +38,7 @@ let GracefulShutdownService = class GracefulShutdownService {
|
|
|
38
38
|
constructor() {
|
|
39
39
|
this.activeInterceptors = [];
|
|
40
40
|
this.isApplicationExit = false;
|
|
41
|
+
this.rmqServers = new Set();
|
|
41
42
|
this.HEALTHY_FILE_PATH = '/tmp/healthy';
|
|
42
43
|
this.isShutdownInProgress = false;
|
|
43
44
|
}
|
|
@@ -58,6 +59,14 @@ let GracefulShutdownService = class GracefulShutdownService {
|
|
|
58
59
|
// eslint-disable-next-line max-len
|
|
59
60
|
msaLogger.debug(`GracefulShutdownService.onApplicationBootstrap isExecLocal=${this.isExecLocal}`, { traceId: 'ON_APPLICATION_BOOTSTRAP' }, { isHealthyFileExist });
|
|
60
61
|
}
|
|
62
|
+
/**
|
|
63
|
+
* @author jun
|
|
64
|
+
* @description 종료 시 신규 수신을 중단할 RMQ 서버를 등록한다.
|
|
65
|
+
* @process bootstrap에서 등록한 서버를 요청 drain보다 먼저 중단한다.
|
|
66
|
+
*/
|
|
67
|
+
registerRmqServer(server) {
|
|
68
|
+
this.rmqServers.add(server);
|
|
69
|
+
}
|
|
61
70
|
pushActiveInterceptors(execInterceptor) {
|
|
62
71
|
this.activeInterceptors.push(execInterceptor);
|
|
63
72
|
}
|
|
@@ -85,9 +94,9 @@ let GracefulShutdownService = class GracefulShutdownService {
|
|
|
85
94
|
}
|
|
86
95
|
async onModuleDestroy() {
|
|
87
96
|
msaLogger.debug('GracefulShutdownService.onModuleDestroy', { traceId: 'ON_MODULE_DESTROY' });
|
|
88
|
-
// onModuleDestroy
|
|
89
|
-
// 실제 대기는 onApplicationShutdown에서 처리됨
|
|
97
|
+
// 신규 수신을 먼저 중단하고, module의 onModuleDestroy에서 기존 요청 완료를 기다린다.
|
|
90
98
|
this.isApplicationExit = true;
|
|
99
|
+
await Promise.all([...this.rmqServers].map(server => server.stopConsuming()));
|
|
91
100
|
}
|
|
92
101
|
async waitForActiveInterceptors() {
|
|
93
102
|
if (this.isExecLocal) {
|
package/package.json
CHANGED
|
@@ -2,7 +2,8 @@ const assert = require('node:assert/strict');
|
|
|
2
2
|
const { EventEmitter } = require('node:events');
|
|
3
3
|
const test = require('node:test');
|
|
4
4
|
const { ServerRMQ, RmqContext } = require('@nestjs/microservices');
|
|
5
|
-
const { lastValueFrom, throwError } = require('rxjs');
|
|
5
|
+
const { from, lastValueFrom, throwError } = require('rxjs');
|
|
6
|
+
const { mergeAll } = require('rxjs/operators');
|
|
6
7
|
const { MsaLogger } = require('@yolo-croket-dev/msa-logger');
|
|
7
8
|
const { GracefulServerRMQ } = require('../modules/graceful-shutdown/graceful-shutdown.rmq');
|
|
8
9
|
const { LoggingServiceInterceptor } = require('../interceptors/logging-service.interceptor');
|
|
@@ -148,3 +149,90 @@ test('RPC-only handler failure records its trace and channel on the existing war
|
|
|
148
149
|
assert.equal(warnings[0].info.diagnosticInfo.event, 'handler_failed');
|
|
149
150
|
assert.equal(acknowledgements.length, 1);
|
|
150
151
|
});
|
|
152
|
+
|
|
153
|
+
test('combined consumer interceptors record one detailed failure in either order', async t => {
|
|
154
|
+
const warnings = [];
|
|
155
|
+
const errors = [];
|
|
156
|
+
t.mock.method(MsaLogger.prototype, 'warn', (message, info, details) => warnings.push({ message, info, details }));
|
|
157
|
+
t.mock.method(MsaLogger.prototype, 'printExceptionFliterService', (info, details) => errors.push({ info, details }));
|
|
158
|
+
for (const method of ['printInputInterceptorService', 'printOutService']) t.mock.method(MsaLogger.prototype, method, () => {});
|
|
159
|
+
|
|
160
|
+
for (const order of ['logging-first', 'rpc-first']) {
|
|
161
|
+
warnings.length = 0;
|
|
162
|
+
errors.length = 0;
|
|
163
|
+
const acknowledgements = [];
|
|
164
|
+
const replies = [];
|
|
165
|
+
const channel = {
|
|
166
|
+
sendToQueue: (...args) => replies.push(args),
|
|
167
|
+
ack: message => acknowledgements.push(message),
|
|
168
|
+
};
|
|
169
|
+
const observer = new RmqConnectionObserver('USER_SERVICE', 'consumer', undefined, () => {});
|
|
170
|
+
registerRmqObservation(channel, observer);
|
|
171
|
+
const data = { payload: {}, param: { traceId: `combined-${order}` } };
|
|
172
|
+
const message = {
|
|
173
|
+
content: Buffer.from(JSON.stringify({ pattern: 'user.fail', data })),
|
|
174
|
+
properties: { headers: {}, correlationId: 'correlation', replyTo: 'reply.queue' },
|
|
175
|
+
fields: { routingKey: 'USER_SERVICE', consumerTag: 'consumer' },
|
|
176
|
+
};
|
|
177
|
+
const rmqContext = new RmqContext([message, channel, 'user.fail']);
|
|
178
|
+
const execution = {
|
|
179
|
+
getArgs: () => [data, rmqContext],
|
|
180
|
+
getHandler: () => function handler() {},
|
|
181
|
+
switchToRpc: () => ({ getContext: () => rmqContext, getData: () => data }),
|
|
182
|
+
switchToHttp: () => ({ getRequest: () => data }),
|
|
183
|
+
};
|
|
184
|
+
const shutdown = { isApplicationExit: false, pushActiveInterceptors: () => {}, popActiveInterceptors: () => {} };
|
|
185
|
+
const rpc = new RPCInterceptor({ get: () => undefined }, shutdown);
|
|
186
|
+
const logging = new LoggingServiceInterceptor();
|
|
187
|
+
const failure = Error(`handler failed ${order}`);
|
|
188
|
+
const handler = { handle: () => throwError(() => failure) };
|
|
189
|
+
const stream = order === 'logging-first'
|
|
190
|
+
? logging.intercept(execution, { handle: () => from(rpc.intercept(execution, handler)).pipe(mergeAll()) })
|
|
191
|
+
: await rpc.intercept(execution, { handle: () => logging.intercept(execution, handler) });
|
|
192
|
+
|
|
193
|
+
await assert.rejects(lastValueFrom(stream), actual => actual === failure);
|
|
194
|
+
assert.equal(errors.length, 1, `${order}: detailed error must be logged once`);
|
|
195
|
+
assert.equal(errors[0].info.traceId, data.param.traceId);
|
|
196
|
+
assert.equal(errors[0].info.diagnosticInfo.connectionId, observer.connectionId);
|
|
197
|
+
assert.equal(errors[0].details.errors[0].message, failure.message);
|
|
198
|
+
assert.equal(warnings.filter(record => record.info.diagnosticInfo).length, 0, `${order}: RPC must not duplicate diagnostics`);
|
|
199
|
+
assert.equal(replies.length, 1);
|
|
200
|
+
assert.equal(acknowledgements.length, 1);
|
|
201
|
+
}
|
|
202
|
+
});
|
|
203
|
+
|
|
204
|
+
test('RPC guard failure retains diagnostics when a logging interceptor surrounds it', async t => {
|
|
205
|
+
const warnings = [];
|
|
206
|
+
t.mock.method(MsaLogger.prototype, 'warn', (message, info, details) => warnings.push({ message, info, details }));
|
|
207
|
+
for (const method of ['printInputInterceptorService', 'printOutService']) t.mock.method(MsaLogger.prototype, method, () => {});
|
|
208
|
+
const replies = [];
|
|
209
|
+
const channel = { sendToQueue: (...args) => replies.push(args), ack: () => {} };
|
|
210
|
+
registerRmqObservation(channel, new RmqConnectionObserver('USER_SERVICE', 'consumer', undefined, () => {}));
|
|
211
|
+
const data = { payload: {}, param: { traceId: 'combined-guard' } };
|
|
212
|
+
const message = {
|
|
213
|
+
content: Buffer.from(JSON.stringify({ pattern: 'user.guard', data })),
|
|
214
|
+
properties: { headers: {}, correlationId: 'correlation', replyTo: 'reply.queue' },
|
|
215
|
+
fields: { routingKey: 'USER_SERVICE', consumerTag: 'consumer' },
|
|
216
|
+
};
|
|
217
|
+
const rmqContext = new RmqContext([message, channel, 'user.guard']);
|
|
218
|
+
const execution = {
|
|
219
|
+
getArgs: () => [data, rmqContext],
|
|
220
|
+
getHandler: () => function handler() {},
|
|
221
|
+
switchToRpc: () => ({ getContext: () => rmqContext, getData: () => data }),
|
|
222
|
+
switchToHttp: () => ({ getRequest: () => data }),
|
|
223
|
+
};
|
|
224
|
+
const rpc = new RPCInterceptor({ get: () => ({ isAdminGuard: true }) }, { isApplicationExit: false });
|
|
225
|
+
let guardProcess;
|
|
226
|
+
new LoggingServiceInterceptor().intercept(execution, {
|
|
227
|
+
handle: () => {
|
|
228
|
+
guardProcess = rpc.intercept(execution, { handle: () => { throw Error('guard should reject first'); } });
|
|
229
|
+
return from(guardProcess).pipe(mergeAll());
|
|
230
|
+
},
|
|
231
|
+
});
|
|
232
|
+
await guardProcess;
|
|
233
|
+
assert.equal(warnings.length, 1);
|
|
234
|
+
assert.equal(warnings[0].info.traceId, 'combined-guard');
|
|
235
|
+
assert.equal(warnings[0].info.diagnosticInfo.event, 'handler_failed');
|
|
236
|
+
assert.equal(warnings[0].details.error.errors[0].message, 'ADMIN_ONLY_ACCESS');
|
|
237
|
+
assert.equal(replies.length, 1);
|
|
238
|
+
});
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
const assert = require('node:assert/strict');
|
|
2
|
+
const test = require('node:test');
|
|
3
|
+
const { RmqContext, ServerRMQ } = require('@nestjs/microservices');
|
|
4
|
+
const { Subject } = require('rxjs');
|
|
5
|
+
const { MsaLogger } = require('@yolo-croket-dev/msa-logger');
|
|
6
|
+
const { RPCInterceptor } = require('../interceptors/rpc.interceptor');
|
|
7
|
+
const { GracefulShutdownService } = require('../modules/graceful-shutdown/graceful-shutdown.service');
|
|
8
|
+
const { GracefulServerRMQ } = require('../modules/graceful-shutdown/graceful-shutdown.rmq');
|
|
9
|
+
|
|
10
|
+
function createRequest(channel, headers = {}) {
|
|
11
|
+
const data = { payload: {}, param: { traceId: 'shutdown-test' } };
|
|
12
|
+
const message = {
|
|
13
|
+
content: Buffer.from(JSON.stringify({ pattern: 'test.read', data })),
|
|
14
|
+
properties: { headers, correlationId: 'request-1', replyTo: 'reply-queue' },
|
|
15
|
+
fields: { consumerTag: 'consumer-1', routingKey: 'test.read' },
|
|
16
|
+
};
|
|
17
|
+
const rmq = new RmqContext([message, channel, 'test.read']);
|
|
18
|
+
return {
|
|
19
|
+
message,
|
|
20
|
+
context: { getHandler: () => 'test.read', getArgs: () => [data, rmq] },
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
test('종료 중 받은 미실행 RPC는 즉시 재전달하고 기존 요청의 ACK에는 손대지 않는다', async t => {
|
|
25
|
+
t.mock.method(MsaLogger.prototype, 'debug', () => {});
|
|
26
|
+
const shutdown = new GracefulShutdownService();
|
|
27
|
+
shutdown.isApplicationExit = true;
|
|
28
|
+
const calls = [];
|
|
29
|
+
const channel = {
|
|
30
|
+
cancel: async tag => { calls.push(['cancel', tag]); },
|
|
31
|
+
nack: (...args) => { calls.push(['nack', ...args]); },
|
|
32
|
+
ack: (...args) => { calls.push(['ack', ...args]); },
|
|
33
|
+
sendToQueue: (...args) => { calls.push(['reply', ...args]); },
|
|
34
|
+
};
|
|
35
|
+
const { message, context } = createRequest(channel);
|
|
36
|
+
const next = { handle: t.mock.fn(() => { throw Error('실행하면 안 되는 신규 요청'); }) };
|
|
37
|
+
const interceptor = new RPCInterceptor({ get: () => undefined }, shutdown);
|
|
38
|
+
await interceptor.intercept(context, next);
|
|
39
|
+
assert.deepEqual(calls, [
|
|
40
|
+
['cancel', 'consumer-1'],
|
|
41
|
+
['nack', message, false, true],
|
|
42
|
+
]);
|
|
43
|
+
assert.equal(next.handle.mock.callCount(), 0);
|
|
44
|
+
assert.equal(shutdown.getActiveInterceptorsCount(), 0);
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
test('종료 전에 실행한 RPC는 계속 처리하고 종료 이후 도착한 메시지만 반환한다', async t => {
|
|
48
|
+
t.mock.method(MsaLogger.prototype, 'debug', () => {});
|
|
49
|
+
const shutdown = new GracefulShutdownService();
|
|
50
|
+
const calls = [];
|
|
51
|
+
let consume;
|
|
52
|
+
const channel = {
|
|
53
|
+
assertQueue: async () => {},
|
|
54
|
+
prefetch: async () => {},
|
|
55
|
+
consume: async (_queue, callback) => { consume = callback; return { consumerTag: 'consumer-1' }; },
|
|
56
|
+
cancel: async tag => { calls.push(['cancel', tag]); },
|
|
57
|
+
nack: (...args) => { calls.push(['nack', ...args]); },
|
|
58
|
+
ack: (...args) => { calls.push(['ack', ...args]); },
|
|
59
|
+
};
|
|
60
|
+
const server = new GracefulServerRMQ({ queue: 'TEST', noAck: false });
|
|
61
|
+
server.channel = { removeSetup: async (_setup, teardown) => teardown(channel) };
|
|
62
|
+
await server.setupChannel(channel, () => {});
|
|
63
|
+
const { message: runningMessage, context } = createRequest(channel);
|
|
64
|
+
const result = new Subject();
|
|
65
|
+
t.after(() => result.complete());
|
|
66
|
+
const interceptor = new RPCInterceptor({ get: () => undefined }, shutdown);
|
|
67
|
+
const response = await interceptor.intercept(context, { handle: () => result });
|
|
68
|
+
const values = [];
|
|
69
|
+
response.subscribe(value => values.push(value));
|
|
70
|
+
assert.equal(shutdown.getActiveInterceptorsCount(), 1);
|
|
71
|
+
|
|
72
|
+
shutdown.registerRmqServer(server);
|
|
73
|
+
await shutdown.onModuleDestroy();
|
|
74
|
+
assert.equal(shutdown.isApplicationExit, true);
|
|
75
|
+
assert.equal(shutdown.getActiveInterceptorsCount(), 1);
|
|
76
|
+
assert.deepEqual(calls, [['cancel', 'consumer-1']]);
|
|
77
|
+
const { message: newMessage } = createRequest(channel);
|
|
78
|
+
await consume(newMessage);
|
|
79
|
+
await consume(null);
|
|
80
|
+
assert.deepEqual(calls, [['cancel', 'consumer-1'], ['nack', newMessage, false, true]]);
|
|
81
|
+
|
|
82
|
+
result.next({ data: { ok: true } });
|
|
83
|
+
result.complete();
|
|
84
|
+
assert.deepEqual(values, [{ data: { ok: true }, isError: false }]);
|
|
85
|
+
assert.deepEqual(calls.at(-1), ['ack', runningMessage]);
|
|
86
|
+
assert.equal(shutdown.getActiveInterceptorsCount(), 0);
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
test('재전달 확인 중 종료가 시작되면 실행 직전에 요청을 반환한다', async t => {
|
|
90
|
+
t.mock.method(MsaLogger.prototype, 'debug', () => {});
|
|
91
|
+
const shutdown = new GracefulShutdownService();
|
|
92
|
+
const calls = [];
|
|
93
|
+
const channel = {
|
|
94
|
+
cancel: async tag => { calls.push(['cancel', tag]); },
|
|
95
|
+
nack: (...args) => { calls.push(['nack', ...args]); },
|
|
96
|
+
};
|
|
97
|
+
const { message, context } = createRequest(channel, { 'x-delivery-count': 1 });
|
|
98
|
+
const next = { handle: t.mock.fn(() => { throw Error('실행하면 안 되는 신규 요청'); }) };
|
|
99
|
+
const interceptor = new RPCInterceptor({ get: () => undefined }, shutdown);
|
|
100
|
+
t.mock.method(interceptor, 'isRedeliveryMessage', async () => {
|
|
101
|
+
shutdown.isApplicationExit = true;
|
|
102
|
+
return true;
|
|
103
|
+
});
|
|
104
|
+
await interceptor.intercept(context, next);
|
|
105
|
+
assert.deepEqual(calls, [['cancel', 'consumer-1'], ['nack', message, false, true]]);
|
|
106
|
+
assert.equal(next.handle.mock.callCount(), 0);
|
|
107
|
+
assert.equal(shutdown.getActiveInterceptorsCount(), 0);
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
for (const scenario of [
|
|
111
|
+
{ result: false, deliveryCount: 1, name: '재전달 확인 실패' },
|
|
112
|
+
{ result: true, deliveryCount: 6, name: '재전달 횟수 초과' },
|
|
113
|
+
]) {
|
|
114
|
+
test(`종료가 시작되면 ${scenario.name} 요청도 DLQ 대신 재큐잉한다`, async t => {
|
|
115
|
+
t.mock.method(MsaLogger.prototype, 'debug', () => {});
|
|
116
|
+
const shutdown = new GracefulShutdownService();
|
|
117
|
+
const calls = [];
|
|
118
|
+
const channel = {
|
|
119
|
+
cancel: async tag => { calls.push(['cancel', tag]); },
|
|
120
|
+
nack: (...args) => { calls.push(['nack', ...args]); },
|
|
121
|
+
sendToQueue: () => { calls.push(['reply']); },
|
|
122
|
+
};
|
|
123
|
+
const { message, context } = createRequest(channel, { 'x-delivery-count': scenario.deliveryCount });
|
|
124
|
+
const next = { handle: t.mock.fn(() => { throw Error('실행하면 안 되는 신규 요청'); }) };
|
|
125
|
+
const interceptor = new RPCInterceptor({ get: () => undefined }, shutdown);
|
|
126
|
+
t.mock.method(interceptor, 'isRedeliveryMessage', async () => {
|
|
127
|
+
shutdown.isApplicationExit = true;
|
|
128
|
+
return scenario.result;
|
|
129
|
+
});
|
|
130
|
+
await interceptor.intercept(context, next);
|
|
131
|
+
assert.deepEqual(calls, [['cancel', 'consumer-1'], ['nack', message, false, true]]);
|
|
132
|
+
assert.equal(next.handle.mock.callCount(), 0);
|
|
133
|
+
assert.equal(shutdown.getActiveInterceptorsCount(), 0);
|
|
134
|
+
});
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
test('수신 중단과 서버 close가 겹쳐도 consumer는 한 번만 취소한다', async t => {
|
|
138
|
+
const calls = [];
|
|
139
|
+
let finishCancel;
|
|
140
|
+
const pendingCancel = new Promise(resolve => { finishCancel = resolve; });
|
|
141
|
+
t.after(() => finishCancel());
|
|
142
|
+
const channel = { cancel: async tag => { calls.push(['cancel', tag]); await pendingCancel; } };
|
|
143
|
+
const server = new GracefulServerRMQ({ queue: 'TEST', waitingEndingHandlersTimeoutMs: 1 });
|
|
144
|
+
server.consumerTag = 'consumer-1';
|
|
145
|
+
server.channel = { removeSetup: async (_setup, teardown) => teardown(channel) };
|
|
146
|
+
t.mock.method(ServerRMQ.prototype, 'close', async () => { calls.push(['close']); });
|
|
147
|
+
const stopping = server.stopConsuming();
|
|
148
|
+
const closing = server.close();
|
|
149
|
+
assert.deepEqual(calls, [['cancel', 'consumer-1']]);
|
|
150
|
+
finishCancel();
|
|
151
|
+
await Promise.all([stopping, closing]);
|
|
152
|
+
assert.deepEqual(calls, [['cancel', 'consumer-1'], ['close']]);
|
|
153
|
+
});
|
|
154
|
+
|
|
155
|
+
test('채널 설정 중 종료가 시작되면 새 consumer를 만들지 않는다', async () => {
|
|
156
|
+
let finishPrefetch;
|
|
157
|
+
const prefetch = new Promise(resolve => { finishPrefetch = resolve; });
|
|
158
|
+
const calls = [];
|
|
159
|
+
const channel = {
|
|
160
|
+
assertQueue: async () => {},
|
|
161
|
+
prefetch: async () => prefetch,
|
|
162
|
+
consume: async () => { calls.push('consume'); return { consumerTag: 'new-consumer' }; },
|
|
163
|
+
};
|
|
164
|
+
const server = new GracefulServerRMQ({ queue: 'TEST' });
|
|
165
|
+
const setup = server.setupChannel(channel, () => calls.push('ready'));
|
|
166
|
+
await server.stopConsuming();
|
|
167
|
+
finishPrefetch();
|
|
168
|
+
await setup;
|
|
169
|
+
assert.deepEqual(calls, []);
|
|
170
|
+
});
|
|
171
|
+
|
|
172
|
+
test('consumer 등록이 끝나기 전에 종료가 시작되면 해당 consumer도 취소한다', async () => {
|
|
173
|
+
let finishConsume;
|
|
174
|
+
let enteredConsume;
|
|
175
|
+
const started = new Promise(resolve => { enteredConsume = resolve; });
|
|
176
|
+
const consumer = new Promise(resolve => { finishConsume = resolve; });
|
|
177
|
+
const calls = [];
|
|
178
|
+
const channel = {
|
|
179
|
+
assertQueue: async () => {},
|
|
180
|
+
prefetch: async () => {},
|
|
181
|
+
consume: async () => { enteredConsume(); return consumer; },
|
|
182
|
+
cancel: async tag => { calls.push(['cancel', tag]); },
|
|
183
|
+
};
|
|
184
|
+
const server = new GracefulServerRMQ({ queue: 'TEST' });
|
|
185
|
+
const setup = server.setupChannel(channel, () => calls.push(['ready']));
|
|
186
|
+
await started;
|
|
187
|
+
await server.stopConsuming();
|
|
188
|
+
finishConsume({ consumerTag: 'new-consumer' });
|
|
189
|
+
await setup;
|
|
190
|
+
assert.deepEqual(calls, [['cancel', 'new-consumer']]);
|
|
191
|
+
});
|