dl-commons 0.1.8 → 0.1.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/dead-messages/classify-error.d.ts +8 -0
- package/dist/dead-messages/classify-error.js +41 -0
- package/dist/dead-messages/classify-error.js.map +1 -0
- package/dist/dead-messages/dead-letter-retry.error.d.ts +3 -0
- package/dist/dead-messages/dead-letter-retry.error.js +11 -0
- package/dist/dead-messages/dead-letter-retry.error.js.map +1 -0
- package/dist/dead-messages/dead-message.schema.d.ts +282 -0
- package/dist/dead-messages/dead-message.schema.js +130 -0
- package/dist/dead-messages/dead-message.schema.js.map +1 -0
- package/dist/dead-messages/dead-messages.const.d.ts +8 -0
- package/dist/dead-messages/dead-messages.const.js +12 -0
- package/dist/dead-messages/dead-messages.const.js.map +1 -0
- package/dist/dead-messages/dead-messages.controller.d.ts +17 -0
- package/dist/dead-messages/dead-messages.controller.js +97 -0
- package/dist/dead-messages/dead-messages.controller.js.map +1 -0
- package/dist/dead-messages/dead-messages.dto.d.ts +14 -0
- package/dist/dead-messages/dead-messages.dto.js +54 -0
- package/dist/dead-messages/dead-messages.dto.js.map +1 -0
- package/dist/dead-messages/dead-messages.module.d.ts +2 -0
- package/dist/dead-messages/dead-messages.module.js +28 -0
- package/dist/dead-messages/dead-messages.module.js.map +1 -0
- package/dist/dead-messages/dead-messages.service.d.ts +94 -0
- package/dist/dead-messages/dead-messages.service.js +281 -0
- package/dist/dead-messages/dead-messages.service.js.map +1 -0
- package/dist/dead-messages/kafka-consumer-runner.d.ts +47 -0
- package/dist/dead-messages/kafka-consumer-runner.js +258 -0
- package/dist/dead-messages/kafka-consumer-runner.js.map +1 -0
- package/dist/dead-messages/kafka-logging.d.ts +9 -0
- package/dist/dead-messages/kafka-logging.js +29 -0
- package/dist/dead-messages/kafka-logging.js.map +1 -0
- package/dist/dead-messages/unprocessable-message.error.d.ts +9 -0
- package/dist/dead-messages/unprocessable-message.error.js +37 -0
- package/dist/dead-messages/unprocessable-message.error.js.map +1 -0
- package/dist/index.d.ts +12 -1
- package/dist/index.js +42 -1
- package/dist/index.js.map +1 -1
- package/dist/models/interfaces/user.interface.d.ts +1 -1
- package/dist/models/schemas/base.schema.d.ts +5 -1
- package/dist/models/schemas/embedded/archive.schema.d.ts +5 -1
- package/dist/models/schemas/embedded/attachment.schema.d.ts +5 -1
- package/dist/models/schemas/embedded/author.schema.d.ts +5 -1
- package/dist/models/schemas/embedded/contact.schema.d.ts +5 -1
- package/dist/models/schemas/embedded/crop-area.schema.d.ts +5 -1
- package/dist/models/schemas/embedded/date-time-range.schema.d.ts +5 -1
- package/dist/models/schemas/embedded/dictionary-embedded.schema.d.ts +5 -1
- package/dist/models/schemas/embedded/group-session.schema.d.ts +5 -1
- package/dist/models/schemas/embedded/i18n.embedded.d.ts +5 -1
- package/dist/models/schemas/embedded/location.schema.d.ts +5 -1
- package/dist/models/schemas/embedded/person.schema.d.ts +5 -1
- package/dist/models/schemas/embedded/physical-address.schema.d.ts +5 -1
- package/dist/models/schemas/embedded/picture.schema.d.ts +5 -1
- package/dist/models/schemas/embedded/record.embedded.d.ts +5 -1
- package/dist/models/schemas/embedded/residence.schema.d.ts +5 -1
- package/dist/models/schemas/embedded/s3.schema.d.ts +40 -8
- package/dist/models/schemas/embedded/session-user.schema.d.ts +5 -1
- package/dist/models/schemas/embedded/student-embedded.schema.d.ts +5 -1
- package/dist/models/schemas/embedded/teacher.schema.d.ts +5 -1
- package/dist/models/schemas/embedded/time.schema.d.ts +5 -1
- package/dist/models/schemas/embedded/virtual-address.schema.d.ts +5 -1
- package/dist/models/schemas/identifable.schema.d.ts +5 -1
- package/dist/models/schemas/parent.schema.d.ts +5 -1
- package/dist/models/schemas/session.schema.d.ts +5 -1
- package/package.json +38 -3
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { RESTRequestBody } from '../models/dtos/wrappers/rest-wrapper.class';
|
|
2
|
+
import { DeadMessageStatus } from './dead-message.schema';
|
|
3
|
+
export declare const DEAD_MESSAGE_STATUSES: DeadMessageStatus[];
|
|
4
|
+
export declare class DeadMessageActionData {
|
|
5
|
+
by: string;
|
|
6
|
+
}
|
|
7
|
+
export declare class DeadMessageActionRB extends RESTRequestBody {
|
|
8
|
+
data: DeadMessageActionData;
|
|
9
|
+
}
|
|
10
|
+
export declare class DeadMessagesQuery {
|
|
11
|
+
status?: DeadMessageStatus[];
|
|
12
|
+
topic?: string;
|
|
13
|
+
limit?: number;
|
|
14
|
+
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
|
+
};
|
|
8
|
+
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
9
|
+
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
|
+
};
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.DeadMessagesQuery = exports.DeadMessageActionRB = exports.DeadMessageActionData = exports.DEAD_MESSAGE_STATUSES = void 0;
|
|
13
|
+
const class_transformer_1 = require("class-transformer");
|
|
14
|
+
const class_validator_1 = require("class-validator");
|
|
15
|
+
const rest_wrapper_class_1 = require("../models/dtos/wrappers/rest-wrapper.class");
|
|
16
|
+
exports.DEAD_MESSAGE_STATUSES = ['retrying', 'dead', 'replayed', 'discarded'];
|
|
17
|
+
class DeadMessageActionData {
|
|
18
|
+
}
|
|
19
|
+
exports.DeadMessageActionData = DeadMessageActionData;
|
|
20
|
+
__decorate([
|
|
21
|
+
(0, class_validator_1.IsString)(),
|
|
22
|
+
(0, class_validator_1.Length)(1, 100),
|
|
23
|
+
__metadata("design:type", String)
|
|
24
|
+
], DeadMessageActionData.prototype, "by", void 0);
|
|
25
|
+
class DeadMessageActionRB extends rest_wrapper_class_1.RESTRequestBody {
|
|
26
|
+
}
|
|
27
|
+
exports.DeadMessageActionRB = DeadMessageActionRB;
|
|
28
|
+
__decorate([
|
|
29
|
+
(0, class_validator_1.ValidateNested)(),
|
|
30
|
+
(0, class_validator_1.IsObject)(),
|
|
31
|
+
(0, class_transformer_1.Type)(() => DeadMessageActionData),
|
|
32
|
+
__metadata("design:type", DeadMessageActionData)
|
|
33
|
+
], DeadMessageActionRB.prototype, "data", void 0);
|
|
34
|
+
class DeadMessagesQuery {
|
|
35
|
+
}
|
|
36
|
+
exports.DeadMessagesQuery = DeadMessagesQuery;
|
|
37
|
+
__decorate([
|
|
38
|
+
(0, class_validator_1.IsOptional)(),
|
|
39
|
+
(0, class_transformer_1.Transform)(({ value }) => (typeof value === 'string' ? value.split(',').filter((s) => s.length > 0) : value)),
|
|
40
|
+
(0, class_validator_1.IsIn)(exports.DEAD_MESSAGE_STATUSES, { each: true }),
|
|
41
|
+
__metadata("design:type", Array)
|
|
42
|
+
], DeadMessagesQuery.prototype, "status", void 0);
|
|
43
|
+
__decorate([
|
|
44
|
+
(0, class_validator_1.IsOptional)(),
|
|
45
|
+
(0, class_validator_1.IsString)(),
|
|
46
|
+
__metadata("design:type", String)
|
|
47
|
+
], DeadMessagesQuery.prototype, "topic", void 0);
|
|
48
|
+
__decorate([
|
|
49
|
+
(0, class_validator_1.IsOptional)(),
|
|
50
|
+
(0, class_transformer_1.Type)(() => Number),
|
|
51
|
+
(0, class_validator_1.IsInt)(),
|
|
52
|
+
__metadata("design:type", Number)
|
|
53
|
+
], DeadMessagesQuery.prototype, "limit", void 0);
|
|
54
|
+
//# sourceMappingURL=dead-messages.dto.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dead-messages.dto.js","sourceRoot":"","sources":["../../src/dead-messages/dead-messages.dto.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,yDAAoD;AACpD,qDAAsG;AACtG,mFAA6E;AAGhE,QAAA,qBAAqB,GAAwB,CAAC,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,WAAW,CAAC,CAAC;AAExG,MAAa,qBAAqB;CAIjC;AAJD,sDAIC;AADG;IAFC,IAAA,0BAAQ,GAAE;IACV,IAAA,wBAAM,EAAC,CAAC,EAAE,GAAG,CAAC;;iDACJ;AAGf,MAAa,mBAAoB,SAAQ,oCAAe;CAMvD;AAND,kDAMC;AADG;IAHC,IAAA,gCAAc,GAAE;IAChB,IAAA,0BAAQ,GAAE;IACV,IAAA,wBAAI,EAAC,GAAG,EAAE,CAAC,qBAAqB,CAAC;8BAC5B,qBAAqB;iDAAC;AAGhC,MAAa,iBAAiB;CAe7B;AAfD,8CAeC;AAXG;IAHC,IAAA,4BAAU,GAAE;IACZ,IAAA,6BAAS,EAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAC5G,IAAA,sBAAI,EAAC,6BAAqB,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;;iDACf;AAI7B;IAFC,IAAA,4BAAU,GAAE;IACZ,IAAA,0BAAQ,GAAE;;gDACI;AAMf;IAHC,IAAA,4BAAU,GAAE;IACZ,IAAA,wBAAI,EAAC,GAAG,EAAE,CAAC,MAAM,CAAC;IAClB,IAAA,uBAAK,GAAE;;gDACO"}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
|
+
};
|
|
8
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
9
|
+
exports.DeadMessagesModule = void 0;
|
|
10
|
+
const common_1 = require("@nestjs/common");
|
|
11
|
+
const mongoose_1 = require("@nestjs/mongoose");
|
|
12
|
+
const app_logger_service_1 = require("../services/app-logger.service");
|
|
13
|
+
const dead_message_schema_1 = require("./dead-message.schema");
|
|
14
|
+
const dead_messages_const_1 = require("./dead-messages.const");
|
|
15
|
+
const dead_messages_controller_1 = require("./dead-messages.controller");
|
|
16
|
+
const dead_messages_service_1 = require("./dead-messages.service");
|
|
17
|
+
let DeadMessagesModule = class DeadMessagesModule {
|
|
18
|
+
};
|
|
19
|
+
exports.DeadMessagesModule = DeadMessagesModule;
|
|
20
|
+
exports.DeadMessagesModule = DeadMessagesModule = __decorate([
|
|
21
|
+
(0, common_1.Module)({
|
|
22
|
+
imports: [mongoose_1.MongooseModule.forFeature([{ name: dead_message_schema_1.DeadMessage.name, schema: dead_message_schema_1.DeadMessageSchema }])],
|
|
23
|
+
controllers: [dead_messages_controller_1.DeadMessagesController],
|
|
24
|
+
providers: [dead_messages_service_1.DeadMessagesService, app_logger_service_1.AppLoggerService, { provide: dead_messages_const_1.KAFKA_ON_FATAL, useValue: () => process.kill(process.pid, 'SIGTERM') }],
|
|
25
|
+
exports: [dead_messages_service_1.DeadMessagesService, dead_messages_const_1.KAFKA_ON_FATAL],
|
|
26
|
+
})
|
|
27
|
+
], DeadMessagesModule);
|
|
28
|
+
//# sourceMappingURL=dead-messages.module.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dead-messages.module.js","sourceRoot":"","sources":["../../src/dead-messages/dead-messages.module.ts"],"names":[],"mappings":";;;;;;;;;AAAA,2CAAwC;AACxC,+CAAkD;AAClD,uEAAkE;AAClE,+DAAuE;AACvE,+DAAuD;AACvD,yEAAoE;AACpE,mEAA8D;AAQvD,IAAM,kBAAkB,GAAxB,MAAM,kBAAkB;CAAG,CAAA;AAArB,gDAAkB;6BAAlB,kBAAkB;IAN9B,IAAA,eAAM,EAAC;QACJ,OAAO,EAAE,CAAC,yBAAc,CAAC,UAAU,CAAC,CAAC,EAAE,IAAI,EAAE,iCAAW,CAAC,IAAI,EAAE,MAAM,EAAE,uCAAiB,EAAE,CAAC,CAAC,CAAC;QAC7F,WAAW,EAAE,CAAC,iDAAsB,CAAC;QACrC,SAAS,EAAE,CAAC,2CAAmB,EAAE,qCAAgB,EAAE,EAAE,OAAO,EAAE,oCAAc,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,SAAS,CAAC,EAAE,CAAC;QACrI,OAAO,EAAE,CAAC,2CAAmB,EAAE,oCAAc,CAAC;KACjD,CAAC;GACW,kBAAkB,CAAG"}
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import type { KafkaMessage } from 'kafkajs';
|
|
2
|
+
import { Model } from 'mongoose';
|
|
3
|
+
import { AppLoggerService } from '../services/app-logger.service';
|
|
4
|
+
import { ErrorKind, IDescribedError } from './classify-error';
|
|
5
|
+
import { DeadMessage, DeadMessageDocument, DeadMessageReason, DeadMessageStatus } from './dead-message.schema';
|
|
6
|
+
export { DeadMessageReason, DeadMessageStatus } from './dead-message.schema';
|
|
7
|
+
export interface IKafkaInput {
|
|
8
|
+
topic: string;
|
|
9
|
+
partition: number;
|
|
10
|
+
message: KafkaMessage;
|
|
11
|
+
}
|
|
12
|
+
export interface IDeadLetterInput extends IKafkaInput {
|
|
13
|
+
group: string;
|
|
14
|
+
}
|
|
15
|
+
export interface IDeadMessageIdentity {
|
|
16
|
+
group: string;
|
|
17
|
+
topic: string;
|
|
18
|
+
partition: number;
|
|
19
|
+
offset: string;
|
|
20
|
+
timestamp: string;
|
|
21
|
+
}
|
|
22
|
+
export interface IRecordOutcome {
|
|
23
|
+
outcome: 'retrying' | 'dead' | 'already';
|
|
24
|
+
status: DeadMessageStatus;
|
|
25
|
+
reason: DeadMessageReason | null;
|
|
26
|
+
attempts: number;
|
|
27
|
+
}
|
|
28
|
+
export type DeadMessageReplayer = (input: IKafkaInput) => Promise<void>;
|
|
29
|
+
export type DeadMessageReplayAnswer = {
|
|
30
|
+
status: 'replayed';
|
|
31
|
+
} | {
|
|
32
|
+
status: 'discarded';
|
|
33
|
+
} | {
|
|
34
|
+
status: 'dead';
|
|
35
|
+
error: {
|
|
36
|
+
name: string;
|
|
37
|
+
keyword: string | null;
|
|
38
|
+
};
|
|
39
|
+
};
|
|
40
|
+
export type IEncodedBytes = {
|
|
41
|
+
encoding: 'utf8' | 'base64';
|
|
42
|
+
text: string;
|
|
43
|
+
};
|
|
44
|
+
export interface IDeadMessageView {
|
|
45
|
+
_id: string;
|
|
46
|
+
group: string;
|
|
47
|
+
topic: string;
|
|
48
|
+
partition: number;
|
|
49
|
+
offset: string;
|
|
50
|
+
timestamp: string;
|
|
51
|
+
key: IEncodedBytes | null;
|
|
52
|
+
value: IEncodedBytes | null;
|
|
53
|
+
headers: {
|
|
54
|
+
key: string;
|
|
55
|
+
value: IEncodedBytes | null;
|
|
56
|
+
}[];
|
|
57
|
+
status: DeadMessageStatus;
|
|
58
|
+
reason: DeadMessageReason | null;
|
|
59
|
+
error: IDescribedError;
|
|
60
|
+
attempts: number;
|
|
61
|
+
firstFailedAt: Date;
|
|
62
|
+
lastFailedAt: Date;
|
|
63
|
+
resolvedAt: Date | null;
|
|
64
|
+
resolvedBy: string | null;
|
|
65
|
+
}
|
|
66
|
+
export declare function identityOf(input: IDeadLetterInput): IDeadMessageIdentity;
|
|
67
|
+
type RowLike = Pick<DeadMessage, 'topic' | 'partition' | 'offset' | 'timestamp' | 'key' | 'value' | 'headers'>;
|
|
68
|
+
export declare function kafkaInputOf(row: RowLike): IKafkaInput;
|
|
69
|
+
export declare class DeadMessagesService {
|
|
70
|
+
private readonly _model;
|
|
71
|
+
private readonly _logger;
|
|
72
|
+
private readonly _replayers;
|
|
73
|
+
constructor(_model: Model<DeadMessageDocument>, _logger: AppLoggerService);
|
|
74
|
+
recordFailure(input: IDeadLetterInput, error: unknown, kind: ErrorKind, now: Date, budgetMs: number): Promise<IRecordOutcome>;
|
|
75
|
+
settleSuccess(identity: IDeadMessageIdentity, now: Date): Promise<void>;
|
|
76
|
+
registerReplayer(group: string, replayer: DeadMessageReplayer): void;
|
|
77
|
+
unregisterReplayer(group: string): void;
|
|
78
|
+
list(query: {
|
|
79
|
+
status?: DeadMessageStatus[] | string;
|
|
80
|
+
topic?: string;
|
|
81
|
+
limit?: number;
|
|
82
|
+
}): Promise<object[]>;
|
|
83
|
+
read(id: string): Promise<IDeadMessageView>;
|
|
84
|
+
replay(id: string, by: string, now: Date): Promise<DeadMessageReplayAnswer>;
|
|
85
|
+
discard(id: string, by: string, now: Date): Promise<{
|
|
86
|
+
status: 'discarded';
|
|
87
|
+
}>;
|
|
88
|
+
private _unclaimed;
|
|
89
|
+
private _answerAsStored;
|
|
90
|
+
private _whyNotClaimed;
|
|
91
|
+
private _replaying;
|
|
92
|
+
private _notFound;
|
|
93
|
+
private _notDead;
|
|
94
|
+
}
|
|
@@ -0,0 +1,281 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
|
+
};
|
|
8
|
+
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
9
|
+
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
|
+
};
|
|
11
|
+
var __param = (this && this.__param) || function (paramIndex, decorator) {
|
|
12
|
+
return function (target, key) { decorator(target, key, paramIndex); }
|
|
13
|
+
};
|
|
14
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
+
exports.DeadMessagesService = void 0;
|
|
16
|
+
exports.identityOf = identityOf;
|
|
17
|
+
exports.kafkaInputOf = kafkaInputOf;
|
|
18
|
+
const common_1 = require("@nestjs/common");
|
|
19
|
+
const mongoose_1 = require("@nestjs/mongoose");
|
|
20
|
+
const buffer_1 = require("buffer");
|
|
21
|
+
const mongoose_2 = require("mongoose");
|
|
22
|
+
const rest_wrapper_class_1 = require("../models/dtos/wrappers/rest-wrapper.class");
|
|
23
|
+
const app_logger_service_1 = require("../services/app-logger.service");
|
|
24
|
+
const classify_error_1 = require("./classify-error");
|
|
25
|
+
const dead_message_schema_1 = require("./dead-message.schema");
|
|
26
|
+
const dead_messages_const_1 = require("./dead-messages.const");
|
|
27
|
+
function identityOf(input) {
|
|
28
|
+
var _a;
|
|
29
|
+
return { group: input.group, topic: input.topic, partition: input.partition, offset: input.message.offset, timestamp: (_a = input.message.timestamp) !== null && _a !== void 0 ? _a : '' };
|
|
30
|
+
}
|
|
31
|
+
const toBuffer = (v) => {
|
|
32
|
+
if (v === null || v === undefined) {
|
|
33
|
+
return null;
|
|
34
|
+
}
|
|
35
|
+
if (Buffer.isBuffer(v)) {
|
|
36
|
+
return v;
|
|
37
|
+
}
|
|
38
|
+
return Buffer.from(v.buffer.subarray(0, v.position));
|
|
39
|
+
};
|
|
40
|
+
function kafkaInputOf(row) {
|
|
41
|
+
var _a, _b;
|
|
42
|
+
const grouped = new Map();
|
|
43
|
+
for (const h of (_a = row.headers) !== null && _a !== void 0 ? _a : []) {
|
|
44
|
+
const list = (_b = grouped.get(h.key)) !== null && _b !== void 0 ? _b : [];
|
|
45
|
+
list.push(toBuffer(h.value));
|
|
46
|
+
grouped.set(h.key, list);
|
|
47
|
+
}
|
|
48
|
+
const headers = {};
|
|
49
|
+
for (const [key, list] of grouped) {
|
|
50
|
+
headers[key] = list.length === 1 ? list[0] : list;
|
|
51
|
+
}
|
|
52
|
+
return {
|
|
53
|
+
topic: row.topic,
|
|
54
|
+
partition: row.partition,
|
|
55
|
+
message: {
|
|
56
|
+
key: toBuffer(row.key),
|
|
57
|
+
value: toBuffer(row.value),
|
|
58
|
+
offset: row.offset,
|
|
59
|
+
timestamp: row.timestamp,
|
|
60
|
+
attributes: 0,
|
|
61
|
+
headers: headers,
|
|
62
|
+
},
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
const encode = (v) => {
|
|
66
|
+
const bytes = toBuffer(v);
|
|
67
|
+
if (bytes === null) {
|
|
68
|
+
return null;
|
|
69
|
+
}
|
|
70
|
+
return (0, buffer_1.isUtf8)(bytes) ? { encoding: 'utf8', text: bytes.toString('utf8') } : { encoding: 'base64', text: bytes.toString('base64') };
|
|
71
|
+
};
|
|
72
|
+
const headerEntries = (headers) => {
|
|
73
|
+
const entries = [];
|
|
74
|
+
for (const [key, raw] of Object.entries(headers !== null && headers !== void 0 ? headers : {})) {
|
|
75
|
+
for (const v of Array.isArray(raw) ? raw : [raw]) {
|
|
76
|
+
entries.push({ key, value: v === undefined || v === null ? null : Buffer.from(v) });
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
return entries;
|
|
80
|
+
};
|
|
81
|
+
const keywordOf = (e) => (e.keyword ? `${e.name} ${e.keyword}` : e.name);
|
|
82
|
+
const where = (r) => `${r.group} ${r.topic}/${r.partition}@${r.offset}`;
|
|
83
|
+
const isDuplicateKey = (e) => (e === null || e === void 0 ? void 0 : e.code) === 11000;
|
|
84
|
+
const statusesOf = (raw) => {
|
|
85
|
+
if (Array.isArray(raw)) {
|
|
86
|
+
return raw;
|
|
87
|
+
}
|
|
88
|
+
if (typeof raw === 'string') {
|
|
89
|
+
return raw.split(',').filter((s) => s.length > 0);
|
|
90
|
+
}
|
|
91
|
+
return [];
|
|
92
|
+
};
|
|
93
|
+
let DeadMessagesService = class DeadMessagesService {
|
|
94
|
+
constructor(_model, _logger) {
|
|
95
|
+
this._model = _model;
|
|
96
|
+
this._logger = _logger;
|
|
97
|
+
this._replayers = new Map();
|
|
98
|
+
}
|
|
99
|
+
async recordFailure(input, error, kind, now, budgetMs) {
|
|
100
|
+
var _a, _b, _c, _d, _e;
|
|
101
|
+
const identity = identityOf(input);
|
|
102
|
+
const update = {
|
|
103
|
+
$setOnInsert: {
|
|
104
|
+
key: (_a = input.message.key) !== null && _a !== void 0 ? _a : null,
|
|
105
|
+
value: (_b = input.message.value) !== null && _b !== void 0 ? _b : null,
|
|
106
|
+
headers: headerEntries(input.message.headers),
|
|
107
|
+
firstFailedAt: now,
|
|
108
|
+
status: 'retrying',
|
|
109
|
+
expireAt: new Date(now.getTime() + dead_messages_const_1.DEAD_MESSAGE_RETRYING_TTL_MS),
|
|
110
|
+
},
|
|
111
|
+
$inc: { attempts: 1 },
|
|
112
|
+
$set: { error: (0, classify_error_1.describeError)(error), lastFailedAt: now },
|
|
113
|
+
};
|
|
114
|
+
const options = { upsert: true, returnDocument: 'after' };
|
|
115
|
+
let stored;
|
|
116
|
+
try {
|
|
117
|
+
stored = await this._model.findOneAndUpdate(identity, update, options);
|
|
118
|
+
}
|
|
119
|
+
catch (e) {
|
|
120
|
+
if (!isDuplicateKey(e)) {
|
|
121
|
+
throw e;
|
|
122
|
+
}
|
|
123
|
+
stored = await this._model.findOneAndUpdate(identity, update, options);
|
|
124
|
+
}
|
|
125
|
+
const attempts = stored.attempts;
|
|
126
|
+
if (stored.status !== 'retrying') {
|
|
127
|
+
return { outcome: 'already', status: stored.status, reason: (_c = stored.reason) !== null && _c !== void 0 ? _c : null, attempts };
|
|
128
|
+
}
|
|
129
|
+
if (kind === 'permanent' || now.getTime() - new Date(stored.firstFailedAt).getTime() >= budgetMs) {
|
|
130
|
+
const reason = kind === 'permanent' ? 'unprocessable' : 'retries_exhausted';
|
|
131
|
+
await this._model.updateOne({ _id: stored._id, status: 'retrying' }, { $set: { status: 'dead', reason, expireAt: null } });
|
|
132
|
+
return { outcome: 'dead', status: 'dead', reason, attempts };
|
|
133
|
+
}
|
|
134
|
+
const res = await this._model.updateOne({ _id: stored._id, status: 'retrying' }, { $set: { expireAt: new Date(new Date(stored.lastFailedAt).getTime() + dead_messages_const_1.DEAD_MESSAGE_RETRYING_TTL_MS) } });
|
|
135
|
+
if (res.matchedCount > 0) {
|
|
136
|
+
return { outcome: 'retrying', status: 'retrying', reason: null, attempts };
|
|
137
|
+
}
|
|
138
|
+
const current = await this._model.findById(stored._id).lean();
|
|
139
|
+
return { outcome: 'already', status: (_d = current === null || current === void 0 ? void 0 : current.status) !== null && _d !== void 0 ? _d : 'retrying', reason: (_e = current === null || current === void 0 ? void 0 : current.reason) !== null && _e !== void 0 ? _e : null, attempts };
|
|
140
|
+
}
|
|
141
|
+
async settleSuccess(identity, now) {
|
|
142
|
+
const res = await this._model.deleteOne(Object.assign(Object.assign({}, identity), { status: 'retrying' }));
|
|
143
|
+
if (res.deletedCount > 0) {
|
|
144
|
+
return;
|
|
145
|
+
}
|
|
146
|
+
await this._model.updateOne(Object.assign(Object.assign({}, identity), { status: 'dead' }), { $set: { status: 'replayed', resolvedAt: now, resolvedBy: 'consumer', expireAt: new Date(now.getTime() + dead_messages_const_1.DEAD_MESSAGE_RESOLVED_TTL_MS) } });
|
|
147
|
+
}
|
|
148
|
+
registerReplayer(group, replayer) {
|
|
149
|
+
this._replayers.set(group, replayer);
|
|
150
|
+
}
|
|
151
|
+
unregisterReplayer(group) {
|
|
152
|
+
this._replayers.delete(group);
|
|
153
|
+
}
|
|
154
|
+
async list(query) {
|
|
155
|
+
const given = statusesOf(query.status);
|
|
156
|
+
const status = given.length > 0 ? given : ['dead', 'retrying'];
|
|
157
|
+
const filter = { status: { $in: status } };
|
|
158
|
+
if (query.topic) {
|
|
159
|
+
filter.topic = query.topic;
|
|
160
|
+
}
|
|
161
|
+
const requested = Number.isFinite(query.limit) ? Math.trunc(query.limit) : dead_messages_const_1.DEAD_MESSAGES_LIST_DEFAULT;
|
|
162
|
+
const limit = Math.min(Math.max(requested, 1), dead_messages_const_1.DEAD_MESSAGES_LIST_MAX);
|
|
163
|
+
return this._model.find(filter, { key: 0, value: 0, headers: 0, 'error.message': 0 }).sort({ lastFailedAt: -1 }).limit(limit).lean();
|
|
164
|
+
}
|
|
165
|
+
async read(id) {
|
|
166
|
+
var _a, _b, _c, _d, _e;
|
|
167
|
+
const row = await this._model.findById(id);
|
|
168
|
+
if (!row) {
|
|
169
|
+
throw this._notFound();
|
|
170
|
+
}
|
|
171
|
+
return {
|
|
172
|
+
_id: String(row._id),
|
|
173
|
+
group: row.group,
|
|
174
|
+
topic: row.topic,
|
|
175
|
+
partition: row.partition,
|
|
176
|
+
offset: row.offset,
|
|
177
|
+
timestamp: row.timestamp,
|
|
178
|
+
key: encode(row.key),
|
|
179
|
+
value: encode(row.value),
|
|
180
|
+
headers: ((_a = row.headers) !== null && _a !== void 0 ? _a : []).map((h) => ({ key: h.key, value: encode(h.value) })),
|
|
181
|
+
status: row.status,
|
|
182
|
+
reason: (_b = row.reason) !== null && _b !== void 0 ? _b : null,
|
|
183
|
+
error: { name: row.error.name, keyword: (_c = row.error.keyword) !== null && _c !== void 0 ? _c : null, message: row.error.message },
|
|
184
|
+
attempts: row.attempts,
|
|
185
|
+
firstFailedAt: row.firstFailedAt,
|
|
186
|
+
lastFailedAt: row.lastFailedAt,
|
|
187
|
+
resolvedAt: (_d = row.resolvedAt) !== null && _d !== void 0 ? _d : null,
|
|
188
|
+
resolvedBy: (_e = row.resolvedBy) !== null && _e !== void 0 ? _e : null,
|
|
189
|
+
};
|
|
190
|
+
}
|
|
191
|
+
async replay(id, by, now) {
|
|
192
|
+
const row = await this._model.findOneAndUpdate({ _id: id, status: 'dead', $or: this._unclaimed(now) }, { $set: { replayStartedAt: now } });
|
|
193
|
+
if (!row) {
|
|
194
|
+
throw await this._whyNotClaimed(id);
|
|
195
|
+
}
|
|
196
|
+
const ownClaim = { _id: id, status: 'dead', replayStartedAt: now };
|
|
197
|
+
const replayer = this._replayers.get(row.group);
|
|
198
|
+
if (!replayer) {
|
|
199
|
+
await this._model.updateOne(ownClaim, { $set: { replayStartedAt: null } });
|
|
200
|
+
throw new rest_wrapper_class_1.RESTError('consumer_not_running', undefined, common_1.HttpStatus.SERVICE_UNAVAILABLE);
|
|
201
|
+
}
|
|
202
|
+
try {
|
|
203
|
+
await replayer(kafkaInputOf(row));
|
|
204
|
+
}
|
|
205
|
+
catch (e) {
|
|
206
|
+
const described = (0, classify_error_1.describeError)(e);
|
|
207
|
+
const error = { name: described.name, keyword: described.keyword };
|
|
208
|
+
const res = await this._model.updateOne(ownClaim, {
|
|
209
|
+
$set: { error: described, lastFailedAt: now, replayStartedAt: null },
|
|
210
|
+
$inc: { attempts: 1 },
|
|
211
|
+
});
|
|
212
|
+
if (res.matchedCount === 0) {
|
|
213
|
+
return this._answerAsStored(id, error);
|
|
214
|
+
}
|
|
215
|
+
await this._logger.info(`[dead-letter] replay failed ${where(row)} by ${by} ${keywordOf(described)}`);
|
|
216
|
+
return { status: 'dead', error };
|
|
217
|
+
}
|
|
218
|
+
const res = await this._model.updateOne({ _id: id, status: 'dead' }, {
|
|
219
|
+
$set: {
|
|
220
|
+
status: 'replayed',
|
|
221
|
+
resolvedAt: now,
|
|
222
|
+
resolvedBy: by,
|
|
223
|
+
expireAt: new Date(now.getTime() + dead_messages_const_1.DEAD_MESSAGE_RESOLVED_TTL_MS),
|
|
224
|
+
replayStartedAt: null,
|
|
225
|
+
},
|
|
226
|
+
});
|
|
227
|
+
if (res.matchedCount === 0) {
|
|
228
|
+
return this._answerAsStored(id, null);
|
|
229
|
+
}
|
|
230
|
+
await this._logger.info(`[dead-letter] replayed ${where(row)} by ${by}`);
|
|
231
|
+
return { status: 'replayed' };
|
|
232
|
+
}
|
|
233
|
+
async discard(id, by, now) {
|
|
234
|
+
const row = await this._model.findOneAndUpdate({ _id: id, status: 'dead', $or: this._unclaimed(now) }, { $set: { status: 'discarded', resolvedAt: now, resolvedBy: by, expireAt: new Date(now.getTime() + dead_messages_const_1.DEAD_MESSAGE_RESOLVED_TTL_MS) } });
|
|
235
|
+
if (!row) {
|
|
236
|
+
throw await this._whyNotClaimed(id);
|
|
237
|
+
}
|
|
238
|
+
await this._logger.info(`[dead-letter] discarded ${where(row)} by ${by}`);
|
|
239
|
+
return { status: 'discarded' };
|
|
240
|
+
}
|
|
241
|
+
_unclaimed(now) {
|
|
242
|
+
return [{ replayStartedAt: null }, { replayStartedAt: { $lt: new Date(now.getTime() - dead_messages_const_1.DEAD_MESSAGE_REPLAY_CLAIM_MS) } }];
|
|
243
|
+
}
|
|
244
|
+
async _answerAsStored(id, failure) {
|
|
245
|
+
const current = await this._model.findById(id).lean();
|
|
246
|
+
if (!current) {
|
|
247
|
+
throw this._notFound();
|
|
248
|
+
}
|
|
249
|
+
if (current.status === 'replayed' || current.status === 'discarded') {
|
|
250
|
+
return { status: current.status };
|
|
251
|
+
}
|
|
252
|
+
if (current.status === 'dead' && !current.replayStartedAt && failure) {
|
|
253
|
+
return { status: 'dead', error: failure };
|
|
254
|
+
}
|
|
255
|
+
throw current.status === 'dead' ? this._replaying() : this._notDead();
|
|
256
|
+
}
|
|
257
|
+
async _whyNotClaimed(id) {
|
|
258
|
+
const current = await this._model.findById(id).lean();
|
|
259
|
+
if (!current) {
|
|
260
|
+
return this._notFound();
|
|
261
|
+
}
|
|
262
|
+
return current.status === 'dead' ? this._replaying() : this._notDead();
|
|
263
|
+
}
|
|
264
|
+
_replaying() {
|
|
265
|
+
return new rest_wrapper_class_1.RESTError('dead_message_replaying', undefined, common_1.HttpStatus.CONFLICT);
|
|
266
|
+
}
|
|
267
|
+
_notFound() {
|
|
268
|
+
return new rest_wrapper_class_1.RESTError('dead_message_not_found', undefined, common_1.HttpStatus.NOT_FOUND);
|
|
269
|
+
}
|
|
270
|
+
_notDead() {
|
|
271
|
+
return new rest_wrapper_class_1.RESTError('dead_message_not_dead', undefined, common_1.HttpStatus.CONFLICT);
|
|
272
|
+
}
|
|
273
|
+
};
|
|
274
|
+
exports.DeadMessagesService = DeadMessagesService;
|
|
275
|
+
exports.DeadMessagesService = DeadMessagesService = __decorate([
|
|
276
|
+
(0, common_1.Injectable)(),
|
|
277
|
+
__param(0, (0, mongoose_1.InjectModel)(dead_message_schema_1.DeadMessage.name)),
|
|
278
|
+
__metadata("design:paramtypes", [mongoose_2.Model,
|
|
279
|
+
app_logger_service_1.AppLoggerService])
|
|
280
|
+
], DeadMessagesService);
|
|
281
|
+
//# sourceMappingURL=dead-messages.service.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dead-messages.service.js","sourceRoot":"","sources":["../../src/dead-messages/dead-messages.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAgEA,gCAEC;AAgBD,oCAuBC;AAzGD,2CAAwD;AACxD,+CAA+C;AAC/C,mCAAgC;AAEhC,uCAAiC;AACjC,mFAAuE;AACvE,uEAAkE;AAClE,qDAA6E;AAC7E,+DAA+G;AAC/G,+DAM+B;AAiD/B,SAAgB,UAAU,CAAC,KAAuB;;IAC9C,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,SAAS,EAAE,MAAA,KAAK,CAAC,OAAO,CAAC,SAAS,mCAAI,EAAE,EAAE,CAAC;AAC1J,CAAC;AAGD,MAAM,QAAQ,GAAG,CAAC,CAAM,EAAiB,EAAE;IACvC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,SAAS,EAAE,CAAC;QAChC,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;QACrB,OAAO,CAAC,CAAC;IACb,CAAC;IACD,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;AACzD,CAAC,CAAC;AAKF,SAAgB,YAAY,CAAC,GAAY;;IACrC,MAAM,OAAO,GAAG,IAAI,GAAG,EAA6B,CAAC;IACrD,KAAK,MAAM,CAAC,IAAI,MAAA,GAAG,CAAC,OAAO,mCAAI,EAAE,EAAE,CAAC;QAChC,MAAM,IAAI,GAAG,MAAA,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,mCAAI,EAAE,CAAC;QACtC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;QAC7B,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IAC7B,CAAC;IACD,MAAM,OAAO,GAAwB,EAAE,CAAC;IACxC,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,OAAO,EAAE,CAAC;QAChC,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACtD,CAAC;IACD,OAAO;QACH,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,SAAS,EAAE,GAAG,CAAC,SAAS;QACxB,OAAO,EAAE;YACL,GAAG,EAAE,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC;YACtB,KAAK,EAAE,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC;YAC1B,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,UAAU,EAAE,CAAC;YACb,OAAO,EAAE,OAAmB;SACf;KACpB,CAAC;AACN,CAAC;AAED,MAAM,MAAM,GAAG,CAAC,CAAM,EAAwB,EAAE;IAC5C,MAAM,KAAK,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;IAC1B,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;QACjB,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,OAAO,IAAA,eAAM,EAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;AACvI,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,CAAC,OAA6B,EAA2C,EAAE;IAC7F,MAAM,OAAO,GAA4C,EAAE,CAAC;IAC5D,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,aAAP,OAAO,cAAP,OAAO,GAAI,EAAE,CAAC,EAAE,CAAC;QACrD,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC;YAC/C,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACxF,CAAC;IACL,CAAC;IACD,OAAO,OAAO,CAAC;AACnB,CAAC,CAAC;AAEF,MAAM,SAAS,GAAG,CAAC,CAA2C,EAAU,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;AAC3H,MAAM,KAAK,GAAG,CAAC,CAAsE,EAAU,EAAE,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;AACrJ,MAAM,cAAc,GAAG,CAAC,CAAM,EAAW,EAAE,CAAC,CAAA,CAAC,aAAD,CAAC,uBAAD,CAAC,CAAE,IAAI,MAAK,KAAK,CAAC;AAG9D,MAAM,UAAU,GAAG,CAAC,GAAY,EAAY,EAAE;IAC1C,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QACrB,OAAO,GAAG,CAAC;IACf,CAAC;IACD,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QAC1B,OAAO,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACtD,CAAC;IACD,OAAO,EAAE,CAAC;AACd,CAAC,CAAC;AAGK,IAAM,mBAAmB,GAAzB,MAAM,mBAAmB;IAM5B,YACmC,MAAmD,EACjE,OAAyB;QADM,WAAM,GAAN,MAAM,CAA4B;QACjE,YAAO,GAAP,OAAO,CAAkB;QAN7B,eAAU,GAAG,IAAI,GAAG,EAA+B,CAAC;IAOlE,CAAC;IAIG,KAAK,CAAC,aAAa,CAAC,KAAuB,EAAE,KAAc,EAAE,IAAe,EAAE,GAAS,EAAE,QAAgB;;QAC5G,MAAM,QAAQ,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QACnC,MAAM,MAAM,GAAG;YACX,YAAY,EAAE;gBACV,GAAG,EAAE,MAAA,KAAK,CAAC,OAAO,CAAC,GAAG,mCAAI,IAAI;gBAC9B,KAAK,EAAE,MAAA,KAAK,CAAC,OAAO,CAAC,KAAK,mCAAI,IAAI;gBAClC,OAAO,EAAE,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;gBAC7C,aAAa,EAAE,GAAG;gBAClB,MAAM,EAAE,UAAU;gBAClB,QAAQ,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,kDAA4B,CAAC;aACnE;YACD,IAAI,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE;YACrB,IAAI,EAAE,EAAE,KAAK,EAAE,IAAA,8BAAa,EAAC,KAAK,CAAC,EAAE,YAAY,EAAE,GAAG,EAAE;SAC3D,CAAC;QACF,MAAM,OAAO,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,cAAc,EAAE,OAAO,EAAW,CAAC;QAEnE,IAAI,MAAW,CAAC;QAChB,IAAI,CAAC;YACD,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,QAAQ,EAAE,MAAa,EAAE,OAAO,CAAC,CAAC;QAClF,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC;gBACrB,MAAM,CAAC,CAAC;YACZ,CAAC;YAED,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,QAAQ,EAAE,MAAa,EAAE,OAAO,CAAC,CAAC;QAClF,CAAC;QAED,MAAM,QAAQ,GAAW,MAAM,CAAC,QAAQ,CAAC;QACzC,IAAI,MAAM,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;YAC/B,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,MAAA,MAAM,CAAC,MAAM,mCAAI,IAAI,EAAE,QAAQ,EAAE,CAAC;QAClG,CAAC;QAED,IAAI,IAAI,KAAK,WAAW,IAAI,GAAG,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,OAAO,EAAE,IAAI,QAAQ,EAAE,CAAC;YAC/F,MAAM,MAAM,GAAsB,IAAI,KAAK,WAAW,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,mBAAmB,CAAC;YAC/F,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;YAC3H,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;QACjE,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CACnC,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,EACvC,EAAE,IAAI,EAAE,EAAE,QAAQ,EAAE,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE,GAAG,kDAA4B,CAAC,EAAE,EAAE,CAC3G,CAAC;QACF,IAAI,GAAG,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;YACvB,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QAC/E,CAAC;QACD,MAAM,OAAO,GAAQ,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QACnE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,MAAM,mCAAI,UAAU,EAAE,MAAM,EAAE,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,MAAM,mCAAI,IAAI,EAAE,QAAQ,EAAE,CAAC;IACpH,CAAC;IAEM,KAAK,CAAC,aAAa,CAAC,QAA8B,EAAE,GAAS;QAChE,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,iCAAM,QAAQ,KAAE,MAAM,EAAE,UAAU,IAAG,CAAC;QAC7E,IAAI,GAAG,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;YACvB,OAAO;QACX,CAAC;QACD,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,iCAClB,QAAQ,KAAE,MAAM,EAAE,MAAM,KAC7B,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,GAAG,EAAE,UAAU,EAAE,UAAU,EAAE,QAAQ,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,kDAA4B,CAAC,EAAE,EAAE,CAC9I,CAAC;IACN,CAAC;IAEM,gBAAgB,CAAC,KAAa,EAAE,QAA6B;QAChE,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;IACzC,CAAC;IAEM,kBAAkB,CAAC,KAAa;QACnC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAClC,CAAC;IAIM,KAAK,CAAC,IAAI,CAAC,KAAgF;QAC9F,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACvC,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;QAC/D,MAAM,MAAM,GAAwB,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC;QAChE,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;YACd,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;QAC/B,CAAC;QACD,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAe,CAAC,CAAC,CAAC,CAAC,gDAA0B,CAAC;QAChH,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC,EAAE,4CAAsB,CAAC,CAAC;QACvE,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;IACzI,CAAC;IAEM,KAAK,CAAC,IAAI,CAAC,EAAU;;QACxB,MAAM,GAAG,GAAQ,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QAChD,IAAI,CAAC,GAAG,EAAE,CAAC;YACP,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QAC3B,CAAC;QACD,OAAO;YACH,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC;YACpB,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC;YACpB,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC;YACxB,OAAO,EAAE,CAAC,MAAA,GAAG,CAAC,OAAO,mCAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YACtF,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,MAAM,EAAE,MAAA,GAAG,CAAC,MAAM,mCAAI,IAAI;YAC1B,KAAK,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,OAAO,EAAE,MAAA,GAAG,CAAC,KAAK,CAAC,OAAO,mCAAI,IAAI,EAAE,OAAO,EAAE,GAAG,CAAC,KAAK,CAAC,OAAO,EAAE;YAC/F,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,aAAa,EAAE,GAAG,CAAC,aAAa;YAChC,YAAY,EAAE,GAAG,CAAC,YAAY;YAC9B,UAAU,EAAE,MAAA,GAAG,CAAC,UAAU,mCAAI,IAAI;YAClC,UAAU,EAAE,MAAA,GAAG,CAAC,UAAU,mCAAI,IAAI;SACrC,CAAC;IACN,CAAC;IAEM,KAAK,CAAC,MAAM,CAAC,EAAU,EAAE,EAAU,EAAE,GAAS;QAEjD,MAAM,GAAG,GAAQ,MAAM,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,eAAe,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;QAChJ,IAAI,CAAC,GAAG,EAAE,CAAC;YACP,MAAM,MAAM,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;QACxC,CAAC;QAED,MAAM,QAAQ,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,EAAE,MAAe,EAAE,eAAe,EAAE,GAAG,EAAE,CAAC;QAE5E,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAChD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACZ,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,EAAE,eAAe,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;YAC3E,MAAM,IAAI,8BAAS,CAAC,sBAAsB,EAAE,SAAS,EAAE,mBAAU,CAAC,mBAAmB,CAAC,CAAC;QAC3F,CAAC;QAED,IAAI,CAAC;YACD,MAAM,QAAQ,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;QACtC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,MAAM,SAAS,GAAG,IAAA,8BAAa,EAAC,CAAC,CAAC,CAAC;YACnC,MAAM,KAAK,GAAG,EAAE,IAAI,EAAE,SAAS,CAAC,IAAI,EAAE,OAAO,EAAE,SAAS,CAAC,OAAO,EAAE,CAAC;YACnE,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,EAAE;gBAC9C,IAAI,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,YAAY,EAAE,GAAG,EAAE,eAAe,EAAE,IAAI,EAAE;gBACpE,IAAI,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE;aACxB,CAAC,CAAC;YACH,IAAI,GAAG,CAAC,YAAY,KAAK,CAAC,EAAE,CAAC;gBACzB,OAAO,IAAI,CAAC,eAAe,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;YAC3C,CAAC;YACD,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,+BAA+B,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,SAAS,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;YACtG,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;QACrC,CAAC;QAGD,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CACnC,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,EAC3B;YACI,IAAI,EAAE;gBACF,MAAM,EAAE,UAAU;gBAClB,UAAU,EAAE,GAAG;gBACf,UAAU,EAAE,EAAE;gBACd,QAAQ,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,kDAA4B,CAAC;gBAChE,eAAe,EAAE,IAAI;aACxB;SACJ,CACJ,CAAC;QACF,IAAI,GAAG,CAAC,YAAY,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,IAAI,CAAC,eAAe,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QAC1C,CAAC;QACD,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,0BAA0B,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QACzE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;IAClC,CAAC;IAEM,KAAK,CAAC,OAAO,CAAC,EAAU,EAAE,EAAU,EAAE,GAAS;QAElD,MAAM,GAAG,GAAQ,MAAM,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAC/C,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,EACtD,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,GAAG,EAAE,UAAU,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,kDAA4B,CAAC,EAAE,EAAE,CACvI,CAAC;QACF,IAAI,CAAC,GAAG,EAAE,CAAC;YACP,MAAM,MAAM,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;QACxC,CAAC;QACD,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,2BAA2B,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QAC1E,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;IACnC,CAAC;IAKO,UAAU,CAAC,GAAS;QACxB,OAAO,CAAC,EAAE,eAAe,EAAE,IAAI,EAAE,EAAE,EAAE,eAAe,EAAE,EAAE,GAAG,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,kDAA4B,CAAC,EAAE,EAAE,CAAC,CAAC;IAC7H,CAAC;IAMO,KAAK,CAAC,eAAe,CAAC,EAAU,EAAE,OAAwD;QAC9F,MAAM,OAAO,GAAQ,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAC3D,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QAC3B,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,KAAK,UAAU,IAAI,OAAO,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;YAClE,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;QACtC,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,IAAI,OAAO,EAAE,CAAC;YAEnE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QAC9C,CAAC;QAED,MAAM,OAAO,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;IAC1E,CAAC;IAEO,KAAK,CAAC,cAAc,CAAC,EAAU;QACnC,MAAM,OAAO,GAAQ,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAC3D,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,OAAO,IAAI,CAAC,SAAS,EAAE,CAAC;QAC5B,CAAC;QACD,OAAO,OAAO,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;IAC3E,CAAC;IAEO,UAAU;QACd,OAAO,IAAI,8BAAS,CAAC,wBAAwB,EAAE,SAAS,EAAE,mBAAU,CAAC,QAAQ,CAAC,CAAC;IACnF,CAAC;IAEO,SAAS;QACb,OAAO,IAAI,8BAAS,CAAC,wBAAwB,EAAE,SAAS,EAAE,mBAAU,CAAC,SAAS,CAAC,CAAC;IACpF,CAAC;IAEO,QAAQ;QACZ,OAAO,IAAI,8BAAS,CAAC,uBAAuB,EAAE,SAAS,EAAE,mBAAU,CAAC,QAAQ,CAAC,CAAC;IAClF,CAAC;CAEJ,CAAA;AAxOY,kDAAmB;8BAAnB,mBAAmB;IAD/B,IAAA,mBAAU,GAAE;IAQJ,WAAA,IAAA,sBAAW,EAAC,iCAAW,CAAC,IAAI,CAAC,CAAA;qCAA0B,gBAAK;QACnC,qCAAgB;GARrC,mBAAmB,CAwO/B"}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import type { ConsumerSubscribeTopics, Kafka } from 'kafkajs';
|
|
2
|
+
import { AppLoggerService } from '../services/app-logger.service';
|
|
3
|
+
import { ErrorKind } from './classify-error';
|
|
4
|
+
import { DeadMessagesService, IKafkaInput } from './dead-messages.service';
|
|
5
|
+
export interface IKafkaConsumerRunnerOptions {
|
|
6
|
+
kafka: Kafka;
|
|
7
|
+
groupId: string;
|
|
8
|
+
subscriptions: ConsumerSubscribeTopics[];
|
|
9
|
+
dispatch: (input: IKafkaInput) => Promise<void>;
|
|
10
|
+
deadMessages: DeadMessagesService;
|
|
11
|
+
logger: AppLoggerService;
|
|
12
|
+
onFatal: () => void;
|
|
13
|
+
retryBudgetMs?: number;
|
|
14
|
+
classify?: (error: unknown) => ErrorKind | undefined;
|
|
15
|
+
now?: () => Date;
|
|
16
|
+
}
|
|
17
|
+
export declare class KafkaConsumerRunner {
|
|
18
|
+
private readonly _options;
|
|
19
|
+
private readonly _group;
|
|
20
|
+
private readonly _budgetMs;
|
|
21
|
+
private readonly _now;
|
|
22
|
+
private _consumer;
|
|
23
|
+
private _starting;
|
|
24
|
+
private _startSettled;
|
|
25
|
+
private _stopping;
|
|
26
|
+
private _fatal;
|
|
27
|
+
private readonly _failed;
|
|
28
|
+
private _joins;
|
|
29
|
+
private readonly _settledJoin;
|
|
30
|
+
constructor(_options: IKafkaConsumerRunnerOptions);
|
|
31
|
+
start(): Promise<void>;
|
|
32
|
+
stop(): Promise<void>;
|
|
33
|
+
private _start;
|
|
34
|
+
private _listen;
|
|
35
|
+
private _onCrash;
|
|
36
|
+
private _onStop;
|
|
37
|
+
private _onGroupJoin;
|
|
38
|
+
private _eachMessage;
|
|
39
|
+
private _handle;
|
|
40
|
+
private _onFailure;
|
|
41
|
+
private _kindOf;
|
|
42
|
+
private _retryError;
|
|
43
|
+
private _log;
|
|
44
|
+
private _unregister;
|
|
45
|
+
private _fatalUnlessStopping;
|
|
46
|
+
private _disconnect;
|
|
47
|
+
}
|