@solidxai/core 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/dtos/create-saved-filters.dto.d.ts.map +1 -1
- package/dist/dtos/create-saved-filters.dto.js.map +1 -1
- package/dist/entities/chatter-message.entity.d.ts.map +1 -1
- package/dist/entities/chatter-message.entity.js +2 -0
- package/dist/entities/chatter-message.entity.js.map +1 -1
- package/dist/helpers/schematic.service.d.ts.map +1 -1
- package/dist/helpers/schematic.service.js +6 -2
- package/dist/helpers/schematic.service.js.map +1 -1
- package/dist/jobs/api-email-queue-options.d.ts.map +1 -1
- package/dist/jobs/api-email-queue-options.js +2 -2
- package/dist/jobs/api-email-queue-options.js.map +1 -1
- package/dist/jobs/chatter-queue-options.js +2 -2
- package/dist/jobs/chatter-queue-options.js.map +1 -1
- package/dist/jobs/computed-field-evaluation-queue-options.js +2 -2
- package/dist/jobs/computed-field-evaluation-queue-options.js.map +1 -1
- package/dist/jobs/database/api-email-queue-options-database.js +2 -2
- package/dist/jobs/database/api-email-queue-options-database.js.map +1 -1
- package/dist/jobs/database/computed-field-evaluation-queue-options-database.js +2 -2
- package/dist/jobs/database/computed-field-evaluation-queue-options-database.js.map +1 -1
- package/dist/jobs/database/generate-code-queue-options-database.js +2 -2
- package/dist/jobs/database/generate-code-queue-options-database.js.map +1 -1
- package/dist/jobs/database/msg91-sms-queue-database-options.d.ts.map +1 -1
- package/dist/jobs/database/msg91-sms-queue-database-options.js +2 -2
- package/dist/jobs/database/msg91-sms-queue-database-options.js.map +1 -1
- package/dist/jobs/database/msg91-whatsapp-queue-options-database.js +2 -2
- package/dist/jobs/database/msg91-whatsapp-queue-options-database.js.map +1 -1
- package/dist/jobs/database/otp-queue-options-database.d.ts.map +1 -1
- package/dist/jobs/database/otp-queue-options-database.js +2 -2
- package/dist/jobs/database/otp-queue-options-database.js.map +1 -1
- package/dist/jobs/database/smtp-email-queue-options-database.js +1 -1
- package/dist/jobs/database/smtp-email-queue-options-database.js.map +1 -1
- package/dist/jobs/database/test-queue-options-database.js +2 -2
- package/dist/jobs/database/test-queue-options-database.js.map +1 -1
- package/dist/jobs/database/three60-whatsapp-queue-options-database.js +2 -2
- package/dist/jobs/database/three60-whatsapp-queue-options-database.js.map +1 -1
- package/dist/jobs/database/trigger-mcp-client-queue-options.js +2 -2
- package/dist/jobs/database/trigger-mcp-client-queue-options.js.map +1 -1
- package/dist/jobs/database/twilio-sms-queue-database-options.js +2 -2
- package/dist/jobs/database/twilio-sms-queue-database-options.js.map +1 -1
- package/dist/jobs/generate-code-queue-options.js +2 -2
- package/dist/jobs/generate-code-queue-options.js.map +1 -1
- package/dist/jobs/msg91-otp-queue-options.d.ts.map +1 -1
- package/dist/jobs/msg91-otp-queue-options.js +2 -2
- package/dist/jobs/msg91-otp-queue-options.js.map +1 -1
- package/dist/jobs/msg91-sms-queue-options.d.ts.map +1 -1
- package/dist/jobs/msg91-sms-queue-options.js +2 -2
- package/dist/jobs/msg91-sms-queue-options.js.map +1 -1
- package/dist/jobs/msg91-whatsapp-queue-options.d.ts.map +1 -1
- package/dist/jobs/msg91-whatsapp-queue-options.js +2 -2
- package/dist/jobs/msg91-whatsapp-queue-options.js.map +1 -1
- package/dist/jobs/smtp-email-queue-options.d.ts.map +1 -1
- package/dist/jobs/smtp-email-queue-options.js +1 -1
- package/dist/jobs/smtp-email-queue-options.js.map +1 -1
- package/dist/jobs/test-queue-options.d.ts.map +1 -1
- package/dist/jobs/test-queue-options.js +2 -2
- package/dist/jobs/test-queue-options.js.map +1 -1
- package/dist/jobs/three60-whatsapp-queue-options.d.ts.map +1 -1
- package/dist/jobs/three60-whatsapp-queue-options.js +2 -2
- package/dist/jobs/three60-whatsapp-queue-options.js.map +1 -1
- package/dist/jobs/three60-whatsapp-subscriber.service.js +2 -2
- package/dist/jobs/three60-whatsapp-subscriber.service.js.map +1 -1
- package/dist/jobs/trigger-mcp-client-queue-options.js +2 -2
- package/dist/jobs/trigger-mcp-client-queue-options.js.map +1 -1
- package/dist/jobs/twilio-sms-queue-options.js +2 -2
- package/dist/jobs/twilio-sms-queue-options.js.map +1 -1
- package/dist/seeders/module-metadata-seeder.service.d.ts.map +1 -1
- package/dist/seeders/module-metadata-seeder.service.js +2 -1
- package/dist/seeders/module-metadata-seeder.service.js.map +1 -1
- package/dist/seeders/seed-data/solid-core-metadata.json +34 -9
- package/dist/services/chatter-message.service.d.ts.map +1 -1
- package/dist/services/chatter-message.service.js +8 -1
- package/dist/services/chatter-message.service.js.map +1 -1
- package/dist/services/crud-helper.service.js.map +1 -1
- package/dist/services/model-metadata.service.d.ts.map +1 -1
- package/dist/services/model-metadata.service.js +2 -1
- package/dist/services/model-metadata.service.js.map +1 -1
- package/dist/services/module-metadata.service.d.ts.map +1 -1
- package/dist/services/module-metadata.service.js +2 -1
- package/dist/services/module-metadata.service.js.map +1 -1
- package/dist/services/queues/database-publisher.service.js +0 -1
- package/dist/services/queues/database-publisher.service.js.map +1 -1
- package/dist/services/queues/database-subscriber.service.d.ts.map +1 -1
- package/dist/services/queues/database-subscriber.service.js +14 -1
- package/dist/services/queues/database-subscriber.service.js.map +1 -1
- package/dist/services/queues/rabbitmq-subscriber.service.d.ts +14 -1
- package/dist/services/queues/rabbitmq-subscriber.service.d.ts.map +1 -1
- package/dist/services/queues/rabbitmq-subscriber.service.js +209 -66
- package/dist/services/queues/rabbitmq-subscriber.service.js.map +1 -1
- package/dist/services/scheduled-jobs/scheduler.service.d.ts +1 -0
- package/dist/services/scheduled-jobs/scheduler.service.d.ts.map +1 -1
- package/dist/services/scheduled-jobs/scheduler.service.js +26 -10
- package/dist/services/scheduled-jobs/scheduler.service.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +1 -1
- package/src/dtos/create-saved-filters.dto.ts +10 -1
- package/src/entities/chatter-message.entity.ts +11 -1
- package/src/helpers/schematic.service.ts +6 -2
- package/src/jobs/api-email-queue-options.ts +2 -6
- package/src/jobs/chatter-queue-options.ts +2 -2
- package/src/jobs/computed-field-evaluation-queue-options.ts +2 -2
- package/src/jobs/database/api-email-queue-options-database.ts +2 -2
- package/src/jobs/database/computed-field-evaluation-queue-options-database.ts +2 -2
- package/src/jobs/database/generate-code-queue-options-database.ts +2 -2
- package/src/jobs/database/msg91-sms-queue-database-options.ts +3 -2
- package/src/jobs/database/msg91-whatsapp-queue-options-database.ts +2 -2
- package/src/jobs/database/otp-queue-options-database.ts +3 -2
- package/src/jobs/database/smtp-email-queue-options-database.ts +1 -1
- package/src/jobs/database/test-queue-options-database.ts +2 -2
- package/src/jobs/database/three60-whatsapp-queue-options-database.ts +2 -2
- package/src/jobs/database/trigger-mcp-client-queue-options.ts +2 -2
- package/src/jobs/database/twilio-sms-queue-database-options.ts +2 -2
- package/src/jobs/generate-code-queue-options.ts +2 -2
- package/src/jobs/msg91-otp-queue-options.ts +3 -8
- package/src/jobs/msg91-sms-queue-options.ts +3 -6
- package/src/jobs/msg91-whatsapp-queue-options.ts +4 -7
- package/src/jobs/smtp-email-queue-options.ts +1 -6
- package/src/jobs/test-queue-options.ts +2 -6
- package/src/jobs/three60-whatsapp-queue-options.ts +4 -7
- package/src/jobs/three60-whatsapp-subscriber.service.ts +1 -1
- package/src/jobs/trigger-mcp-client-queue-options.ts +2 -2
- package/src/jobs/twilio-sms-queue-options.ts +3 -3
- package/src/seeders/module-metadata-seeder.service.ts +2 -1
- package/src/seeders/seed-data/solid-core-metadata.json +34 -9
- package/src/services/chatter-message.service.ts +32 -26
- package/src/services/crud-helper.service.ts +1 -1
- package/src/services/model-metadata.service.ts +2 -1
- package/src/services/module-metadata.service.ts +2 -1
- package/src/services/queues/database-publisher.service.ts +1 -1
- package/src/services/queues/database-subscriber.service.ts +17 -28
- package/src/services/queues/rabbitmq-subscriber.service.ts +250 -110
- package/src/services/scheduled-jobs/scheduler.service.ts +31 -14
|
@@ -18,6 +18,7 @@ import { ChatterMessageDetails } from '../entities/chatter-message-details.entit
|
|
|
18
18
|
import { ChatterMessage } from '../entities/chatter-message.entity';
|
|
19
19
|
import { getMediaStorageProvider } from './mediaStorageProviders';
|
|
20
20
|
import { RequestContextService } from './request-context.service';
|
|
21
|
+
import { take } from 'rxjs';
|
|
21
22
|
@Injectable()
|
|
22
23
|
export class ChatterMessageService extends CRUDService<ChatterMessage> {
|
|
23
24
|
constructor(
|
|
@@ -331,28 +332,28 @@ export class ChatterMessageService extends CRUDService<ChatterMessage> {
|
|
|
331
332
|
|
|
332
333
|
if (field.type === 'relation') {
|
|
333
334
|
if (field.relationType === "many-to-one") {
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
}
|
|
337
|
-
|
|
338
|
-
try {
|
|
339
|
-
const relatedModel = await this.modelMetadataRepo.findOne({
|
|
340
|
-
where: { singularName: field.relationCoModelSingularName || field.relation },
|
|
341
|
-
relations: { userKeyField: true }
|
|
342
|
-
});
|
|
343
|
-
|
|
344
|
-
if (relatedModel && relatedModel.userKeyField) {
|
|
345
|
-
const userKeyFieldName = relatedModel.userKeyField.name;
|
|
346
|
-
return value[userKeyFieldName] ? value[userKeyFieldName].toString() : '';
|
|
335
|
+
if (value.name) {
|
|
336
|
+
return value.name;
|
|
347
337
|
}
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
338
|
+
|
|
339
|
+
try {
|
|
340
|
+
const relatedModel = await this.modelMetadataRepo.findOne({
|
|
341
|
+
where: { singularName: field.relationCoModelSingularName || field.relation },
|
|
342
|
+
relations: { userKeyField: true }
|
|
343
|
+
});
|
|
344
|
+
|
|
345
|
+
if (relatedModel && relatedModel.userKeyField) {
|
|
346
|
+
const userKeyFieldName = relatedModel.userKeyField.name;
|
|
347
|
+
return value[userKeyFieldName] ? value[userKeyFieldName].toString() : '';
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
if (value.id) {
|
|
351
|
+
return value.id.toString();
|
|
352
|
+
}
|
|
353
|
+
} catch (error) {
|
|
354
|
+
console.error('Error fetching related model metadata:', error);
|
|
355
|
+
return value.id ? value.id.toString() : '';
|
|
351
356
|
}
|
|
352
|
-
} catch (error) {
|
|
353
|
-
console.error('Error fetching related model metadata:', error);
|
|
354
|
-
return value.id ? value.id.toString() : '';
|
|
355
|
-
}
|
|
356
357
|
}
|
|
357
358
|
|
|
358
359
|
if (field.relationType === 'many-to-many') {
|
|
@@ -501,11 +502,9 @@ export class ChatterMessageService extends CRUDService<ChatterMessage> {
|
|
|
501
502
|
return populatedEntity;
|
|
502
503
|
}
|
|
503
504
|
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
query: any
|
|
508
|
-
) {
|
|
505
|
+
// [2026-02-05T23:31:21.025Z] INFO: [200 OK]
|
|
506
|
+
// GET /api/chatter-message/getChatterMessages/216/mswipeBoomboxBulkUpload?populateMedia[0]=messageAttachments&populate[0]=user&populate[1]=chatterMessageDetails&limit=25 22747ms
|
|
507
|
+
async getChatterMessages(entityId: number, entityName: string, query: any) {
|
|
509
508
|
const { limit = 25, offset = 0, populate = [], populateMedia = [], filters } = query;
|
|
510
509
|
|
|
511
510
|
const model = await this.modelMetadataRepo.findOne({
|
|
@@ -524,6 +523,9 @@ export class ChatterMessageService extends CRUDService<ChatterMessage> {
|
|
|
524
523
|
const relatedEntitiesMap = new Map<string, number[]>();
|
|
525
524
|
|
|
526
525
|
for (const field of oneToManyFields) {
|
|
526
|
+
if (field.enableAuditTracking === false) {
|
|
527
|
+
continue
|
|
528
|
+
}
|
|
527
529
|
const coModelName = field.relationCoModelSingularName;
|
|
528
530
|
const coModelFieldName = field.relationCoModelFieldName;
|
|
529
531
|
|
|
@@ -539,7 +541,11 @@ export class ChatterMessageService extends CRUDService<ChatterMessage> {
|
|
|
539
541
|
const relatedEntityRepository = em.getRepository(classify(coModelName));
|
|
540
542
|
|
|
541
543
|
const relatedEntities = await relatedEntityRepository.find({
|
|
542
|
-
|
|
544
|
+
select: {
|
|
545
|
+
id: true,
|
|
546
|
+
},
|
|
547
|
+
where: { [coModelFieldName]: { id: entityId } },
|
|
548
|
+
take: 5,
|
|
543
549
|
});
|
|
544
550
|
|
|
545
551
|
const relatedIds = relatedEntities.map((entity: any) => entity.id);
|
|
@@ -164,7 +164,7 @@ export class CrudHelperService {
|
|
|
164
164
|
}
|
|
165
165
|
|
|
166
166
|
normalize(value: string | string[]): string[] {
|
|
167
|
-
if (!value) return []
|
|
167
|
+
if (!value) return []; // if the value is nullish, then return an empty array
|
|
168
168
|
return Array.isArray(value) ? value : [value]; // if the value is an array, return it as is, otherwise return it as an array
|
|
169
169
|
}
|
|
170
170
|
|
|
@@ -876,7 +876,8 @@ export class ModelMetadataService {
|
|
|
876
876
|
sequenceNumber: 1,
|
|
877
877
|
actionUserKey: actionName,
|
|
878
878
|
moduleUserKey: `${model.module.name}`,
|
|
879
|
-
parentMenuItemUserKey: ""
|
|
879
|
+
parentMenuItemUserKey: "",
|
|
880
|
+
iconName : ""
|
|
880
881
|
};
|
|
881
882
|
|
|
882
883
|
const modelListview = {
|
|
@@ -181,7 +181,8 @@ export class ModuleMetadataService {
|
|
|
181
181
|
sequenceNumber: 1,
|
|
182
182
|
actionUserKey: `${module?.name}-home-action`,
|
|
183
183
|
moduleUserKey: module?.name,
|
|
184
|
-
parentMenuItemUserKey: ""
|
|
184
|
+
parentMenuItemUserKey: "",
|
|
185
|
+
iconName : "home"
|
|
185
186
|
}
|
|
186
187
|
],
|
|
187
188
|
views: [],
|
|
@@ -18,7 +18,7 @@ export abstract class DatabasePublisher<T> implements QueuePublisher<T> {
|
|
|
18
18
|
if (!this.serviceRole) {
|
|
19
19
|
this.logger.debug('Queue service Role is not defined in the environment variables');
|
|
20
20
|
}
|
|
21
|
-
this.logger.debug(`DatabasePublisher instance created with options: ${JSON.stringify(this.options())}`);
|
|
21
|
+
// this.logger.debug(`DatabasePublisher instance created with options: ${JSON.stringify(this.options())}`);
|
|
22
22
|
}
|
|
23
23
|
|
|
24
24
|
abstract options(): QueuesModuleOptions;
|
|
@@ -20,7 +20,7 @@ export abstract class DatabaseSubscriber<T> implements OnModuleInit, QueueSubscr
|
|
|
20
20
|
if (!this.serviceRole) {
|
|
21
21
|
this.logger.debug('Queue service Role is not defined in the environment variables');
|
|
22
22
|
}
|
|
23
|
-
this.logger.debug(`DatabaseSubscriber instance created with options: ${JSON.stringify(this.options())}`);
|
|
23
|
+
// this.logger.debug(`DatabaseSubscriber instance created with options: ${JSON.stringify(this.options())}`);
|
|
24
24
|
}
|
|
25
25
|
|
|
26
26
|
abstract subscribe(message: QueueMessage<T>);
|
|
@@ -72,42 +72,31 @@ export abstract class DatabaseSubscriber<T> implements OnModuleInit, QueueSubscr
|
|
|
72
72
|
// this.logger.debug(`#### DatabaseSubscriber finished processing message from queue: ${queueName}`);
|
|
73
73
|
}
|
|
74
74
|
|
|
75
|
-
// async onModuleInit(): Promise<void> {
|
|
76
|
-
// // we will start subscriber only if the current service role is subscriber.
|
|
77
|
-
// if (['both', 'subscriber'].includes(this.serviceRole)) {
|
|
78
|
-
|
|
79
|
-
// const options = this.options();
|
|
80
|
-
|
|
81
|
-
// const queueName = options.queueName;
|
|
82
|
-
// // setInterval(() => this.processNext(queueName), 1000);
|
|
83
|
-
// const poll = async () => {
|
|
84
|
-
// try {
|
|
85
|
-
// await this.processNext(queueName);
|
|
86
|
-
// } catch (err) {
|
|
87
|
-
// this.logger.error(`Polling error: ${err.message}`);
|
|
88
|
-
// } finally {
|
|
89
|
-
// setTimeout(poll, 1000); // Wait 1s *after* processing finishes
|
|
90
|
-
// }
|
|
91
|
-
// };
|
|
92
|
-
|
|
93
|
-
// // start the loop
|
|
94
|
-
// poll();
|
|
95
|
-
|
|
96
|
-
// this.logger.log(`DatabaseSubscriber ready to consume messages: ${JSON.stringify(this.options())}`);
|
|
97
|
-
// }
|
|
98
|
-
// }
|
|
99
|
-
|
|
100
75
|
async onModuleInit(): Promise<void> {
|
|
76
|
+
// Not using SettingService here as that will necessitate all implementors of DatabaseSubscriber to also inject SettingService which is not ideal.
|
|
77
|
+
// Instead we directly read the environment variables here.
|
|
101
78
|
const defaultBroker = process.env.QUEUES_DEFAULT_BROKER || 'database';
|
|
102
79
|
const solidCliRunning = process.env.SOLID_CLI_RUNNING || "false";
|
|
80
|
+
const queueNameRegex = (process.env.QUEUES_QUEUE_NAME_REGEX_TO_ENABLE || '').trim();
|
|
103
81
|
|
|
104
82
|
// we will start subscriber only if the current service role is subscriber.
|
|
105
83
|
if (['both', 'subscriber'].includes(this.serviceRole) && defaultBroker === 'database' && solidCliRunning === "false") {
|
|
106
|
-
|
|
107
84
|
const options = this.options();
|
|
108
|
-
|
|
109
85
|
const queueName = options.queueName;
|
|
110
86
|
|
|
87
|
+
if (queueNameRegex && queueNameRegex !== "all") {
|
|
88
|
+
try {
|
|
89
|
+
const regex = new RegExp(queueNameRegex);
|
|
90
|
+
if (!regex.test(queueName)) {
|
|
91
|
+
this.logger.log(`DatabaseSubscriber for queue ${queueName} is disabled because it does not match QUEUES_QUEUE_NAME_REGEX_TO_ENABLE=${queueNameRegex}`);
|
|
92
|
+
return;
|
|
93
|
+
}
|
|
94
|
+
} catch (error) {
|
|
95
|
+
this.logger.error(`Invalid QUEUES_QUEUE_NAME_REGEX_TO_ENABLE regex "${queueNameRegex}". Subscriber for queue ${queueName} will not start.`);
|
|
96
|
+
return;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
111
100
|
this.poller.start(queueName, (q) => this.processNext(q), {
|
|
112
101
|
baseDelayMs: 1000,
|
|
113
102
|
maxDelayMs: 30_000,
|
|
@@ -10,11 +10,14 @@ export abstract class RabbitMqSubscriber<T> implements OnModuleInit, QueueSubscr
|
|
|
10
10
|
private readonly logger = new Logger(RabbitMqSubscriber.name);
|
|
11
11
|
private readonly url: string;
|
|
12
12
|
private readonly serviceRole: string;
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
13
|
+
private connection: amqp.Connection | null = null;
|
|
14
|
+
private channel: amqp.Channel | null = null;
|
|
15
|
+
private consumerTag: string | null = null;
|
|
16
|
+
private reconnectPromise: Promise<void> | null = null;
|
|
17
|
+
private reconnectAttempt = 0;
|
|
18
|
+
private stopping = false;
|
|
19
|
+
|
|
20
|
+
constructor(protected readonly mqMessageService: MqMessageService, protected readonly mqMessageQueueService: MqMessageQueueService) {
|
|
18
21
|
this.url = process.env.QUEUES_RABBIT_MQ_URL;
|
|
19
22
|
this.serviceRole = process.env.QUEUES_SERVICE_ROLE;
|
|
20
23
|
if (!this.url) {
|
|
@@ -46,100 +49,266 @@ export abstract class RabbitMqSubscriber<T> implements OnModuleInit, QueueSubscr
|
|
|
46
49
|
username: url.username,
|
|
47
50
|
password: decodeURIComponent(url.password),
|
|
48
51
|
frameMax: 131072,
|
|
52
|
+
heartbeat: 30,
|
|
49
53
|
});
|
|
50
54
|
|
|
51
55
|
return connection
|
|
52
56
|
}
|
|
53
57
|
|
|
54
58
|
async onModuleInit(): Promise<void> {
|
|
59
|
+
// Not using SettingService here as that will necessitate all implementors of RabbitMqSubscriber to also inject SettingService which is not ideal.
|
|
60
|
+
// Instead we directly read the environment variables here.
|
|
61
|
+
const defaultBroker = process.env.QUEUES_DEFAULT_BROKER || 'rabbitmq';
|
|
55
62
|
const solidCliRunning = process.env.SOLID_CLI_RUNNING || "false";
|
|
63
|
+
const queueNameRegex = (process.env.QUEUES_QUEUE_NAME_REGEX_TO_ENABLE || '').trim();
|
|
56
64
|
|
|
57
65
|
// we will start subscriber only if the current service role is subscriber.
|
|
58
|
-
if (this.url && ['both', 'subscriber'].includes(this.serviceRole) && solidCliRunning === "false") {
|
|
66
|
+
if (this.url && ['both', 'subscriber'].includes(this.serviceRole) && solidCliRunning === "false" && defaultBroker === 'rabbitmq') {
|
|
67
|
+
const options = this.options();
|
|
68
|
+
const queueName = options.queueName;
|
|
59
69
|
|
|
60
|
-
|
|
61
|
-
|
|
70
|
+
if (queueNameRegex && queueNameRegex !== "all") {
|
|
71
|
+
try {
|
|
72
|
+
const regex = new RegExp(queueNameRegex);
|
|
73
|
+
if (!regex.test(queueName)) {
|
|
74
|
+
this.logger.log(`RabbitMqSubscriber for queue ${queueName} is disabled because it does not match QUEUES_QUEUE_NAME_REGEX_TO_ENABLE=${queueNameRegex}`);
|
|
75
|
+
return;
|
|
76
|
+
}
|
|
77
|
+
} catch (error) {
|
|
78
|
+
this.logger.error(`Invalid QUEUES_QUEUE_NAME_REGEX_TO_ENABLE regex "${queueNameRegex}". Subscriber for queue ${queueName} will not start.`);
|
|
79
|
+
return;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
62
82
|
|
|
63
|
-
let connection;
|
|
64
83
|
try {
|
|
65
|
-
|
|
66
|
-
|
|
84
|
+
await this.connectAndConsume(queueName);
|
|
85
|
+
} catch (err) {
|
|
86
|
+
this.logger.error(`Failed to connect to RabbitMQ for queue ${queueName}: ${(err as Error).message}`, (err as Error).stack);
|
|
87
|
+
this.triggerReconnect(queueName, 'initial connection failure');
|
|
67
88
|
}
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
89
|
+
|
|
90
|
+
this.logger.log(`RabbitMqSubscriber ready to consume messages: ${JSON.stringify(this.options())} and url: ${this.url}`);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
private async connectAndConsume(queueName: string): Promise<void> {
|
|
95
|
+
await this.cleanup();
|
|
96
|
+
|
|
97
|
+
let connection: amqp.Connection;
|
|
98
|
+
try {
|
|
99
|
+
connection = await this.establishConnection();
|
|
100
|
+
} catch (err) {
|
|
101
|
+
this.logger.error(`Failed to connect to RabbitMQ for queue ${queueName}: ${(err as Error).message}`, (err as Error).stack);
|
|
102
|
+
throw err;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
this.connection = connection;
|
|
106
|
+
|
|
107
|
+
connection.on('error', (err) => {
|
|
108
|
+
if (connection !== this.connection) return;
|
|
109
|
+
this.logger.error(`RabbitMqSubscriber connection error for queue ${queueName}: ${(err as Error).message}`);
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
connection.on('close', () => {
|
|
113
|
+
if (connection !== this.connection) return;
|
|
114
|
+
this.logger.warn(`RabbitMqSubscriber connection closed for queue ${queueName}`);
|
|
115
|
+
this.triggerReconnect(queueName, 'connection closed');
|
|
116
|
+
});
|
|
117
|
+
|
|
118
|
+
const channel = await connection.createChannel();
|
|
119
|
+
this.channel = channel;
|
|
120
|
+
|
|
121
|
+
channel.on('error', (err) => {
|
|
122
|
+
if (channel !== this.channel) return;
|
|
123
|
+
this.logger.error(`RabbitMqSubscriber channel error for queue ${queueName}: ${(err as Error).message}`);
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
channel.on('close', () => {
|
|
127
|
+
if (channel !== this.channel) return;
|
|
128
|
+
this.logger.warn(`RabbitMqSubscriber channel closed for queue ${queueName}`);
|
|
129
|
+
this.triggerReconnect(queueName, 'channel closed');
|
|
130
|
+
});
|
|
131
|
+
|
|
132
|
+
// Process one message at a time per consumer to avoid parallel work on the same subscriber instance.
|
|
133
|
+
await channel.prefetch(1);
|
|
134
|
+
|
|
135
|
+
// Use a direct exchange with a stable routing key so retry DLX can route back to the main queue.
|
|
136
|
+
const exchangeName = `${queueName}.exchange`;
|
|
137
|
+
const routingKey = `${queueName}.routing-key`;
|
|
138
|
+
const retryQueue = `${queueName}.retry`;
|
|
139
|
+
const failedQueue = `${queueName}.failed`;
|
|
140
|
+
|
|
141
|
+
await channel.assertExchange(exchangeName, 'direct', {});
|
|
142
|
+
await channel.assertQueue(queueName, {});
|
|
143
|
+
await channel.bindQueue(queueName, exchangeName, routingKey);
|
|
144
|
+
|
|
145
|
+
// Retry queue uses DLX to route expired messages back to the main exchange/routing key.
|
|
146
|
+
await channel.assertQueue(retryQueue, {
|
|
147
|
+
arguments: {
|
|
148
|
+
'x-dead-letter-exchange': exchangeName,
|
|
149
|
+
'x-dead-letter-routing-key': routingKey,
|
|
71
150
|
}
|
|
151
|
+
});
|
|
72
152
|
|
|
73
|
-
|
|
74
|
-
// this.logger.debug(`RabbitMqSubscriber channel created: ${JSON.stringify(this.options())} and url: ${url}`);
|
|
153
|
+
await channel.assertQueue(failedQueue, {});
|
|
75
154
|
|
|
76
|
-
|
|
155
|
+
const consumeResult = await channel.consume(
|
|
156
|
+
queueName,
|
|
157
|
+
async (rawMessage) => {
|
|
158
|
+
if (!rawMessage) {
|
|
159
|
+
return;
|
|
160
|
+
}
|
|
77
161
|
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
const routingKey = `${queueName}.routing-key`;
|
|
81
|
-
|
|
82
|
-
await channel.assertExchange(exchangeName, 'direct', {});
|
|
83
|
-
// this.logger.debug(`RabbitMqSubscriber channel asserted: ${JSON.stringify(this.options())} and url: ${url}`);
|
|
84
|
-
|
|
85
|
-
const queue = await channel.assertQueue(queueName, {});
|
|
86
|
-
// this.logger.debug(`RabbitMqSubscriber queue asserted: ${JSON.stringify(this.options())} and url: ${url}`);
|
|
87
|
-
|
|
88
|
-
await channel.bindQueue(queue.queue, exchangeName, routingKey);
|
|
89
|
-
// this.logger.debug(`RabbitMqSubscriber queue bound: ${JSON.stringify(this.options())} and url: ${url}`);
|
|
90
|
-
|
|
91
|
-
// Consume messages from the queue
|
|
92
|
-
channel.consume(
|
|
93
|
-
queue.queue,
|
|
94
|
-
async (rawMessage) => {
|
|
95
|
-
if (rawMessage) {
|
|
96
|
-
const messageContentString = rawMessage.content.toString();
|
|
97
|
-
// this.logger.debug(`RabbitMqSubscriber Received raw message: ${messageContentString}`);
|
|
98
|
-
|
|
99
|
-
let message: QueueMessage<T> = null;
|
|
100
|
-
|
|
101
|
-
try {
|
|
102
|
-
message = JSON.parse(messageContentString) as QueueMessage<T>;
|
|
103
|
-
|
|
104
|
-
// this is the first time we are receiving the message so we set the currentRetry to 0
|
|
105
|
-
if (!message.retryCount) message.retryCount = 0;
|
|
106
|
-
if (!message.retryInterval) message.retryInterval = 1000;
|
|
107
|
-
if (!message.currentRetry) message.currentRetry = 0;
|
|
108
|
-
|
|
109
|
-
await this.processMessage(message, rawMessage, channel);
|
|
110
|
-
}
|
|
111
|
-
catch (error) {
|
|
112
|
-
this.logger.error(`Error processing message: ${error.message}`);
|
|
113
|
-
|
|
114
|
-
// if an error occurs then if retryCount is set we start retrying.
|
|
115
|
-
if (message) {
|
|
116
|
-
if (message.currentRetry < message.retryCount) {
|
|
117
|
-
await this.updateStatusInDatabase('retrying', message);
|
|
118
|
-
|
|
119
|
-
message.currentRetry++;
|
|
120
|
-
this.logger.warn(`Retrying message (${message.currentRetry}/${message.retryCount}) after ${message.retryInterval}ms`);
|
|
121
|
-
setTimeout(() => {
|
|
122
|
-
this.retryMessage(message, rawMessage, channel);
|
|
123
|
-
}, message.retryInterval);
|
|
124
|
-
} else {
|
|
125
|
-
await this.updateStatusInDatabase('failed', message, error.message, '');
|
|
126
|
-
|
|
127
|
-
this.logger.error(`Message failed after ${message.retryCount} attempts: ${error.message}`);
|
|
128
|
-
channel.ack(rawMessage); // Discard the message after max retries
|
|
129
|
-
}
|
|
130
|
-
}
|
|
131
|
-
|
|
132
|
-
}
|
|
133
|
-
}
|
|
134
|
-
},
|
|
135
|
-
// { noAck: true },
|
|
136
|
-
{},
|
|
137
|
-
);
|
|
162
|
+
const messageContentString = rawMessage.content.toString();
|
|
163
|
+
let message: QueueMessage<T> = null;
|
|
138
164
|
|
|
139
|
-
|
|
165
|
+
try {
|
|
166
|
+
message = JSON.parse(messageContentString) as QueueMessage<T>;
|
|
167
|
+
} catch (error) {
|
|
168
|
+
this.logger.error(`Invalid JSON message on queue ${queueName}: ${(error as Error).message}`);
|
|
169
|
+
await this.publishToFailedQueue(queueName, rawMessage.content, channel, error);
|
|
170
|
+
channel.ack(rawMessage);
|
|
171
|
+
return;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
if (!message.retryCount) message.retryCount = 0;
|
|
175
|
+
if (!message.retryInterval) message.retryInterval = 1000;
|
|
176
|
+
if (!message.currentRetry) message.currentRetry = 0;
|
|
177
|
+
|
|
178
|
+
try {
|
|
179
|
+
await this.processMessage(message, rawMessage, channel);
|
|
180
|
+
} catch (error) {
|
|
181
|
+
await this.handleProcessingError(message, rawMessage, channel, error, queueName);
|
|
182
|
+
}
|
|
183
|
+
},
|
|
184
|
+
// Explicit ack enables reliable processing and retry routing.
|
|
185
|
+
{ noAck: false },
|
|
186
|
+
);
|
|
187
|
+
|
|
188
|
+
this.consumerTag = consumeResult.consumerTag;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
// Retry flow: update DB -> increment retry -> send to retry queue with per-message expiration -> ack original.
|
|
192
|
+
private async handleProcessingError(message: QueueMessage<T>, rawMessage: amqp.ConsumeMessage, channel: amqp.Channel, error: any, queueName: string): Promise<void> {
|
|
193
|
+
const errorMessage = (error as Error)?.message || String(error);
|
|
194
|
+
this.logger.error(`Error processing message on queue ${queueName}: ${errorMessage}`);
|
|
195
|
+
|
|
196
|
+
if (message.currentRetry < message.retryCount) {
|
|
197
|
+
await this.updateStatusInDatabase('retrying', message);
|
|
198
|
+
|
|
199
|
+
message.currentRetry++;
|
|
200
|
+
const retryQueue = `${queueName}.retry`;
|
|
201
|
+
const payload = Buffer.from(JSON.stringify(message));
|
|
202
|
+
|
|
203
|
+
// Per-message expiration keeps the message in the retry queue until TTL, then DLX routes it back.
|
|
204
|
+
channel.sendToQueue(retryQueue, payload, {
|
|
205
|
+
expiration: String(message.retryInterval || 1000),
|
|
206
|
+
headers: {
|
|
207
|
+
'x-error': errorMessage,
|
|
208
|
+
}
|
|
209
|
+
});
|
|
210
|
+
|
|
211
|
+
channel.ack(rawMessage);
|
|
212
|
+
this.logger.warn(`Retrying message (${message.currentRetry}/${message.retryCount}) after ${message.retryInterval}ms on queue ${queueName}`);
|
|
213
|
+
return;
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
await this.updateStatusInDatabase('failed', message, errorMessage, '');
|
|
217
|
+
channel.ack(rawMessage);
|
|
218
|
+
await this.publishToFailedQueue(queueName, Buffer.from(JSON.stringify(message)), channel, error);
|
|
219
|
+
this.logger.error(`Message failed after ${message.retryCount} attempts on queue ${queueName}: ${errorMessage}`);
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
private async publishToFailedQueue(queueName: string, payload: Buffer | string, channel: amqp.Channel, error?: any): Promise<void> {
|
|
223
|
+
const failedQueue = `${queueName}.failed`;
|
|
224
|
+
const body = Buffer.isBuffer(payload) ? payload : Buffer.from(payload);
|
|
225
|
+
const errorMessage = (error as Error)?.message || String(error || '');
|
|
226
|
+
|
|
227
|
+
try {
|
|
228
|
+
channel.sendToQueue(failedQueue, body, errorMessage ? {
|
|
229
|
+
headers: { 'x-error': errorMessage }
|
|
230
|
+
} : undefined);
|
|
231
|
+
} catch (err) {
|
|
232
|
+
this.logger.error(`Failed to publish to failed queue ${failedQueue}: ${(err as Error).message}`);
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
private triggerReconnect(queueName: string, reason: string) {
|
|
237
|
+
if (this.stopping) return;
|
|
238
|
+
if (this.reconnectPromise) return;
|
|
239
|
+
|
|
240
|
+
this.reconnectPromise = this.reconnectLoop(queueName, reason)
|
|
241
|
+
.finally(() => {
|
|
242
|
+
this.reconnectPromise = null;
|
|
243
|
+
});
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
// Reconnect with backoff to avoid hammering the broker during outages.
|
|
247
|
+
private async reconnectLoop(queueName: string, reason: string): Promise<void> {
|
|
248
|
+
this.logger.warn(`RabbitMqSubscriber reconnecting for queue ${queueName}: ${reason}`);
|
|
249
|
+
|
|
250
|
+
while (!this.stopping) {
|
|
251
|
+
try {
|
|
252
|
+
await this.connectAndConsume(queueName);
|
|
253
|
+
this.reconnectAttempt = 0;
|
|
254
|
+
this.logger.log(`RabbitMqSubscriber reconnected for queue ${queueName}`);
|
|
255
|
+
return;
|
|
256
|
+
} catch (err) {
|
|
257
|
+
this.reconnectAttempt += 1;
|
|
258
|
+
const delay = this.backoff();
|
|
259
|
+
this.logger.warn(`RabbitMqSubscriber reconnect failed for queue ${queueName}; retrying in ${delay}ms`);
|
|
260
|
+
await this.sleep(delay);
|
|
261
|
+
}
|
|
140
262
|
}
|
|
141
263
|
}
|
|
142
264
|
|
|
265
|
+
private async cleanup(): Promise<void> {
|
|
266
|
+
const channel = this.channel;
|
|
267
|
+
const connection = this.connection;
|
|
268
|
+
const consumerTag = this.consumerTag;
|
|
269
|
+
|
|
270
|
+
this.channel = null;
|
|
271
|
+
this.connection = null;
|
|
272
|
+
this.consumerTag = null;
|
|
273
|
+
|
|
274
|
+
if (channel) {
|
|
275
|
+
try {
|
|
276
|
+
if (consumerTag) {
|
|
277
|
+
await channel.cancel(consumerTag);
|
|
278
|
+
}
|
|
279
|
+
} catch (_) {
|
|
280
|
+
// ignore
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
try {
|
|
284
|
+
await channel.close();
|
|
285
|
+
} catch (_) {
|
|
286
|
+
// ignore
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
if (connection) {
|
|
291
|
+
try {
|
|
292
|
+
await connection.close();
|
|
293
|
+
} catch (_) {
|
|
294
|
+
// ignore
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
private sleep(ms: number): Promise<void> {
|
|
300
|
+
return new Promise(resolve => setTimeout(resolve, ms));
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
// Exponential backoff with jitter, capped to 30s.
|
|
304
|
+
private backoff(): number {
|
|
305
|
+
const baseMs = 1000;
|
|
306
|
+
const maxMs = 30_000;
|
|
307
|
+
const exp = Math.min(maxMs, baseMs * Math.pow(2, this.reconnectAttempt));
|
|
308
|
+
const jitter = Math.floor(Math.random() * (exp * 0.2));
|
|
309
|
+
return Math.min(maxMs, exp + jitter);
|
|
310
|
+
}
|
|
311
|
+
|
|
143
312
|
/**
|
|
144
313
|
* Abstract method for message processing logic.
|
|
145
314
|
*/
|
|
@@ -152,43 +321,14 @@ export abstract class RabbitMqSubscriber<T> implements OnModuleInit, QueueSubscr
|
|
|
152
321
|
// Ack the message.
|
|
153
322
|
channel.ack(rawMessage);
|
|
154
323
|
|
|
155
|
-
//
|
|
324
|
+
// Persist success output and timing.
|
|
156
325
|
await this.updateStatusInDatabase('succeeded', message, '', result ? JSON.stringify(result, null, 2) : '');
|
|
157
326
|
|
|
158
327
|
}
|
|
159
328
|
|
|
160
|
-
/**
|
|
161
|
-
* Retry the message by invoking the processing logic again.
|
|
162
|
-
*/
|
|
163
|
-
private async retryMessage(message: QueueMessage<T>, rawMessage, channel) {
|
|
164
|
-
try {
|
|
165
|
-
await this.processMessage(message, rawMessage, channel);
|
|
166
|
-
} catch (error) {
|
|
167
|
-
if (message.currentRetry < message.retryCount) {
|
|
168
|
-
await this.updateStatusInDatabase('retrying', message);
|
|
169
|
-
|
|
170
|
-
message.currentRetry++;
|
|
171
|
-
this.logger.warn(`Retrying message (${message.currentRetry}/${message.retryCount}) after ${message.retryInterval}ms: ${error.message}`);
|
|
172
|
-
setTimeout(() => {
|
|
173
|
-
this.retryMessage(message, rawMessage, channel);
|
|
174
|
-
}, message.retryInterval);
|
|
175
|
-
} else {
|
|
176
|
-
|
|
177
|
-
this.logger.error(`Message failed after ${message.retryCount} attempts: ${error.message}`);
|
|
178
|
-
|
|
179
|
-
// Discard the message after max retries
|
|
180
|
-
channel.ack(rawMessage);
|
|
181
|
-
|
|
182
|
-
// TODO: Store the error in the database and update the status accordingly.
|
|
183
|
-
await this.updateStatusInDatabase('failed', message, error.message, '');
|
|
184
|
-
|
|
185
|
-
}
|
|
186
|
-
}
|
|
187
|
-
}
|
|
188
|
-
|
|
189
329
|
private async updateStatusInDatabase(stage: string, message: QueueMessage<T>, error: string = '', result: string = '') {
|
|
190
330
|
|
|
191
|
-
//
|
|
331
|
+
// Update the existing message record by messageId; creation happens upstream.
|
|
192
332
|
try {
|
|
193
333
|
// 1. resolve the queue first
|
|
194
334
|
const mqMessage = await this.mqMessageService.repo.findOne({
|
|
@@ -220,4 +360,4 @@ export abstract class RabbitMqSubscriber<T> implements OnModuleInit, QueueSubscr
|
|
|
220
360
|
|
|
221
361
|
}
|
|
222
362
|
|
|
223
|
-
}
|
|
363
|
+
}
|