@solidxai/core 0.1.0 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (131) hide show
  1. package/dist/dtos/create-saved-filters.dto.d.ts.map +1 -1
  2. package/dist/dtos/create-saved-filters.dto.js.map +1 -1
  3. package/dist/entities/chatter-message.entity.d.ts.map +1 -1
  4. package/dist/entities/chatter-message.entity.js +2 -0
  5. package/dist/entities/chatter-message.entity.js.map +1 -1
  6. package/dist/helpers/schematic.service.d.ts.map +1 -1
  7. package/dist/helpers/schematic.service.js +6 -2
  8. package/dist/helpers/schematic.service.js.map +1 -1
  9. package/dist/jobs/api-email-queue-options.d.ts.map +1 -1
  10. package/dist/jobs/api-email-queue-options.js +2 -2
  11. package/dist/jobs/api-email-queue-options.js.map +1 -1
  12. package/dist/jobs/chatter-queue-options.js +2 -2
  13. package/dist/jobs/chatter-queue-options.js.map +1 -1
  14. package/dist/jobs/computed-field-evaluation-queue-options.js +2 -2
  15. package/dist/jobs/computed-field-evaluation-queue-options.js.map +1 -1
  16. package/dist/jobs/database/api-email-queue-options-database.js +2 -2
  17. package/dist/jobs/database/api-email-queue-options-database.js.map +1 -1
  18. package/dist/jobs/database/computed-field-evaluation-queue-options-database.js +2 -2
  19. package/dist/jobs/database/computed-field-evaluation-queue-options-database.js.map +1 -1
  20. package/dist/jobs/database/generate-code-queue-options-database.js +2 -2
  21. package/dist/jobs/database/generate-code-queue-options-database.js.map +1 -1
  22. package/dist/jobs/database/msg91-sms-queue-database-options.d.ts.map +1 -1
  23. package/dist/jobs/database/msg91-sms-queue-database-options.js +2 -2
  24. package/dist/jobs/database/msg91-sms-queue-database-options.js.map +1 -1
  25. package/dist/jobs/database/msg91-whatsapp-queue-options-database.js +2 -2
  26. package/dist/jobs/database/msg91-whatsapp-queue-options-database.js.map +1 -1
  27. package/dist/jobs/database/otp-queue-options-database.d.ts.map +1 -1
  28. package/dist/jobs/database/otp-queue-options-database.js +2 -2
  29. package/dist/jobs/database/otp-queue-options-database.js.map +1 -1
  30. package/dist/jobs/database/smtp-email-queue-options-database.js +1 -1
  31. package/dist/jobs/database/smtp-email-queue-options-database.js.map +1 -1
  32. package/dist/jobs/database/test-queue-options-database.js +2 -2
  33. package/dist/jobs/database/test-queue-options-database.js.map +1 -1
  34. package/dist/jobs/database/three60-whatsapp-queue-options-database.js +2 -2
  35. package/dist/jobs/database/three60-whatsapp-queue-options-database.js.map +1 -1
  36. package/dist/jobs/database/trigger-mcp-client-queue-options.js +2 -2
  37. package/dist/jobs/database/trigger-mcp-client-queue-options.js.map +1 -1
  38. package/dist/jobs/database/twilio-sms-queue-database-options.js +2 -2
  39. package/dist/jobs/database/twilio-sms-queue-database-options.js.map +1 -1
  40. package/dist/jobs/generate-code-queue-options.js +2 -2
  41. package/dist/jobs/generate-code-queue-options.js.map +1 -1
  42. package/dist/jobs/msg91-otp-queue-options.d.ts.map +1 -1
  43. package/dist/jobs/msg91-otp-queue-options.js +2 -2
  44. package/dist/jobs/msg91-otp-queue-options.js.map +1 -1
  45. package/dist/jobs/msg91-sms-queue-options.d.ts.map +1 -1
  46. package/dist/jobs/msg91-sms-queue-options.js +2 -2
  47. package/dist/jobs/msg91-sms-queue-options.js.map +1 -1
  48. package/dist/jobs/msg91-whatsapp-queue-options.d.ts.map +1 -1
  49. package/dist/jobs/msg91-whatsapp-queue-options.js +2 -2
  50. package/dist/jobs/msg91-whatsapp-queue-options.js.map +1 -1
  51. package/dist/jobs/smtp-email-queue-options.d.ts.map +1 -1
  52. package/dist/jobs/smtp-email-queue-options.js +1 -1
  53. package/dist/jobs/smtp-email-queue-options.js.map +1 -1
  54. package/dist/jobs/test-queue-options.d.ts.map +1 -1
  55. package/dist/jobs/test-queue-options.js +2 -2
  56. package/dist/jobs/test-queue-options.js.map +1 -1
  57. package/dist/jobs/three60-whatsapp-queue-options.d.ts.map +1 -1
  58. package/dist/jobs/three60-whatsapp-queue-options.js +2 -2
  59. package/dist/jobs/three60-whatsapp-queue-options.js.map +1 -1
  60. package/dist/jobs/three60-whatsapp-subscriber.service.js +2 -2
  61. package/dist/jobs/three60-whatsapp-subscriber.service.js.map +1 -1
  62. package/dist/jobs/trigger-mcp-client-queue-options.js +2 -2
  63. package/dist/jobs/trigger-mcp-client-queue-options.js.map +1 -1
  64. package/dist/jobs/twilio-sms-queue-options.js +2 -2
  65. package/dist/jobs/twilio-sms-queue-options.js.map +1 -1
  66. package/dist/seeders/module-metadata-seeder.service.d.ts.map +1 -1
  67. package/dist/seeders/module-metadata-seeder.service.js +2 -1
  68. package/dist/seeders/module-metadata-seeder.service.js.map +1 -1
  69. package/dist/seeders/seed-data/solid-core-metadata.json +34 -9
  70. package/dist/services/chatter-message.service.d.ts.map +1 -1
  71. package/dist/services/chatter-message.service.js +8 -1
  72. package/dist/services/chatter-message.service.js.map +1 -1
  73. package/dist/services/crud-helper.service.js.map +1 -1
  74. package/dist/services/model-metadata.service.d.ts.map +1 -1
  75. package/dist/services/model-metadata.service.js +2 -1
  76. package/dist/services/model-metadata.service.js.map +1 -1
  77. package/dist/services/module-metadata.service.d.ts.map +1 -1
  78. package/dist/services/module-metadata.service.js +2 -1
  79. package/dist/services/module-metadata.service.js.map +1 -1
  80. package/dist/services/queues/database-publisher.service.js +0 -1
  81. package/dist/services/queues/database-publisher.service.js.map +1 -1
  82. package/dist/services/queues/database-subscriber.service.d.ts.map +1 -1
  83. package/dist/services/queues/database-subscriber.service.js +14 -1
  84. package/dist/services/queues/database-subscriber.service.js.map +1 -1
  85. package/dist/services/queues/rabbitmq-subscriber.service.d.ts +14 -1
  86. package/dist/services/queues/rabbitmq-subscriber.service.d.ts.map +1 -1
  87. package/dist/services/queues/rabbitmq-subscriber.service.js +209 -66
  88. package/dist/services/queues/rabbitmq-subscriber.service.js.map +1 -1
  89. package/dist/services/scheduled-jobs/scheduler.service.d.ts +1 -0
  90. package/dist/services/scheduled-jobs/scheduler.service.d.ts.map +1 -1
  91. package/dist/services/scheduled-jobs/scheduler.service.js +26 -10
  92. package/dist/services/scheduled-jobs/scheduler.service.js.map +1 -1
  93. package/dist/tsconfig.tsbuildinfo +1 -1
  94. package/package.json +1 -1
  95. package/src/dtos/create-saved-filters.dto.ts +10 -1
  96. package/src/entities/chatter-message.entity.ts +11 -1
  97. package/src/helpers/schematic.service.ts +6 -2
  98. package/src/jobs/api-email-queue-options.ts +2 -6
  99. package/src/jobs/chatter-queue-options.ts +2 -2
  100. package/src/jobs/computed-field-evaluation-queue-options.ts +2 -2
  101. package/src/jobs/database/api-email-queue-options-database.ts +2 -2
  102. package/src/jobs/database/computed-field-evaluation-queue-options-database.ts +2 -2
  103. package/src/jobs/database/generate-code-queue-options-database.ts +2 -2
  104. package/src/jobs/database/msg91-sms-queue-database-options.ts +3 -2
  105. package/src/jobs/database/msg91-whatsapp-queue-options-database.ts +2 -2
  106. package/src/jobs/database/otp-queue-options-database.ts +3 -2
  107. package/src/jobs/database/smtp-email-queue-options-database.ts +1 -1
  108. package/src/jobs/database/test-queue-options-database.ts +2 -2
  109. package/src/jobs/database/three60-whatsapp-queue-options-database.ts +2 -2
  110. package/src/jobs/database/trigger-mcp-client-queue-options.ts +2 -2
  111. package/src/jobs/database/twilio-sms-queue-database-options.ts +2 -2
  112. package/src/jobs/generate-code-queue-options.ts +2 -2
  113. package/src/jobs/msg91-otp-queue-options.ts +3 -8
  114. package/src/jobs/msg91-sms-queue-options.ts +3 -6
  115. package/src/jobs/msg91-whatsapp-queue-options.ts +4 -7
  116. package/src/jobs/smtp-email-queue-options.ts +1 -6
  117. package/src/jobs/test-queue-options.ts +2 -6
  118. package/src/jobs/three60-whatsapp-queue-options.ts +4 -7
  119. package/src/jobs/three60-whatsapp-subscriber.service.ts +1 -1
  120. package/src/jobs/trigger-mcp-client-queue-options.ts +2 -2
  121. package/src/jobs/twilio-sms-queue-options.ts +3 -3
  122. package/src/seeders/module-metadata-seeder.service.ts +2 -1
  123. package/src/seeders/seed-data/solid-core-metadata.json +34 -9
  124. package/src/services/chatter-message.service.ts +32 -26
  125. package/src/services/crud-helper.service.ts +1 -1
  126. package/src/services/model-metadata.service.ts +2 -1
  127. package/src/services/module-metadata.service.ts +2 -1
  128. package/src/services/queues/database-publisher.service.ts +1 -1
  129. package/src/services/queues/database-subscriber.service.ts +17 -28
  130. package/src/services/queues/rabbitmq-subscriber.service.ts +250 -110
  131. package/src/services/scheduled-jobs/scheduler.service.ts +31 -14
@@ -18,6 +18,7 @@ import { ChatterMessageDetails } from '../entities/chatter-message-details.entit
18
18
  import { ChatterMessage } from '../entities/chatter-message.entity';
19
19
  import { getMediaStorageProvider } from './mediaStorageProviders';
20
20
  import { RequestContextService } from './request-context.service';
21
+ import { take } from 'rxjs';
21
22
  @Injectable()
22
23
  export class ChatterMessageService extends CRUDService<ChatterMessage> {
23
24
  constructor(
@@ -331,28 +332,28 @@ export class ChatterMessageService extends CRUDService<ChatterMessage> {
331
332
 
332
333
  if (field.type === 'relation') {
333
334
  if (field.relationType === "many-to-one") {
334
- if (value.name) {
335
- return value.name;
336
- }
337
-
338
- try {
339
- const relatedModel = await this.modelMetadataRepo.findOne({
340
- where: { singularName: field.relationCoModelSingularName || field.relation },
341
- relations: { userKeyField: true }
342
- });
343
-
344
- if (relatedModel && relatedModel.userKeyField) {
345
- const userKeyFieldName = relatedModel.userKeyField.name;
346
- return value[userKeyFieldName] ? value[userKeyFieldName].toString() : '';
335
+ if (value.name) {
336
+ return value.name;
347
337
  }
348
-
349
- if (value.id) {
350
- return value.id.toString();
338
+
339
+ try {
340
+ const relatedModel = await this.modelMetadataRepo.findOne({
341
+ where: { singularName: field.relationCoModelSingularName || field.relation },
342
+ relations: { userKeyField: true }
343
+ });
344
+
345
+ if (relatedModel && relatedModel.userKeyField) {
346
+ const userKeyFieldName = relatedModel.userKeyField.name;
347
+ return value[userKeyFieldName] ? value[userKeyFieldName].toString() : '';
348
+ }
349
+
350
+ if (value.id) {
351
+ return value.id.toString();
352
+ }
353
+ } catch (error) {
354
+ console.error('Error fetching related model metadata:', error);
355
+ return value.id ? value.id.toString() : '';
351
356
  }
352
- } catch (error) {
353
- console.error('Error fetching related model metadata:', error);
354
- return value.id ? value.id.toString() : '';
355
- }
356
357
  }
357
358
 
358
359
  if (field.relationType === 'many-to-many') {
@@ -501,11 +502,9 @@ export class ChatterMessageService extends CRUDService<ChatterMessage> {
501
502
  return populatedEntity;
502
503
  }
503
504
 
504
- async getChatterMessages(
505
- entityId: number,
506
- entityName: string,
507
- query: any
508
- ) {
505
+ // [2026-02-05T23:31:21.025Z] INFO: [200 OK]
506
+ // GET /api/chatter-message/getChatterMessages/216/mswipeBoomboxBulkUpload?populateMedia[0]=messageAttachments&populate[0]=user&populate[1]=chatterMessageDetails&limit=25 22747ms
507
+ async getChatterMessages(entityId: number, entityName: string, query: any) {
509
508
  const { limit = 25, offset = 0, populate = [], populateMedia = [], filters } = query;
510
509
 
511
510
  const model = await this.modelMetadataRepo.findOne({
@@ -524,6 +523,9 @@ export class ChatterMessageService extends CRUDService<ChatterMessage> {
524
523
  const relatedEntitiesMap = new Map<string, number[]>();
525
524
 
526
525
  for (const field of oneToManyFields) {
526
+ if (field.enableAuditTracking === false) {
527
+ continue
528
+ }
527
529
  const coModelName = field.relationCoModelSingularName;
528
530
  const coModelFieldName = field.relationCoModelFieldName;
529
531
 
@@ -539,7 +541,11 @@ export class ChatterMessageService extends CRUDService<ChatterMessage> {
539
541
  const relatedEntityRepository = em.getRepository(classify(coModelName));
540
542
 
541
543
  const relatedEntities = await relatedEntityRepository.find({
542
- where: { [coModelFieldName]: { id: entityId } }
544
+ select: {
545
+ id: true,
546
+ },
547
+ where: { [coModelFieldName]: { id: entityId } },
548
+ take: 5,
543
549
  });
544
550
 
545
551
  const relatedIds = relatedEntities.map((entity: any) => entity.id);
@@ -164,7 +164,7 @@ export class CrudHelperService {
164
164
  }
165
165
 
166
166
  normalize(value: string | string[]): string[] {
167
- if (!value) return [];// if the value is nullish, then return an empty array
167
+ if (!value) return []; // if the value is nullish, then return an empty array
168
168
  return Array.isArray(value) ? value : [value]; // if the value is an array, return it as is, otherwise return it as an array
169
169
  }
170
170
 
@@ -876,7 +876,8 @@ export class ModelMetadataService {
876
876
  sequenceNumber: 1,
877
877
  actionUserKey: actionName,
878
878
  moduleUserKey: `${model.module.name}`,
879
- parentMenuItemUserKey: ""
879
+ parentMenuItemUserKey: "",
880
+ iconName : ""
880
881
  };
881
882
 
882
883
  const modelListview = {
@@ -181,7 +181,8 @@ export class ModuleMetadataService {
181
181
  sequenceNumber: 1,
182
182
  actionUserKey: `${module?.name}-home-action`,
183
183
  moduleUserKey: module?.name,
184
- parentMenuItemUserKey: ""
184
+ parentMenuItemUserKey: "",
185
+ iconName : "home"
185
186
  }
186
187
  ],
187
188
  views: [],
@@ -18,7 +18,7 @@ export abstract class DatabasePublisher<T> implements QueuePublisher<T> {
18
18
  if (!this.serviceRole) {
19
19
  this.logger.debug('Queue service Role is not defined in the environment variables');
20
20
  }
21
- this.logger.debug(`DatabasePublisher instance created with options: ${JSON.stringify(this.options())}`);
21
+ // this.logger.debug(`DatabasePublisher instance created with options: ${JSON.stringify(this.options())}`);
22
22
  }
23
23
 
24
24
  abstract options(): QueuesModuleOptions;
@@ -20,7 +20,7 @@ export abstract class DatabaseSubscriber<T> implements OnModuleInit, QueueSubscr
20
20
  if (!this.serviceRole) {
21
21
  this.logger.debug('Queue service Role is not defined in the environment variables');
22
22
  }
23
- this.logger.debug(`DatabaseSubscriber instance created with options: ${JSON.stringify(this.options())}`);
23
+ // this.logger.debug(`DatabaseSubscriber instance created with options: ${JSON.stringify(this.options())}`);
24
24
  }
25
25
 
26
26
  abstract subscribe(message: QueueMessage<T>);
@@ -72,42 +72,31 @@ export abstract class DatabaseSubscriber<T> implements OnModuleInit, QueueSubscr
72
72
  // this.logger.debug(`#### DatabaseSubscriber finished processing message from queue: ${queueName}`);
73
73
  }
74
74
 
75
- // async onModuleInit(): Promise<void> {
76
- // // we will start subscriber only if the current service role is subscriber.
77
- // if (['both', 'subscriber'].includes(this.serviceRole)) {
78
-
79
- // const options = this.options();
80
-
81
- // const queueName = options.queueName;
82
- // // setInterval(() => this.processNext(queueName), 1000);
83
- // const poll = async () => {
84
- // try {
85
- // await this.processNext(queueName);
86
- // } catch (err) {
87
- // this.logger.error(`Polling error: ${err.message}`);
88
- // } finally {
89
- // setTimeout(poll, 1000); // Wait 1s *after* processing finishes
90
- // }
91
- // };
92
-
93
- // // start the loop
94
- // poll();
95
-
96
- // this.logger.log(`DatabaseSubscriber ready to consume messages: ${JSON.stringify(this.options())}`);
97
- // }
98
- // }
99
-
100
75
  async onModuleInit(): Promise<void> {
76
+ // Not using SettingService here as that will necessitate all implementors of DatabaseSubscriber to also inject SettingService which is not ideal.
77
+ // Instead we directly read the environment variables here.
101
78
  const defaultBroker = process.env.QUEUES_DEFAULT_BROKER || 'database';
102
79
  const solidCliRunning = process.env.SOLID_CLI_RUNNING || "false";
80
+ const queueNameRegex = (process.env.QUEUES_QUEUE_NAME_REGEX_TO_ENABLE || '').trim();
103
81
 
104
82
  // we will start subscriber only if the current service role is subscriber.
105
83
  if (['both', 'subscriber'].includes(this.serviceRole) && defaultBroker === 'database' && solidCliRunning === "false") {
106
-
107
84
  const options = this.options();
108
-
109
85
  const queueName = options.queueName;
110
86
 
87
+ if (queueNameRegex && queueNameRegex !== "all") {
88
+ try {
89
+ const regex = new RegExp(queueNameRegex);
90
+ if (!regex.test(queueName)) {
91
+ this.logger.log(`DatabaseSubscriber for queue ${queueName} is disabled because it does not match QUEUES_QUEUE_NAME_REGEX_TO_ENABLE=${queueNameRegex}`);
92
+ return;
93
+ }
94
+ } catch (error) {
95
+ this.logger.error(`Invalid QUEUES_QUEUE_NAME_REGEX_TO_ENABLE regex "${queueNameRegex}". Subscriber for queue ${queueName} will not start.`);
96
+ return;
97
+ }
98
+ }
99
+
111
100
  this.poller.start(queueName, (q) => this.processNext(q), {
112
101
  baseDelayMs: 1000,
113
102
  maxDelayMs: 30_000,
@@ -10,11 +10,14 @@ export abstract class RabbitMqSubscriber<T> implements OnModuleInit, QueueSubscr
10
10
  private readonly logger = new Logger(RabbitMqSubscriber.name);
11
11
  private readonly url: string;
12
12
  private readonly serviceRole: string;
13
-
14
- constructor(
15
- protected readonly mqMessageService: MqMessageService,
16
- protected readonly mqMessageQueueService: MqMessageQueueService,
17
- ) {
13
+ private connection: amqp.Connection | null = null;
14
+ private channel: amqp.Channel | null = null;
15
+ private consumerTag: string | null = null;
16
+ private reconnectPromise: Promise<void> | null = null;
17
+ private reconnectAttempt = 0;
18
+ private stopping = false;
19
+
20
+ constructor(protected readonly mqMessageService: MqMessageService, protected readonly mqMessageQueueService: MqMessageQueueService) {
18
21
  this.url = process.env.QUEUES_RABBIT_MQ_URL;
19
22
  this.serviceRole = process.env.QUEUES_SERVICE_ROLE;
20
23
  if (!this.url) {
@@ -46,100 +49,266 @@ export abstract class RabbitMqSubscriber<T> implements OnModuleInit, QueueSubscr
46
49
  username: url.username,
47
50
  password: decodeURIComponent(url.password),
48
51
  frameMax: 131072,
52
+ heartbeat: 30,
49
53
  });
50
54
 
51
55
  return connection
52
56
  }
53
57
 
54
58
  async onModuleInit(): Promise<void> {
59
+ // Not using SettingService here as that will necessitate all implementors of RabbitMqSubscriber to also inject SettingService which is not ideal.
60
+ // Instead we directly read the environment variables here.
61
+ const defaultBroker = process.env.QUEUES_DEFAULT_BROKER || 'rabbitmq';
55
62
  const solidCliRunning = process.env.SOLID_CLI_RUNNING || "false";
63
+ const queueNameRegex = (process.env.QUEUES_QUEUE_NAME_REGEX_TO_ENABLE || '').trim();
56
64
 
57
65
  // we will start subscriber only if the current service role is subscriber.
58
- if (this.url && ['both', 'subscriber'].includes(this.serviceRole) && solidCliRunning === "false") {
66
+ if (this.url && ['both', 'subscriber'].includes(this.serviceRole) && solidCliRunning === "false" && defaultBroker === 'rabbitmq') {
67
+ const options = this.options();
68
+ const queueName = options.queueName;
59
69
 
60
- // this.logger.debug(`RabbitMqSubscriber instance created with options: ${JSON.stringify(this.options())} and url: ${this.url}`);
61
- // const connection = await amqp.connect(this.url);
70
+ if (queueNameRegex && queueNameRegex !== "all") {
71
+ try {
72
+ const regex = new RegExp(queueNameRegex);
73
+ if (!regex.test(queueName)) {
74
+ this.logger.log(`RabbitMqSubscriber for queue ${queueName} is disabled because it does not match QUEUES_QUEUE_NAME_REGEX_TO_ENABLE=${queueNameRegex}`);
75
+ return;
76
+ }
77
+ } catch (error) {
78
+ this.logger.error(`Invalid QUEUES_QUEUE_NAME_REGEX_TO_ENABLE regex "${queueNameRegex}". Subscriber for queue ${queueName} will not start.`);
79
+ return;
80
+ }
81
+ }
62
82
 
63
- let connection;
64
83
  try {
65
- connection = await this.establishConnection();
66
- // this.logger.debug(`RabbitMqSubscriber connection established: ${JSON.stringify(this.options())} and url: ${this.url}`);
84
+ await this.connectAndConsume(queueName);
85
+ } catch (err) {
86
+ this.logger.error(`Failed to connect to RabbitMQ for queue ${queueName}: ${(err as Error).message}`, (err as Error).stack);
87
+ this.triggerReconnect(queueName, 'initial connection failure');
67
88
  }
68
- catch (err) {
69
- this.logger.error(`Failed to connect to RabbitMQ: ${(err as Error).message}`, (err as Error).stack);
70
- throw err;
89
+
90
+ this.logger.log(`RabbitMqSubscriber ready to consume messages: ${JSON.stringify(this.options())} and url: ${this.url}`);
91
+ }
92
+ }
93
+
94
+ private async connectAndConsume(queueName: string): Promise<void> {
95
+ await this.cleanup();
96
+
97
+ let connection: amqp.Connection;
98
+ try {
99
+ connection = await this.establishConnection();
100
+ } catch (err) {
101
+ this.logger.error(`Failed to connect to RabbitMQ for queue ${queueName}: ${(err as Error).message}`, (err as Error).stack);
102
+ throw err;
103
+ }
104
+
105
+ this.connection = connection;
106
+
107
+ connection.on('error', (err) => {
108
+ if (connection !== this.connection) return;
109
+ this.logger.error(`RabbitMqSubscriber connection error for queue ${queueName}: ${(err as Error).message}`);
110
+ });
111
+
112
+ connection.on('close', () => {
113
+ if (connection !== this.connection) return;
114
+ this.logger.warn(`RabbitMqSubscriber connection closed for queue ${queueName}`);
115
+ this.triggerReconnect(queueName, 'connection closed');
116
+ });
117
+
118
+ const channel = await connection.createChannel();
119
+ this.channel = channel;
120
+
121
+ channel.on('error', (err) => {
122
+ if (channel !== this.channel) return;
123
+ this.logger.error(`RabbitMqSubscriber channel error for queue ${queueName}: ${(err as Error).message}`);
124
+ });
125
+
126
+ channel.on('close', () => {
127
+ if (channel !== this.channel) return;
128
+ this.logger.warn(`RabbitMqSubscriber channel closed for queue ${queueName}`);
129
+ this.triggerReconnect(queueName, 'channel closed');
130
+ });
131
+
132
+ // Process one message at a time per consumer to avoid parallel work on the same subscriber instance.
133
+ await channel.prefetch(1);
134
+
135
+ // Use a direct exchange with a stable routing key so retry DLX can route back to the main queue.
136
+ const exchangeName = `${queueName}.exchange`;
137
+ const routingKey = `${queueName}.routing-key`;
138
+ const retryQueue = `${queueName}.retry`;
139
+ const failedQueue = `${queueName}.failed`;
140
+
141
+ await channel.assertExchange(exchangeName, 'direct', {});
142
+ await channel.assertQueue(queueName, {});
143
+ await channel.bindQueue(queueName, exchangeName, routingKey);
144
+
145
+ // Retry queue uses DLX to route expired messages back to the main exchange/routing key.
146
+ await channel.assertQueue(retryQueue, {
147
+ arguments: {
148
+ 'x-dead-letter-exchange': exchangeName,
149
+ 'x-dead-letter-routing-key': routingKey,
71
150
  }
151
+ });
72
152
 
73
- const channel = await connection.createChannel();
74
- // this.logger.debug(`RabbitMqSubscriber channel created: ${JSON.stringify(this.options())} and url: ${url}`);
153
+ await channel.assertQueue(failedQueue, {});
75
154
 
76
- const options = this.options();
155
+ const consumeResult = await channel.consume(
156
+ queueName,
157
+ async (rawMessage) => {
158
+ if (!rawMessage) {
159
+ return;
160
+ }
77
161
 
78
- const queueName = options.queueName;
79
- const exchangeName = `${queueName}.exchange`;
80
- const routingKey = `${queueName}.routing-key`;
81
-
82
- await channel.assertExchange(exchangeName, 'direct', {});
83
- // this.logger.debug(`RabbitMqSubscriber channel asserted: ${JSON.stringify(this.options())} and url: ${url}`);
84
-
85
- const queue = await channel.assertQueue(queueName, {});
86
- // this.logger.debug(`RabbitMqSubscriber queue asserted: ${JSON.stringify(this.options())} and url: ${url}`);
87
-
88
- await channel.bindQueue(queue.queue, exchangeName, routingKey);
89
- // this.logger.debug(`RabbitMqSubscriber queue bound: ${JSON.stringify(this.options())} and url: ${url}`);
90
-
91
- // Consume messages from the queue
92
- channel.consume(
93
- queue.queue,
94
- async (rawMessage) => {
95
- if (rawMessage) {
96
- const messageContentString = rawMessage.content.toString();
97
- // this.logger.debug(`RabbitMqSubscriber Received raw message: ${messageContentString}`);
98
-
99
- let message: QueueMessage<T> = null;
100
-
101
- try {
102
- message = JSON.parse(messageContentString) as QueueMessage<T>;
103
-
104
- // this is the first time we are receiving the message so we set the currentRetry to 0
105
- if (!message.retryCount) message.retryCount = 0;
106
- if (!message.retryInterval) message.retryInterval = 1000;
107
- if (!message.currentRetry) message.currentRetry = 0;
108
-
109
- await this.processMessage(message, rawMessage, channel);
110
- }
111
- catch (error) {
112
- this.logger.error(`Error processing message: ${error.message}`);
113
-
114
- // if an error occurs then if retryCount is set we start retrying.
115
- if (message) {
116
- if (message.currentRetry < message.retryCount) {
117
- await this.updateStatusInDatabase('retrying', message);
118
-
119
- message.currentRetry++;
120
- this.logger.warn(`Retrying message (${message.currentRetry}/${message.retryCount}) after ${message.retryInterval}ms`);
121
- setTimeout(() => {
122
- this.retryMessage(message, rawMessage, channel);
123
- }, message.retryInterval);
124
- } else {
125
- await this.updateStatusInDatabase('failed', message, error.message, '');
126
-
127
- this.logger.error(`Message failed after ${message.retryCount} attempts: ${error.message}`);
128
- channel.ack(rawMessage); // Discard the message after max retries
129
- }
130
- }
131
-
132
- }
133
- }
134
- },
135
- // { noAck: true },
136
- {},
137
- );
162
+ const messageContentString = rawMessage.content.toString();
163
+ let message: QueueMessage<T> = null;
138
164
 
139
- this.logger.log(`RabbitMqSubscriber ready to consume messages: ${JSON.stringify(this.options())} and url: ${this.url}`);
165
+ try {
166
+ message = JSON.parse(messageContentString) as QueueMessage<T>;
167
+ } catch (error) {
168
+ this.logger.error(`Invalid JSON message on queue ${queueName}: ${(error as Error).message}`);
169
+ await this.publishToFailedQueue(queueName, rawMessage.content, channel, error);
170
+ channel.ack(rawMessage);
171
+ return;
172
+ }
173
+
174
+ if (!message.retryCount) message.retryCount = 0;
175
+ if (!message.retryInterval) message.retryInterval = 1000;
176
+ if (!message.currentRetry) message.currentRetry = 0;
177
+
178
+ try {
179
+ await this.processMessage(message, rawMessage, channel);
180
+ } catch (error) {
181
+ await this.handleProcessingError(message, rawMessage, channel, error, queueName);
182
+ }
183
+ },
184
+ // Explicit ack enables reliable processing and retry routing.
185
+ { noAck: false },
186
+ );
187
+
188
+ this.consumerTag = consumeResult.consumerTag;
189
+ }
190
+
191
+ // Retry flow: update DB -> increment retry -> send to retry queue with per-message expiration -> ack original.
192
+ private async handleProcessingError(message: QueueMessage<T>, rawMessage: amqp.ConsumeMessage, channel: amqp.Channel, error: any, queueName: string): Promise<void> {
193
+ const errorMessage = (error as Error)?.message || String(error);
194
+ this.logger.error(`Error processing message on queue ${queueName}: ${errorMessage}`);
195
+
196
+ if (message.currentRetry < message.retryCount) {
197
+ await this.updateStatusInDatabase('retrying', message);
198
+
199
+ message.currentRetry++;
200
+ const retryQueue = `${queueName}.retry`;
201
+ const payload = Buffer.from(JSON.stringify(message));
202
+
203
+ // Per-message expiration keeps the message in the retry queue until TTL, then DLX routes it back.
204
+ channel.sendToQueue(retryQueue, payload, {
205
+ expiration: String(message.retryInterval || 1000),
206
+ headers: {
207
+ 'x-error': errorMessage,
208
+ }
209
+ });
210
+
211
+ channel.ack(rawMessage);
212
+ this.logger.warn(`Retrying message (${message.currentRetry}/${message.retryCount}) after ${message.retryInterval}ms on queue ${queueName}`);
213
+ return;
214
+ }
215
+
216
+ await this.updateStatusInDatabase('failed', message, errorMessage, '');
217
+ channel.ack(rawMessage);
218
+ await this.publishToFailedQueue(queueName, Buffer.from(JSON.stringify(message)), channel, error);
219
+ this.logger.error(`Message failed after ${message.retryCount} attempts on queue ${queueName}: ${errorMessage}`);
220
+ }
221
+
222
+ private async publishToFailedQueue(queueName: string, payload: Buffer | string, channel: amqp.Channel, error?: any): Promise<void> {
223
+ const failedQueue = `${queueName}.failed`;
224
+ const body = Buffer.isBuffer(payload) ? payload : Buffer.from(payload);
225
+ const errorMessage = (error as Error)?.message || String(error || '');
226
+
227
+ try {
228
+ channel.sendToQueue(failedQueue, body, errorMessage ? {
229
+ headers: { 'x-error': errorMessage }
230
+ } : undefined);
231
+ } catch (err) {
232
+ this.logger.error(`Failed to publish to failed queue ${failedQueue}: ${(err as Error).message}`);
233
+ }
234
+ }
235
+
236
+ private triggerReconnect(queueName: string, reason: string) {
237
+ if (this.stopping) return;
238
+ if (this.reconnectPromise) return;
239
+
240
+ this.reconnectPromise = this.reconnectLoop(queueName, reason)
241
+ .finally(() => {
242
+ this.reconnectPromise = null;
243
+ });
244
+ }
245
+
246
+ // Reconnect with backoff to avoid hammering the broker during outages.
247
+ private async reconnectLoop(queueName: string, reason: string): Promise<void> {
248
+ this.logger.warn(`RabbitMqSubscriber reconnecting for queue ${queueName}: ${reason}`);
249
+
250
+ while (!this.stopping) {
251
+ try {
252
+ await this.connectAndConsume(queueName);
253
+ this.reconnectAttempt = 0;
254
+ this.logger.log(`RabbitMqSubscriber reconnected for queue ${queueName}`);
255
+ return;
256
+ } catch (err) {
257
+ this.reconnectAttempt += 1;
258
+ const delay = this.backoff();
259
+ this.logger.warn(`RabbitMqSubscriber reconnect failed for queue ${queueName}; retrying in ${delay}ms`);
260
+ await this.sleep(delay);
261
+ }
140
262
  }
141
263
  }
142
264
 
265
+ private async cleanup(): Promise<void> {
266
+ const channel = this.channel;
267
+ const connection = this.connection;
268
+ const consumerTag = this.consumerTag;
269
+
270
+ this.channel = null;
271
+ this.connection = null;
272
+ this.consumerTag = null;
273
+
274
+ if (channel) {
275
+ try {
276
+ if (consumerTag) {
277
+ await channel.cancel(consumerTag);
278
+ }
279
+ } catch (_) {
280
+ // ignore
281
+ }
282
+
283
+ try {
284
+ await channel.close();
285
+ } catch (_) {
286
+ // ignore
287
+ }
288
+ }
289
+
290
+ if (connection) {
291
+ try {
292
+ await connection.close();
293
+ } catch (_) {
294
+ // ignore
295
+ }
296
+ }
297
+ }
298
+
299
+ private sleep(ms: number): Promise<void> {
300
+ return new Promise(resolve => setTimeout(resolve, ms));
301
+ }
302
+
303
+ // Exponential backoff with jitter, capped to 30s.
304
+ private backoff(): number {
305
+ const baseMs = 1000;
306
+ const maxMs = 30_000;
307
+ const exp = Math.min(maxMs, baseMs * Math.pow(2, this.reconnectAttempt));
308
+ const jitter = Math.floor(Math.random() * (exp * 0.2));
309
+ return Math.min(maxMs, exp + jitter);
310
+ }
311
+
143
312
  /**
144
313
  * Abstract method for message processing logic.
145
314
  */
@@ -152,43 +321,14 @@ export abstract class RabbitMqSubscriber<T> implements OnModuleInit, QueueSubscr
152
321
  // Ack the message.
153
322
  channel.ack(rawMessage);
154
323
 
155
- // TODO: Update the database to indicate that the task is finished.
324
+ // Persist success output and timing.
156
325
  await this.updateStatusInDatabase('succeeded', message, '', result ? JSON.stringify(result, null, 2) : '');
157
326
 
158
327
  }
159
328
 
160
- /**
161
- * Retry the message by invoking the processing logic again.
162
- */
163
- private async retryMessage(message: QueueMessage<T>, rawMessage, channel) {
164
- try {
165
- await this.processMessage(message, rawMessage, channel);
166
- } catch (error) {
167
- if (message.currentRetry < message.retryCount) {
168
- await this.updateStatusInDatabase('retrying', message);
169
-
170
- message.currentRetry++;
171
- this.logger.warn(`Retrying message (${message.currentRetry}/${message.retryCount}) after ${message.retryInterval}ms: ${error.message}`);
172
- setTimeout(() => {
173
- this.retryMessage(message, rawMessage, channel);
174
- }, message.retryInterval);
175
- } else {
176
-
177
- this.logger.error(`Message failed after ${message.retryCount} attempts: ${error.message}`);
178
-
179
- // Discard the message after max retries
180
- channel.ack(rawMessage);
181
-
182
- // TODO: Store the error in the database and update the status accordingly.
183
- await this.updateStatusInDatabase('failed', message, error.message, '');
184
-
185
- }
186
- }
187
- }
188
-
189
329
  private async updateStatusInDatabase(stage: string, message: QueueMessage<T>, error: string = '', result: string = '') {
190
330
 
191
- // TODO: make an entry in the relevant database table, generate a unique id earlier.
331
+ // Update the existing message record by messageId; creation happens upstream.
192
332
  try {
193
333
  // 1. resolve the queue first
194
334
  const mqMessage = await this.mqMessageService.repo.findOne({
@@ -220,4 +360,4 @@ export abstract class RabbitMqSubscriber<T> implements OnModuleInit, QueueSubscr
220
360
 
221
361
  }
222
362
 
223
- }
363
+ }