@solidxai/core 0.1.7 → 0.1.8-beta.0

package/src/jobs/redis/twilio-sms-publisher-redis.service.ts

@@ -0,0 +1,23 @@
+import { Injectable } from '@nestjs/common';
+
+import { RedisPublisher } from 'src/services/queues/redis-publisher.service';
+import twilioSmsQueueConfig from './twilio-sms-queue-options-redis';
+import { MqMessageQueueService } from '../../services/mq-message-queue.service';
+import { MqMessageService } from '../../services/mq-message.service';
+import { QueuesModuleOptions } from "../../interfaces";
+
+@Injectable()
+export class TwilioSmsQueuePublisherRedis extends RedisPublisher<any> {
+    constructor(
+        protected readonly mqMessageService: MqMessageService,
+        protected readonly mqMessageQueueService: MqMessageQueueService,
+    ) {
+        super(mqMessageService, mqMessageQueueService);
+    }
+
+    options(): QueuesModuleOptions {
+        return {
+            ...twilioSmsQueueConfig
+        }
+    }
+}

package/src/jobs/redis/twilio-sms-queue-options-redis.ts

@@ -0,0 +1,9 @@
+import { BrokerType } from "../../interfaces";
+
+const QUEUE_NAME = 'solid_twilio_sms_queue_redis';
+
+export default {
+    name: QUEUE_NAME,
+    type: BrokerType.Redis,
+    queueName: QUEUE_NAME,
+};

package/src/jobs/redis/twilio-sms-subscriber-redis.service.ts

@@ -0,0 +1,34 @@
+import { Injectable } from '@nestjs/common';
+
+import { RedisSubscriber } from 'src/services/queues/redis-subscriber.service';
+import { QueueMessage } from 'src/interfaces/mq';
+import twilioSmsQueueConfig from './twilio-sms-queue-options-redis';
+import { MqMessageService } from '../../services/mq-message.service';
+import { MqMessageQueueService } from '../../services/mq-message-queue.service';
+import { QueuesModuleOptions } from "../../interfaces";
+import { TwilioSMSService } from '../../services/sms/TwilioSMSService';
+import { SmsFactory } from 'src/factories/sms.factory';
+import { PollerService } from '../../services/poller.service';
+
+@Injectable()
+export class TwilioSmsQueueSubscriberRedis extends RedisSubscriber<any> {
+    constructor(
+        private readonly smsFactory: SmsFactory,
+        readonly mqMessageService: MqMessageService,
+        readonly mqMessageQueueService: MqMessageQueueService,
+        readonly poller: PollerService,
+    ) {
+        super(mqMessageService, mqMessageQueueService);
+    }
+
+    options(): QueuesModuleOptions {
+        return {
+            ...twilioSmsQueueConfig
+        }
+    }
+
+    subscribe(message: QueueMessage<any>) {
+        const smsService: TwilioSMSService = this.smsFactory.getSmsService(TwilioSMSService.name) as TwilioSMSService;
+        return smsService.sendSMSSynchronously(message);
+    }
+}

package/src/seeders/seed-data/solid-core-metadata.json

@@ -7239,35 +7239,28 @@
           {
             "type": "field",
             "attrs": {
-              "name": "fieldName",
-              "isSearchable": true
-            }
-          },
-          {
-            "type": "field",
-            "attrs": {
-              "name": "oldValueDisplay",
+              "name": "fieldDisplayName",
               "isSearchable": true
             }
           },
           {
             "type": "field",
             "attrs": {
-              "name": "newValueDisplay",
+              "name": "fieldName",
               "isSearchable": true
             }
           },
           {
             "type": "field",
             "attrs": {
-              "name": "oldValue",
+              "name": "oldValueDisplay",
               "isSearchable": true
             }
           },
           {
             "type": "field",
             "attrs": {
-              "name": "newValue",
+              "name": "newValueDisplay",
               "isSearchable": true
             }
           }
@@ -7309,36 +7302,43 @@
                   {
                     "type": "field",
                     "attrs": {
-                      "name": "fieldName",
-                      "isSearchable": true
+                      "name": "fieldDisplayName"
                     }
                   },
                   {
                     "type": "field",
                     "attrs": {
-                      "name": "oldValueDisplay",
-                      "isSearchable": true
+                      "name": "fieldName"
                     }
                   },
                   {
                     "type": "field",
                     "attrs": {
-                      "name": "newValueDisplay",
-                      "isSearchable": true
+                      "name": "oldValueDisplay"
                     }
                   },
                   {
                     "type": "field",
                     "attrs": {
-                      "name": "oldValue",
-                      "isSearchable": true
+                      "name": "newValueDisplay"
                     }
                   },
                   {
                     "type": "field",
                     "attrs": {
-                      "name": "newValue",
-                      "isSearchable": true
+                      "name": "oldValue"
+                    }
+                  },
+                  {
+                    "type": "field",
+                    "attrs": {
+                      "name": "newValue"
+                    }
+                  },
+                  {
+                    "type": "field",
+                    "attrs": {
+                      "name": "chatterMessage"
                     }
                   }
                 ]
@@ -7387,16 +7387,22 @@
           {
             "type": "field",
             "attrs": {
-              "name": "coModelEntityId",
+              "name": "modelDisplayName",
               "isSearchable": true
             }
           },
           {
             "type": "field",
             "attrs": {
-              "name": "coModelName",
+              "name": "coModelEntityId",
               "isSearchable": true
             }
+          },
+          {
+            "type": "field",
+            "attrs": {
+              "name": "user"
+            }
           }
         ]
       }
@@ -7436,29 +7442,49 @@
                   {
                     "type": "field",
                     "attrs": {
-                      "name": "messageType",
-                      "isSearchable": true
+                      "name": "messageType"
                     }
                   },
                   {
                     "type": "field",
                     "attrs": {
-                      "name": "messageSubType",
-                      "isSearchable": true
+                      "name": "messageSubType"
                     }
                   },
                   {
                     "type": "field",
                     "attrs": {
-                      "name": "coModelEntityId",
-                      "isSearchable": true
+                      "name": "modelDisplayName"
                     }
                   },
                   {
                     "type": "field",
                     "attrs": {
-                      "name": "coModelName",
-                      "isSearchable": true
+                      "name": "coModelEntityId"
+                    }
+                  },
+                  {
+                    "type": "field",
+                    "attrs": {
+                      "name": "modelUserKey"
+                    }
+                  },
+                  {
+                    "type": "field",
+                    "attrs": {
+                      "name": "user"
+                    }
+                  },
+                  {
+                    "type": "field",
+                    "attrs": {
+                      "name": "messageBody"
+                    }
+                  },
+                  {
+                    "type": "field",
+                    "attrs": {
+                      "name": "messageAttachments"
                     }
                   },
                   {

package/src/services/queues/redis-publisher.service.ts

@@ -0,0 +1,94 @@
+import { Logger, OnModuleDestroy } from '@nestjs/common';
+import Redis from 'ioredis';
+import { v4 as uuidv4 } from 'uuid';
+import { QueuesModuleOptions } from '../../interfaces';
+import { QueueMessage, QueuePublisher } from '../../interfaces/mq';
+import { MqMessageQueueService } from '../mq-message-queue.service';
+import { MqMessageService } from '../mq-message.service';
+import { buildNamespacedQueueName } from './common';
+
+export abstract class RedisPublisher<T> implements OnModuleDestroy, QueuePublisher<T> {
+    private readonly logger = new Logger(RedisPublisher.name);
+    private readonly serviceRole: string;
+    private client: Redis | null = null;
+
+    constructor(
+        protected readonly mqMessageService: MqMessageService,
+        protected readonly mqMessageQueueService: MqMessageQueueService,
+    ) {
+        this.serviceRole = process.env.QUEUES_SERVICE_ROLE;
+        if (!process.env.QUEUES_REDIS_URL) {
+            this.logger.debug('RedisPublisher: QUEUES_REDIS_URL is not defined in the environment variables');
+        }
+    }
+
+    abstract options(): QueuesModuleOptions;
+
+    private getClient(): Redis {
+        if (!this.client) {
+            this.client = new Redis(process.env.QUEUES_REDIS_URL);
+            this.client.on('error', (err) => {
+                this.logger.error(`RedisPublisher connection error: ${err.message}`, err.stack);
+            });
+        }
+        return this.client;
+    }
+
+    async onModuleDestroy(): Promise<void> {
+        if (this.client) {
+            await this.client.quit();
+            this.client = null;
+        }
+    }
+
+    async publish(message: QueueMessage<T>): Promise<string> {
+        if (!process.env.QUEUES_REDIS_URL) {
+            throw new Error('RedisPublisher: QUEUES_REDIS_URL is not defined in the environment variables');
+        }
+        if (!this.serviceRole) {
+            throw new Error('RedisPublisher: Queue service Role is not defined in the environment variables');
+        }
+        if (this.serviceRole === 'subscriber') {
+            throw new Error('RedisPublisher: Queue service Role is subscriber, cannot publish messages');
+        }
+
+        if (!message.retryCount) message.retryCount = 0;
+        if (!message.retryInterval) message.retryInterval = 1000;
+        message.messageId = uuidv4();
+
+        const queueName = this.options().queueName;
+        const namespacedQueueName = buildNamespacedQueueName(queueName);
+
+        await this.persistToDatabase(namespacedQueueName, message);
+
+        try {
+            const client = this.getClient();
+            await client.publish(namespacedQueueName, JSON.stringify(message));
+            this.logger.debug(`RedisPublisher published message ${message.messageId} to channel ${namespacedQueueName}`);
+        } catch (err) {
+            this.logger.error(`RedisPublisher failed to publish message: ${(err as Error).message}`, (err as Error).stack);
+        }
+
+        return message.messageId;
+    }
+
+    private async persistToDatabase(queueName: string, message: QueueMessage<T>): Promise<void> {
+        try {
+            const mqMessageQueue = await this.mqMessageQueueService.resolveQueue(queueName);
+            await this.mqMessageService.create({
+                messageBroker: this.options().type,
+                messageId: message.messageId,
+                retryCount: message.retryCount,
+                retryInterval: message.retryInterval,
+                stage: 'pending',
+                startedAt: new Date(),
+                input: JSON.stringify(message, null, 2),
+                parentEntityId: message.parentEntityId,
+                parentEntity: message.parentEntity,
+                mqMessageQueueId: mqMessageQueue.id,
+            });
+        } catch (error) {
+            this.logger.error(error.message, error.stack);
+        }
+    }
+}

package/src/services/queues/redis-subscriber.service.ts

@@ -0,0 +1,208 @@
+import { Logger, OnModuleDestroy, OnModuleInit } from '@nestjs/common';
+import Redis from 'ioredis';
+import { QueuesModuleOptions } from '../../interfaces';
+import { QueueMessage, QueueSubscriber } from '../../interfaces/mq';
+import { MqMessageQueueService } from '../mq-message-queue.service';
+import { MqMessageService } from '../mq-message.service';
+import { buildNamespacedQueueName } from './common';
+
+export abstract class RedisSubscriber<T> implements OnModuleInit, OnModuleDestroy, QueueSubscriber<T> {
+    private readonly logger = new Logger(RedisSubscriber.name);
+    private readonly serviceRole: string;
+    private client: Redis | null = null;
+    private reconnectAttempt = 0;
+    private stopping = false;
+
+    constructor(
+        protected readonly mqMessageService: MqMessageService,
+        protected readonly mqMessageQueueService: MqMessageQueueService,
+    ) {
+        this.serviceRole = process.env.QUEUES_SERVICE_ROLE;
+        if (!process.env.QUEUES_REDIS_URL) {
+            this.logger.debug('RedisSubscriber: QUEUES_REDIS_URL is not defined in the environment variables');
+        }
+    }
+
+    abstract subscribe(message: QueueMessage<T>): void;
+
+    abstract options(): QueuesModuleOptions;
+
+    async onModuleInit(): Promise<void> {
+        const defaultBroker = process.env.QUEUES_DEFAULT_BROKER || 'database';
+        const solidCliRunning = process.env.SOLID_CLI_RUNNING || 'false';
+        const queueNameRegex = (process.env.QUEUES_QUEUE_NAME_REGEX_TO_ENABLE || '').trim();
+
+        if (
+            process.env.QUEUES_REDIS_URL &&
+            ['both', 'subscriber'].includes(this.serviceRole) &&
+            solidCliRunning === 'false' &&
+            defaultBroker === 'redis'
+        ) {
+            const options = this.options();
+            const queueName = options.queueName;
+
+            if (queueNameRegex && queueNameRegex !== 'all') {
+                try {
+                    const regex = new RegExp(queueNameRegex);
+                    if (!regex.test(queueName)) {
+                        this.logger.log(
+                            `RedisSubscriber for queue ${queueName} is disabled because it does not match QUEUES_QUEUE_NAME_REGEX_TO_ENABLE=${queueNameRegex}`,
+                        );
+                        return;
+                    }
+                } catch (error) {
+                    this.logger.error(
+                        `Invalid QUEUES_QUEUE_NAME_REGEX_TO_ENABLE regex "${queueNameRegex}". Subscriber for queue ${queueName} will not start.`,
+                    );
+                    return;
+                }
+            }
+
+            const namespacedQueueName = buildNamespacedQueueName(queueName);
+            await this.connectAndSubscribe(namespacedQueueName);
+            this.logger.log(`RedisSubscriber ready to consume messages: ${JSON.stringify(options)}`);
+        }
+    }
+
+    async onModuleDestroy(): Promise<void> {
+        this.stopping = true;
+        await this.cleanup();
+    }
+
+    private async connectAndSubscribe(channel: string): Promise<void> {
+        await this.cleanup();
+
+        const client = new Redis(process.env.QUEUES_REDIS_URL);
+        this.client = client;
+
+        client.on('error', (err) => {
+            if (client !== this.client) return;
+            this.logger.error(`RedisSubscriber connection error for channel ${channel}: ${err.message}`, err.stack);
+        });
+
+        client.on('close', () => {
+            if (client !== this.client) return;
+            this.logger.warn(`RedisSubscriber connection closed for channel ${channel}`);
+            this.triggerReconnect(channel, 'connection closed');
+        });
+
+        client.on('message', async (receivedChannel, rawMessage) => {
+            if (receivedChannel !== channel) return;
+
+            let message: QueueMessage<T> = null;
+            try {
+                message = JSON.parse(rawMessage) as QueueMessage<T>;
+            } catch (error) {
+                this.logger.error(`RedisSubscriber invalid JSON on channel ${channel}: ${(error as Error).message}`);
+                return;
+            }
+
+            if (!message.retryCount) message.retryCount = 0;
+            if (!message.retryInterval) message.retryInterval = 1000;
+            if (!message.currentRetry) message.currentRetry = 0;
+
+            try {
+                await this.processMessage(message);
+            } catch (error) {
+                await this.handleProcessingError(message, error, channel);
+            }
+        });
+
+        await client.subscribe(channel);
+        this.reconnectAttempt = 0;
+    }
+
+    private async handleProcessingError(message: QueueMessage<T>, error: any, channel: string): Promise<void> {
+        const errorMessage = (error as Error)?.message || String(error);
+        this.logger.error(`Error processing message on channel ${channel}: ${errorMessage}`);
+
+        if (message.currentRetry < message.retryCount) {
+            await this.updateStatusInDatabase('retrying', message);
+            message.currentRetry++;
+            this.logger.warn(
+                `Retrying message (${message.currentRetry}/${message.retryCount}) after ${message.retryInterval}ms on channel ${channel}`,
+            );
+            setTimeout(async () => {
+                try {
+                    await this.processMessage(message);
+                } catch (retryError) {
+                    await this.handleProcessingError(message, retryError, channel);
+                }
+            }, message.retryInterval);
+        } else {
+            await this.updateStatusInDatabase('failed', message, errorMessage, '');
+            this.logger.error(`Message failed after ${message.retryCount} attempts on channel ${channel}: ${errorMessage}`);
+        }
+    }
+
+    private triggerReconnect(channel: string, reason: string): void {
+        if (this.stopping) return;
+
+        this.reconnectAttempt++;
+        const delay = this.backoff();
+        this.logger.warn(`RedisSubscriber reconnecting for channel ${channel} (${reason}) in ${delay}ms`);
+
+        setTimeout(async () => {
+            if (this.stopping) return;
+            try {
+                await this.connectAndSubscribe(channel);
+                this.logger.log(`RedisSubscriber reconnected for channel ${channel}`);
+            } catch (err) {
+                this.triggerReconnect(channel, `reconnect failed: ${(err as Error).message}`);
+            }
+        }, delay);
+    }
+
+    private async cleanup(): Promise<void> {
+        const client = this.client;
+        this.client = null;
+
+        if (client) {
+            try {
+                await client.quit();
+            } catch (_) {
+                // ignore
+            }
+        }
+    }
+
+    private backoff(): number {
+        const baseMs = 1000;
+        const maxMs = 30_000;
+        const exp = Math.min(maxMs, baseMs * Math.pow(2, this.reconnectAttempt));
+        const jitter = Math.floor(Math.random() * (exp * 0.2));
+        return Math.min(maxMs, exp + jitter);
+    }
+
+    protected async processMessage(message: QueueMessage<T>): Promise<void> {
+        await this.updateStatusInDatabase('started', message);
+        const result: any = await (this.subscribe(message) as any);
+        await this.updateStatusInDatabase('succeeded', message, '', result != null ? JSON.stringify(result, null, 2) : '');
+    }
+
+    private async updateStatusInDatabase(
+        stage: string,
+        message: QueueMessage<T>,
+        error: string = '',
+        result: string = '',
+    ): Promise<void> {
+        try {
+            const mqMessage = await this.mqMessageService.repo.findOne({
+                where: { messageId: message.messageId },
+            });
+
+            if (mqMessage) {
+                const updatedFields: Record<string, any> = { stage };
+                if (stage === 'failed' || stage === 'succeeded') {
+                    updatedFields['finishedAt'] = new Date();
+                    updatedFields['elapsedMillis'] = updatedFields['finishedAt'].getTime() - mqMessage.startedAt.getTime();
+                }
+                if (stage === 'succeeded') updatedFields['output'] = result;
+                if (stage === 'failed') updatedFields['error'] = error;
+                await this.mqMessageService.repo.update(mqMessage.id, updatedFields);
+            }
+        } catch (err) {
+            this.logger.error(err.message, err.stack);
+        }
+    }
+}