@solidstarters/solid-core 1.2.0

package/src/services/rabbitmq-publisher.service.ts

@@ -0,0 +1,121 @@
+import { v4 as uuidv4 } from 'uuid';
+import { Injectable, Logger } from '@nestjs/common';
+import * as amqp from 'amqplib';
+import { QueueMessage, QueuePublisher } from '../interfaces/mq';
+import { MqMessageQueueService } from './mq-message-queue.service';
+import { MqMessageService } from './mq-message.service';
+import { QueuesModuleOptions } from "../interfaces";
+
+@Injectable()
+export abstract class RabbitMqPublisher<T> implements QueuePublisher<T> { // TODO This can be made a generic type for better type visibility
+    private readonly logger = new Logger(RabbitMqPublisher.name);
+
+    constructor(
+        protected readonly mqMessageService: MqMessageService,
+        protected readonly mqMessageQueueService: MqMessageQueueService,
+    ) {
+        const url = process.env.QUEUES_RABBIT_MQ_URL;
+        this.logger.debug(`RabbitMqPublisher instance created with options: ${JSON.stringify(this.options())} and url: ${url}`);
+    }
+
+    abstract options(): QueuesModuleOptions;
+
+    async publish(message: QueueMessage<T>): Promise<string> {
+        const url = process.env.QUEUES_RABBIT_MQ_URL;
+        this.logger.debug(`RabbitMqPublisher publishing with options: ${JSON.stringify(this.options())} and url: ${url}`);
+
+        const connection = await amqp.connect(url);
+        // this.logger.debug(`RabbitMqPublisher publisher connected options: ${JSON.stringify(this.options())} and url: ${url}`);
+
+        const channel = await connection.createChannel();
+        // this.logger.debug(`RabbitMqPublisher publisher channel created options: ${JSON.stringify(this.options())} and url: ${url}`);
+
+        const options = this.options();
+
+        const queueName = options.queueName;
+        const exchangeName = `${queueName}.exchange`;
+        const routingKey = `${queueName}.routing-key`;
+
+        await channel.assertExchange(exchangeName, 'direct', {});
+        // this.logger.debug(`RabbitMqPublisher channel asserted: ${JSON.stringify(this.options())} and url: ${url}`);
+
+        const queue = await channel.assertQueue(queueName, {});
+        // this.logger.debug(`RabbitMqPublisher queue asserted: ${JSON.stringify(this.options())} and url: ${url}`);
+
+        await channel.bindQueue(queue.queue, exchangeName, routingKey);
+        // this.logger.debug(`RabbitMqPublisher queue bound: ${JSON.stringify(this.options())} and url: ${url}`);
+
+        // Set default values for retry. 
+        // by default there are no retries.
+        if (!message.retryCount) message.retryCount = 0;
+        if (!message.retryInterval) message.retryInterval = 1000;
+
+        // generate a new message id 
+        message.messageId = uuidv4();
+
+        // Save the message to the DB so that we can then change its status in the subscriber...
+        await this.persistToDatabase(queueName, message);
+
+        // wait for the channel to confirm 
+        try {
+            // Publish the message
+            // const publishStatus = channel.publish(exchangeName, routingKey, Buffer.from(JSON.stringify(message)));
+            const publishStatus = channel.publish(exchangeName, routingKey, Buffer.from(JSON.stringify(message)), { mandatory: true });
+            // this.logger.debug(`RabbitMqPublisher publish status: ${JSON.stringify(publishStatus)}`);
+            // if (!publishStatus) {
+            //     this.logger.warn('RabbitMqPublisher Message buffering failed!');
+            // }
+            // await channel.waitForConfirms();
+            // this.logger.debug('RabbitMqPublisher Message published successfully');
+        } catch (err) {
+            this.logger.error(`RabbitMqPublisher Message publish failed: ${JSON.stringify(err)}`);
+            if (err instanceof Error) {
+                this.logger.error(`RabbitMqPublisher Error stack: ${err.stack}`);
+            }
+        }
+        finally {
+            // TODO: check if we want to do this or keep the connection open all the time. 
+            // connection.close();
+            // setTimeout(() => {
+            //     connection.close();
+            //     this.logger.error('RabbitMqPublisher connection closed');
+            // }, 3000);
+        }
+        // this.logger.debug(`Sent message: ${JSON.stringify(message)}`);
+
+        // TODO: check if we want to do this or keep the connection open all the time. 
+        // setTimeout(() => {
+        //     connection.close();
+        // }, 500);
+
+        // return the newly created message id.
+        return message.messageId;
+    }
+
+    private async persistToDatabase(queueName: string, message: QueueMessage<T>) {
+
+        // TODO: make an entry in the relevant database table, generate a unique id earlier.
+        try {
+            // 1. resolve the queue first
+            const mqMessageQueue = await this.mqMessageQueueService.resolveQueue(queueName);
+
+            // 2. Next create an entry in the mqMessage table. 
+            await this.mqMessageService.create({
+                messageId: message.messageId,
+                retryCount: message.retryCount,
+                retryInterval: message.retryInterval,
+                stage: 'pending',
+                startedAt: new Date(),
+                input: JSON.stringify(message, null, 2),
+                parentEntityId: message.parentEntityId,
+                parentEntity: message.parentEntity,
+                mqMessageQueueId: mqMessageQueue.id,
+            });
+        }
+        catch (error) {
+            this.logger.error(error.message, error.stack);
+        }
+
+    }
+
+}

package/src/services/rabbitmq-subscriber.service.ts

@@ -0,0 +1,192 @@
+import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
+import * as amqp from 'amqplib';
+import { QueueMessage, QueueSubscriber } from '../interfaces/mq';
+import { MqMessageService } from './mq-message.service';
+import { MqMessageQueueService } from './mq-message-queue.service';
+import { QueuesModuleOptions } from "../interfaces";
+
+
+@Injectable()
+export abstract class RabbitMqSubscriber<T> implements OnModuleInit, QueueSubscriber<T> { // TODO This can be made a generic type for better type visibility
+    private readonly logger = new Logger(RabbitMqSubscriber.name);
+
+    constructor(
+        protected readonly mqMessageService: MqMessageService,
+        protected readonly mqMessageQueueService: MqMessageQueueService,
+    ) {
+    }
+
+    abstract subscribe(message: QueueMessage<T>);
+
+    abstract options(): QueuesModuleOptions;
+
+    // setReceiveHandler(receiveHandler: (message: any) => Promise<any>): void {
+    //     this.options.receive = receiveHandler; // Overwrite the default receive handler
+    // }
+
+    async onModuleInit(): Promise<void> {
+        // we will start subscriber only if the current service role is subscriber. 
+        if (['both', 'subscriber'].includes(process.env.QUEUES_SERVICE_ROLE)) {
+
+            const url = process.env.QUEUES_RABBIT_MQ_URL;
+            if (!url) {
+                this.logger.warn(`Unable to create RabbitMqSubscriber instance: ${JSON.stringify(this.options())} as rabbitmq url is not configured.`);
+                return;
+            }
+
+            // this.logger.debug(`RabbitMqSubscriber instance created with options: ${JSON.stringify(this.options())} and url: ${url}`);
+            const connection = await amqp.connect(url);
+            // this.logger.debug(`RabbitMqSubscriber connection established: ${JSON.stringify(this.options())} and url: ${url}`);
+
+            const channel = await connection.createChannel();
+            // this.logger.debug(`RabbitMqSubscriber channel created: ${JSON.stringify(this.options())} and url: ${url}`);
+
+            const options = this.options();
+
+            const queueName = options.queueName;
+            const exchangeName = `${queueName}.exchange`;
+            const routingKey = `${queueName}.routing-key`;
+
+            await channel.assertExchange(exchangeName, 'direct', {});
+            // this.logger.debug(`RabbitMqSubscriber channel asserted: ${JSON.stringify(this.options())} and url: ${url}`);
+
+            const queue = await channel.assertQueue(queueName, {});
+            // this.logger.debug(`RabbitMqSubscriber queue asserted: ${JSON.stringify(this.options())} and url: ${url}`);
+
+            await channel.bindQueue(queue.queue, exchangeName, routingKey);
+            // this.logger.debug(`RabbitMqSubscriber queue bound: ${JSON.stringify(this.options())} and url: ${url}`);
+
+            // Consume messages from the queue
+            channel.consume(
+                queue.queue,
+                async (rawMessage) => {
+                    if (rawMessage) {
+                        const messageContentString = rawMessage.content.toString();
+                        this.logger.debug(`RabbitMqSubscriber Received raw message: ${messageContentString}`);
+
+                        let message: QueueMessage<T> = null;
+
+                        try {
+                            message = JSON.parse(messageContentString) as QueueMessage<T>;
+
+                            // this is the first time we are receiving the message so we set the currentRetry to 0
+                            if (!message.retryCount) message.retryCount = 0;
+                            if (!message.retryInterval) message.retryInterval = 1000;
+                            if (!message.currentRetry) message.currentRetry = 0;
+
+                            await this.processMessage(message, rawMessage, channel);
+                        }
+                        catch (error) {
+                            this.logger.error(`Error processing message: ${error.message}`);
+
+                            // if an error occurs then if retryCount is set we start retrying. 
+                            if (message) {
+                                if (message.currentRetry < message.retryCount) {
+                                    await this.updateStatusInDatabase('retrying', message);
+
+                                    message.currentRetry++;
+                                    this.logger.warn(`Retrying message (${message.currentRetry}/${message.retryCount}) after ${message.retryInterval}ms`);
+                                    setTimeout(() => {
+                                        this.retryMessage(message, rawMessage, channel);
+                                    }, message.retryInterval);
+                                } else {
+                                    await this.updateStatusInDatabase('failed', message, error.message, '');
+
+                                    this.logger.error(`Message failed after ${message.retryCount} attempts: ${error.message}`);
+                                    channel.ack(rawMessage); // Discard the message after max retries
+                                }
+                            }
+
+                        }
+                    }
+                },
+                // { noAck: true },
+                {},
+            );
+
+            this.logger.debug(`RabbitMqSubscriber ready to consume messages: ${JSON.stringify(this.options())} and url: ${url}`);
+        }
+
+    }
+
+    /**
+     * Abstract method for message processing logic.
+     */
+    protected async processMessage(message: QueueMessage<T>, rawMessage, channel): Promise<void> {
+        await this.updateStatusInDatabase('started', message);
+
+        // Capture the results of handling the task.
+        const result = await this.subscribe(message);
+
+        // Ack the message. 
+        channel.ack(rawMessage);
+
+        // TODO: Update the database to indicate that the task is finished.
+        await this.updateStatusInDatabase('succeeded', message, '', result ? JSON.stringify(result, null, 2) : '');
+
+    }
+
+    /**
+     * Retry the message by invoking the processing logic again.
+     */
+    private async retryMessage(message: QueueMessage<T>, rawMessage, channel) {
+        try {
+            await this.processMessage(message, rawMessage, channel);
+        } catch (error) {
+            if (message.currentRetry < message.retryCount) {
+                await this.updateStatusInDatabase('retrying', message);
+
+                message.currentRetry++;
+                this.logger.warn(`Retrying message (${message.currentRetry}/${message.retryCount}) after ${message.retryInterval}ms: ${error.message}`);
+                setTimeout(() => {
+                    this.retryMessage(message, rawMessage, channel);
+                }, message.retryInterval);
+            } else {
+
+                this.logger.error(`Message failed after ${message.retryCount} attempts: ${error.message}`);
+
+                // Discard the message after max retries
+                channel.ack(rawMessage);
+
+                // TODO: Store the error in the database and update the status accordingly.
+                await this.updateStatusInDatabase('failed', message, error.message, '');
+
+            }
+        }
+    }
+
+    private async updateStatusInDatabase(stage: string, message: QueueMessage<T>, error: string = '', result: string = '') {
+
+        // TODO: make an entry in the relevant database table, generate a unique id earlier.
+        try {
+            // 1. resolve the queue first
+            const mqMessage = await this.mqMessageService.repo.findOne({
+                where: {
+                    messageId: message.messageId,
+                }
+            });
+
+            if (mqMessage) {
+                const updatedFields = {
+                    stage: stage
+                };
+                if (stage === 'failed' || stage === 'succeeded') {
+                    updatedFields['finishedAt'] = new Date();
+                    updatedFields['elapsedMillis'] = updatedFields['finishedAt'].getTime() - mqMessage.startedAt.getTime();
+                }
+                if (stage === 'succeeded') {
+                    updatedFields['output'] = result;
+                }
+                if (stage === 'failed') {
+                    updatedFields['error'] = error;
+                }
+                await this.mqMessageService.repo.update(mqMessage.id, updatedFields);
+            }
+        }
+        catch (error) {
+            this.logger.error(error.message, error.stack);
+        }
+
+    }
+
+}

package/src/services/refresh-token-ids-storage.service.ts

@@ -0,0 +1,45 @@
+import { CACHE_MANAGER } from '@nestjs/cache-manager';
+import { Inject, Injectable } from '@nestjs/common';
+import { Cache } from 'cache-manager';
+
+// TODO: Ideally this should be in a separate file - putting this here for brevity
+export class InvalidatedRefreshTokenError extends Error { }
+
+@Injectable()
+// export class RefreshTokenIdsStorageService implements OnApplicationBootstrap, OnApplicationShutdown {
+export class RefreshTokenIdsStorageService {
+    // private redisClient: Redis;
+    // onApplicationBootstrap() {
+    //     // TODO: Ideally, we should move this to the dedicated "RedisModule" instead of initiating the connection here.
+    //     this.redisClient = new Redis({
+    //         // TODO: According to best practices, we should use the environment variables here instead.
+    //         host: 'localhost',
+    //         port: 6379,
+    //     });
+    // }
+    // onApplicationShutdown(signal?: string) {
+    //     return this.redisClient.quit();
+    // }
+
+    constructor(@Inject(CACHE_MANAGER) private cacheManager: Cache) {}
+
+    async insert(userId: number, tokenId: string): Promise<void> {
+        await this.cacheManager.set(this.getKey(userId), tokenId);
+    }
+
+    async validate(userId: number, tokenId: string): Promise<boolean> {
+        const storedId = await this.cacheManager.get(this.getKey(userId));
+        if (storedId !== tokenId) {
+            throw new InvalidatedRefreshTokenError();
+        }
+        return storedId === tokenId;
+    }
+
+    async invalidate(userId: number): Promise<void> {
+        await this.cacheManager.del(this.getKey(userId));
+    }
+
+    private getKey(userId: number): string {
+        return `user-${userId}`;
+    }
+}

package/src/services/role-metadata.service.ts

@@ -0,0 +1,205 @@
+import { forwardRef, Inject, Injectable, Logger, NotFoundException } from '@nestjs/common';
+import { InjectEntityManager, InjectRepository } from '@nestjs/typeorm';
+import { DiscoveryService } from "@nestjs/core";
+import { EntityManager, In, Repository } from 'typeorm';
+import { CRUDService } from 'src/services/crud.service';
+import { ModelMetadataService } from 'src/services/model-metadata.service';
+import { ModuleMetadataService } from 'src/services/module-metadata.service';
+import { MediaStorageProviderMetadataService } from 'src/services/media-storage-provider-metadata.service';
+import { ConfigService } from '@nestjs/config';
+import { MediaService } from "src/services/media.service";
+import { FileService } from "src/services/file.service";
+import { CrudHelperService } from "src/services/crud-helper.service";
+
+import { RoleMetadata } from '../entities/role-metadata.entity';
+import { PermissionMetadata } from '../entities/permission-metadata.entity';
+import { CreateRoleMetadataDto } from '../dtos/create-role-metadata.dto';
+
+@Injectable()
+export class RoleMetadataService extends CRUDService<RoleMetadata> {
+  private readonly logger = new Logger(RoleMetadataService.name);
+
+  constructor(
+    @Inject(forwardRef(() => ModelMetadataService))
+    readonly modelMetadataService: ModelMetadataService,
+    readonly moduleMetadataService: ModuleMetadataService,
+    readonly mediaStorageProviderService: MediaStorageProviderMetadataService,
+    readonly configService: ConfigService,
+    readonly fileService: FileService,
+    readonly mediaService: MediaService,
+    readonly discoveryService: DiscoveryService,
+    readonly crudHelperService: CrudHelperService,
+    @InjectEntityManager()
+    readonly entityManager: EntityManager,
+    @InjectRepository(RoleMetadata, 'default')
+    readonly repo: Repository<RoleMetadata>,
+    @InjectRepository(PermissionMetadata)
+    private readonly permissionRepository: Repository<PermissionMetadata>,
+  ) {
+    super(modelMetadataService, moduleMetadataService, mediaStorageProviderService, configService, fileService, mediaService, discoveryService, crudHelperService, entityManager, repo, 'roleMetadata', 'solid-core');
+  }
+
+  async findRoleByName(roleName: string) {
+    const entity = await this.repo.findOne({
+      where: {
+        name: roleName
+      },
+      relations: {
+        permissions: true
+      }
+    });
+    if (!entity) {
+      throw new NotFoundException(`Entity #${roleName} not found`);
+    }
+    return entity;
+  }
+
+  async createRolesIfNotExists(roles: CreateRoleMetadataDto[]) {
+    for (let id = 0; id < roles.length; id++) {
+      try {
+        const roleObj = roles[id];
+        this.logger.log(`Resolving role: ${JSON.stringify(roleObj)}`);
+
+        const existingRole = await this.repo.findOne({
+          where: {
+            name: roleObj.name,
+          },
+          relations: {},
+        });
+
+        // Create only if not existing already.
+        if (!existingRole) {
+          this.logger.log(`Role ${roleObj.name} does not exist, hence creating`);
+
+          let permissions = [];
+
+          if (roleObj.permissions && roleObj.permissions.length > 0) {
+            await Promise.all(
+              roleObj.permissions.map(permission => this.preloadPermissionByName(permission.name)),
+            );
+          }
+
+          // const role = this.repo.create({ ...roleObj, permissions });
+          const role = this.repo.create({ ...roleObj });
+          await this.repo.save(role);
+        } else {
+          this.logger.log(`Role ${roleObj.name} already exists`);
+        }
+      } catch (error) {
+        this.logger.error(error);
+      }
+    }
+  }
+
+  async addAllPermissionsToRole(roleName: string): Promise<RoleMetadata> {
+    return await this._addPermissionsToRole(roleName, null);
+  }
+
+  async addPermissionsToRole(roleName: string, permissionNames: string[]): Promise<RoleMetadata> {
+    return await this._addPermissionsToRole(roleName, permissionNames);
+  }
+
+  async addPermissionToRole(roleName: string, permissionName: string) {
+    return await this._addPermissionsToRole(roleName, [permissionName]);
+  }
+
+  private async _addPermissionsToRole(roleName: string, permissionNames: string[]): Promise<RoleMetadata> {
+    const role = await this.repo.findOne({
+      where: { name: roleName },
+      relations: {
+        permissions: true
+      }
+    });
+    if (!role) {
+      throw new Error(`Role '${roleName}' not found.`);
+    }
+
+    this.logger.log(`Found role ${roleName}`);
+
+
+    // The new set of permissions which are to be added to this role.
+    let newPermissions: PermissionMetadata[];
+
+    // Load all the specified permissions in the system. 
+    if (permissionNames && permissionNames.length != 0) {
+      this.logger.log(`Loading specified permissions.`);
+
+      newPermissions = await this.permissionRepository.find({ where: { name: In(permissionNames) } });
+      if (newPermissions.length !== permissionNames.length) {
+        throw new Error(`One or more permissions not found.`);
+      }
+    }
+    else {
+      this.logger.log(`Loading all permissions in system.`);
+
+      // Load all permissions in the system. 
+      // TODO: Do we want to convert this to a paginated query to avoid having to load a very large permissions table into memory?
+      newPermissions = await this.permissionRepository.find();
+      if (newPermissions.length == 0) {
+        throw new Error(`No permissions configured in the system. Did you forget to run the PermissionSeederService?`);
+      }
+    }
+
+    this.logger.log(`Adding ${newPermissions.length} permissions to role ${roleName}.`);
+
+    // if there are already permissions assigned. 
+    if (role.permissions && role.permissions.length > 0) {
+      for (let i = 0; i < newPermissions.length; i++) {
+        const newPermission = newPermissions[i];
+        let newPermissionFound = true;
+        for (let j = 0; j < role.permissions.length; j++) {
+          const existingPermission = role.permissions[j];
+          if (existingPermission.name === newPermission.name) {
+            newPermissionFound = false;
+            break;
+          }
+        }
+
+        if (newPermissionFound) {
+          role.permissions.push(newPermission);
+        }
+      }
+
+    }
+    // else we create a new permissions set. 
+    else {
+      role.permissions = newPermissions;
+    }
+
+    return await this.repo.save(role);
+  }
+
+  async removePermissionsFromRole(roleName: string, permissionNames: string[]): Promise<RoleMetadata> {
+
+    // load the role with the respective permissions.
+    const role = await this.repo.findOne({
+      where: {
+        name: roleName
+      },
+      relations: {
+        permissions: true
+      }
+    });
+
+    if (!role) {
+      throw new Error(`Role ${roleName} not found.`);
+    }
+
+    // modify the permissions array.
+    role.permissions = role.permissions.filter(permission => !permissionNames.includes(permission.name));
+
+    return await this.repo.save(role);
+  }
+
+  private async preloadPermissionByName(name: string): Promise<PermissionMetadata> {
+    const existingPermission = await this.permissionRepository.findOne({
+      where: { name },
+    });
+    if (!existingPermission) {
+      throw new NotFoundException(`Permission ${name} does not exist`)
+    }
+    return existingPermission;
+  }
+
+
+}

package/src/services/selection-providers/list-of-models-selection-provider.service.ts

@@ -0,0 +1,70 @@
+import { Injectable } from "@nestjs/common";
+import { ModelMetadataService } from "src/services/model-metadata.service";
+import { SelectionProvider } from "src/decorators/selection-provider.decorator";
+import { ISelectionProvider, ISelectionProviderContext, ISelectionProviderValues } from "../../interfaces";
+
+
+@SelectionProvider()
+@Injectable()
+export class ListOfModelsSelectionProvider implements ISelectionProvider<ISelectionProviderContext> {
+    private readonly modelMetadataService: ModelMetadataService;
+    constructor(modelMetadataService: ModelMetadataService) {
+        this.modelMetadataService = modelMetadataService;
+    }
+    help(): string {
+        return "# Allows one to dynamically fetch models Across all models";
+    }
+    name(): string {
+        return 'ListOfModelsSelectionProvider';
+    }
+    async value(optionValue: string, ctxt: ISelectionProviderContext): Promise<ISelectionProviderValues | any> {
+        // Create a unified array with unique languages
+
+        const queryData = {
+            fields: [],
+            filters: [],
+            sort: [],
+            populate: [],
+            populateMedia: [],
+            groupBy: [],
+            limit: 100,
+            offset: 0,
+        }
+        const models = (await this.modelMetadataService.findMany(queryData)).records;
+
+        const model = models.filter(i => i.singularName === optionValue)[0];
+        return { label: model.singularName, value: model.id }
+    }
+
+    async values(query: string, ctxt: ISelectionProviderContext): Promise<readonly ISelectionProviderValues[]> {
+        // Filter function 
+
+        // TODO: Instead of using limit 1000, we can pass filters here so that we are able to query directly.
+        const queryData = {
+            fields: [],
+            filters: [],
+            sort: [],
+            populate: [],
+            populateMedia: [],
+            groupBy: [],
+            limit: 1000,
+            offset: 0,
+        }
+        const models = (await this.modelMetadataService.findMany(queryData)).records;
+        const getModelsByQuery = (query: string) => {
+            const lowerCaseQuery = query.toLowerCase();
+            return models.filter(model => model.singularName.toLowerCase().includes(lowerCaseQuery));
+        }
+
+        const filteredModels = query ? getModelsByQuery(query) : models;
+
+        // return filteredModels.map(i => {
+        //     return { label: i.singularName, value: String(i.id) };
+        // });
+
+        return filteredModels.map(i => {
+            return { label: i.singularName, value: i.singularName };
+        });
+
+    }
+}