@solidxai/core 0.0.2

package/src/services/mq-message.service.ts

@@ -0,0 +1,178 @@
+import { Injectable } from '@nestjs/common';
+import { InjectEntityManager } from '@nestjs/typeorm';
+import { ModuleRef } from "@nestjs/core";
+import { EntityManager } from 'typeorm';
+import { CRUDService } from 'src/services/crud.service';
+import { MqMessage } from '../entities/mq-message.entity';
+import { Logger } from '@nestjs/common';
+import { MqMessageRepository } from 'src/repository/mq-message.repository';
+
+@Injectable()
+export class MqMessageService extends CRUDService<MqMessage> {
+  private readonly logger = new Logger(MqMessageService.name);
+
+  constructor(
+    @InjectEntityManager()
+    readonly entityManager: EntityManager,
+    // @InjectRepository(MqMessage)
+    // readonly repo: Repository<MqMessage>,
+    readonly repo: MqMessageRepository,
+    readonly moduleRef: ModuleRef
+  ) {
+    super(entityManager, repo, 'mqMessage', 'solid-core', moduleRef);
+  }
+
+  async lockNextPendingMessage(queueName: string): Promise<MqMessage | null> {
+    // this.logger.debug(`Locking next pending message for queue: ${queueName}`);
+
+    return await this.entityManager.transaction(async manager => {
+      // Use raw SQL to skip locked rows (FOR UPDATE SKIP LOCKED)
+      const rawJob = await manager.query(`
+          SELECT ss_mq_message.id
+            FROM ss_mq_message
+            left join ss_mq_message_queue on ss_mq_message.mq_message_queue_id = ss_mq_message_queue.id
+           WHERE ss_mq_message_queue."name" = $1
+             AND ss_mq_message.stage = 'pending'
+           ORDER BY ss_mq_message.created_at ASC
+             FOR UPDATE SKIP LOCKED
+           LIMIT 1`,
+        [queueName]
+      );
+
+      // this.logger.debug(`Raw job fetched: ${JSON.stringify(rawJob)}`);
+      if (!rawJob || rawJob.length === 0) {
+        // this.logger.debug(`No pending job found for queue: ${queueName}`);
+        return null;
+      }
+      const job = await manager.getRepository(MqMessage).findOne({ where: { id: rawJob[0].id }, relations: ['mqMessageQueue'] });
+      if (job) {
+        // this.logger.debug(`Locked job id: ${job.id}, queue: ${job.mqMessageQueue.name}, stage: ${job.stage}`);
+
+        job.stage = 'scheduled';
+        await manager.save(job);
+
+        return job;
+      }
+
+      return null;
+    });
+  }
+
+  /**
+   * Wait until a queue message reaches a terminal status (succeeded/failed).
+   *
+   * @param messageId string – the external message id you store in `ss_mq_message.messageId`
+   * @param opts.timeoutMs total time to wait before giving up (default 60s)
+   * @param opts.intervalMs initial poll interval (default 500ms)
+   * @param opts.maxIntervalMs cap for exponential backoff (default 2000ms)
+   * @param opts.throwOnFailure if true, throws when stage === 'failed' (default false)
+   * @param opts.parseJson try JSON.parse on `output` and `error` (default true)
+   * @returns resolves with the final MqMessage row when terminal, rejects on timeout (or failure if throwOnFailure)
+   */
+  async waitForTerminalStatus(
+    messageId: string,
+    opts?: {
+      timeoutMs?: number;
+      intervalMs?: number;
+      maxIntervalMs?: number;
+      throwOnFailure?: boolean;
+      parseJson?: boolean;
+      logEveryN?: number;
+    },
+  ): Promise<MqMessage> {
+    const {
+      timeoutMs = 60_000,
+      intervalMs = 500,
+      maxIntervalMs = 2_000,
+      throwOnFailure = false,
+      parseJson = true,
+      logEveryN = 10, // log every N polls to avoid noisy logs
+    } = opts || {};
+
+    const start = Date.now();
+    let attempt = 0;
+    let delay = intervalMs;
+
+    // Small helper
+    const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+    while (true) {
+      attempt++;
+
+      // Fetch minimal columns needed for quick polling
+      const rec = await this.repo.findOne({
+        where: { messageId },
+        select: {
+          id: true,
+          messageId: true,
+          stage: true,
+          finishedAt: true,
+          elapsedMillis: true,
+          output: true,
+          error: true,
+          input: true,
+          // add other fields if you need to return them
+        } as any,
+        loadEagerRelations: false,
+      });
+
+      if (attempt % logEveryN === 0) {
+        this.logger.debug(
+          `waitForTerminalStatus(${messageId}) poll #${attempt} -> ${rec?.stage ?? 'not_found'}`
+        );
+      }
+
+      if (!rec) {
+        // Not found yet – keep waiting until timeout
+      } else if (rec.stage === 'succeeded' || rec.stage === 'failed') {
+        // Optionally parse output/error if they contain JSON strings
+        if (parseJson) {
+          // rec.output = this.safeJsonParse(rec.output);
+          rec.output = rec.output;
+          // rec.error = this.safeJsonParse(rec.error);
+          rec.error = rec.error;
+        }
+
+        if (rec.stage === 'failed' && throwOnFailure) {
+          throw new Error(
+            `Queue message ${messageId} failed` + (rec.error ? `: ${JSON.stringify(rec.error).slice(0, 500)}` : '')
+          );
+        }
+        return rec;
+      }
+
+      // Timeout?
+      const elapsed = Date.now() - start;
+      if (elapsed >= timeoutMs) {
+        throw new Error(`Timed out after ${timeoutMs}ms waiting for message ${messageId} to reach terminal status`);
+      }
+
+      // Backoff with cap
+      await sleep(delay);
+      delay = Math.min(Math.floor(delay * 1.5), maxIntervalMs);
+    }
+  }
+
+  // /**
+  //  * Optional wrapper: publish and then wait (if your publisher returns the messageId).
+  //  */
+  // async publishAndWait<T>(
+  //   publishFn: () => Promise<string>, // returns messageId
+  //   waitOpts?: Parameters<MqMessageService['waitForTerminalStatus']>[1],
+  // ): Promise<MqMessage> {
+  //   const messageId = await publishFn();
+  //   return this.waitForTerminalStatus(messageId, waitOpts);
+  // }
+
+  private safeJsonParse(value: unknown): unknown {
+    if (value == null) return value;
+    if (typeof value !== 'string') return value;
+    const s = value.trim();
+    if (!s) return s;
+    try {
+      return JSON.parse(s);
+    } catch {
+      return value; // leave as-is if not valid JSON
+    }
+  }
+}

package/src/services/pdf.service.ts

@@ -0,0 +1,29 @@
+import { Injectable } from '@nestjs/common';
+import * as puppeteer from 'puppeteer';
+
+@Injectable()
+export class PdfService {
+  async generatePdf(html: string): Promise<Uint8Array> {
+    // Launch a new browser instance
+    const browser = await puppeteer.launch({
+      args: ['--no-sandbox', '--disable-setuid-sandbox'],
+    });
+    const page = await browser.newPage();
+  
+    // Set the HTML content
+    await page.setContent(html);
+  
+    // Generate the PDF and store it in a buffer
+    const pdfBuffer = await page.pdf({ format: 'A4' });
+  
+    // Close the browser
+    await browser.close();
+  
+    // Convert the Uint8Array to a Buffer
+    // const buffer = Buffer.from(pdfBuffer);
+  
+    // Return the generated PDF as a buffer
+    // return buffer;
+    return pdfBuffer
+  }
+}

package/src/services/permission-metadata.service.ts

@@ -0,0 +1,51 @@
+import { Injectable } from '@nestjs/common';
+import { ModuleRef } from "@nestjs/core";
+import { InjectEntityManager } from '@nestjs/typeorm';
+import { CRUDService } from 'src/services/crud.service';
+import { EntityManager, In } from 'typeorm';
+
+
+import { PermissionMetadataRepository } from 'src/repository/permission-metadata.repository';
+import { PermissionMetadata } from '../entities/permission-metadata.entity';
+
+@Injectable()
+export class PermissionMetadataService extends CRUDService<PermissionMetadata> {
+  constructor(
+    @InjectEntityManager()
+    readonly entityManager: EntityManager,
+    // @InjectRepository(PermissionMetadata, 'default')
+    // readonly repo: Repository<PermissionMetadata>,
+    readonly repo: PermissionMetadataRepository,
+    readonly moduleRef: ModuleRef
+
+  ) {
+    super(entityManager, repo, 'permissionMetadata', 'solid-core',moduleRef);
+  }
+
+
+  findAllUsingRoles(roles: string[]) {
+    return this.repo.find({
+      where: {
+        roles: {
+          name: In(roles)
+        }
+      },
+      relations: {}
+    });
+  }
+
+  permissionExistsInRole(role: string, permission: string,) {
+    return this.repo.find({
+      where: {
+        name: permission,
+        roles: {
+          name: role
+        }
+      },
+      relations: {}
+    });
+  }
+
+}
+
+

package/src/services/poller.service.ts

@@ -0,0 +1,163 @@
+// src/common/poller/poller.service.ts
+import {
+    Injectable,
+    Logger,
+    OnModuleDestroy,
+    BeforeApplicationShutdown,
+} from '@nestjs/common';
+
+export interface PollOptions {
+    /** Wait after a successful iteration */
+    baseDelayMs?: number;            // default 1000
+    /** Maximum delay after repeated failures */
+    maxDelayMs?: number;             // default 30000
+    /** Per-iteration timeout guard */
+    timeoutPerIterationMs?: number;  // default 60000
+    /** Add jitter to spread load */
+    jitter?: boolean;                // default true
+}
+
+type ProcessNextFn = (queueName: string) => Promise<unknown>;
+
+interface PollerState {
+    queueName: string;
+    processNext: ProcessNextFn;
+    opts: Required<PollOptions>;
+    inFlight: boolean;
+    stopped: boolean;
+    backoff: number;
+    nextTimer?: NodeJS.Timeout;
+}
+
+@Injectable()
+export class PollerService implements OnModuleDestroy, BeforeApplicationShutdown {
+    private readonly logger = new Logger(PollerService.name);
+    private readonly pollers = new Map<string, PollerState>();
+
+    start(queueName: string, processNext: ProcessNextFn, options: PollOptions = {}): void {
+        if (this.pollers.has(queueName)) {
+            this.logger.warn(`Poller "${queueName}" already started; ignoring.`);
+            return;
+        }
+
+        const opts: Required<PollOptions> = {
+            baseDelayMs: options.baseDelayMs ?? 1000,
+            maxDelayMs: options.maxDelayMs ?? 30_000,
+            timeoutPerIterationMs: options.timeoutPerIterationMs ?? 5 * 60_000,
+            jitter: options.jitter ?? true,
+        };
+
+        const state: PollerState = {
+            queueName,
+            processNext,
+            opts,
+            inFlight: false,
+            stopped: false,
+            backoff: opts.baseDelayMs,
+            nextTimer: undefined,
+        };
+
+        this.pollers.set(queueName, state);
+        // kick off on next tick
+        setImmediate(() => this.poll(state).catch(() => { }));
+        this.logger.log(`Started poller "${queueName}"`);
+    }
+
+    stop(queueName: string): void {
+        const state = this.pollers.get(queueName);
+        if (!state) return;
+
+        state.stopped = true;
+        if (state.nextTimer) {
+            clearTimeout(state.nextTimer);
+            state.nextTimer = undefined;
+        }
+        this.pollers.delete(queueName);
+        this.logger.log(`Stopped poller "${queueName}"`);
+    }
+
+    stopAll(): void {
+        for (const name of Array.from(this.pollers.keys())) {
+            this.stop(name);
+        }
+    }
+
+    async onModuleDestroy(): Promise<void> {
+        this.stopAll();
+    }
+
+    async beforeApplicationShutdown(): Promise<void> {
+        this.stopAll();
+    }
+
+    // ---- internals ----
+
+    private async poll(state: PollerState): Promise<void> {
+        if (state.stopped || state.inFlight) return;
+        state.inFlight = true;
+
+        try {
+            await this.withTimeout(
+                state.processNext(state.queueName),
+                state.opts.timeoutPerIterationMs,
+            );
+
+            // success: reset backoff and schedule next run after base delay
+            state.backoff = state.opts.baseDelayMs;
+            // this.logger.debug(`[${state.queueName}] iteration completed`);
+            this.schedule(state, state.opts.baseDelayMs);
+        } catch (err: unknown) {
+            const msg = this.errorToString(err);
+            this.logger.error(`[${state.queueName}] iteration failed: ${msg}`);
+
+            // failure: schedule with backoff + optional jitter, then increase backoff
+            const wait = this.computeWait(state.backoff, state.opts);
+            state.backoff = Math.min(state.backoff * 2, state.opts.maxDelayMs);
+            this.schedule(state, wait);
+        } finally {
+            state.inFlight = false;
+        }
+    }
+
+    private schedule(state: PollerState, delayMs: number) {
+        if (state.stopped) return;
+        if (state.nextTimer) clearTimeout(state.nextTimer);
+
+        state.nextTimer = setTimeout(() => {
+            // clear reference before calling poll to avoid re-entrancy confusion
+            state.nextTimer = undefined;
+            this.poll(state).catch(() => { });
+        }, delayMs);
+    }
+
+    private computeWait(currentBackoff: number, opts: Required<PollOptions>): number {
+        if (!opts.jitter) return currentBackoff;
+        // Full jitter: random in [250ms, currentBackoff * 2], clamped to maxDelayMs
+        const doubled = Math.min(currentBackoff * 2, opts.maxDelayMs);
+        const jittered = Math.floor(Math.random() * doubled);
+        return Math.max(250, jittered);
+    }
+
+    private async withTimeout<T>(p: Promise<T>, ms: number): Promise<T> {
+        let timer: NodeJS.Timeout | undefined;
+        try {
+            return await Promise.race<T>([
+                p,
+                new Promise<never>((_, rej) => {
+                    timer = setTimeout(() => rej(new Error(`Iteration timed out after ${ms} ms`)), ms);
+                }),
+            ]);
+        } finally {
+            if (timer) clearTimeout(timer);
+        }
+    }
+
+    private errorToString(err: unknown): string {
+        if (err instanceof Error) return err.stack ?? err.message;
+        try {
+            return JSON.stringify(err);
+        } catch {
+            return String(err);
+        }
+    }
+}

package/src/services/question-data-providers/chartjs-sql-data-provider.service.ts

@@ -0,0 +1,125 @@
+import { Injectable, Logger } from "@nestjs/common";
+import { DashboardQuestionDataProvider } from "src/decorators/dashboard-question-data-provider.decorator";
+import { DashboardQuestion } from "src/entities/dashboard-question.entity";
+import { IDashboardQuestionDataProvider } from "src/interfaces";
+import { EntityManager } from "typeorm";
+import { SqlExpressionResolverService } from "../sql-expression-resolver.service";
+import { getKpi, getLabels } from "./helpers";
+
+export interface QuestionSqlDataProviderContext {
+    // questionSqlDatasetConfig: QuestionSqlDatasetConfig;
+    // questionId: number;
+    // question: Question;
+}
+
+export enum SqlExpressionOperator {
+    EQUALS = '$equals',
+    NOT_EQUALS = '$notEquals',
+    CONTAINS = '$contains',
+    NOT_CONTAINS = '$notContains',
+    STARTS_WITH = '$startsWith',
+    ENDS_WITH = '$endsWith',
+    IN = '$in',
+    NOT_IN = '$notIn',
+    BETWEEN = '$between',
+    LT = '$lt',
+    LTE = '$lte',
+    GT = '$gt',
+    GTE = '$gte'
+}
+
+export interface SqlExpression {
+    variableName: string; // The name of the variable in the SQL query
+    operator: SqlExpressionOperator; // The operator to use for the replacement (e.g., '=', '>', '<', etc.)
+    value: string[]; // The value to replace the variable with
+}
+
+@DashboardQuestionDataProvider()
+@Injectable()
+export class ChartJsSqlDataProvider implements IDashboardQuestionDataProvider<QuestionSqlDataProviderContext, any> {
+    private readonly logger = new Logger(ChartJsSqlDataProvider.name);
+
+    constructor(private readonly entityManager: EntityManager, private readonly sqlExpressionResolver: SqlExpressionResolverService) { }
+
+    help(): string {
+        return "Provides data for dashboard questions using a SQL dataset configuration. Configure your SQL dataset in the admin panel, then reference it in your dashboard question to fetch data.";
+    }
+
+    name(): string {
+        return "ChartJsSqlDataProvider";
+    }
+
+    async getData(question: DashboardQuestion, expressions?: SqlExpression[], context?: QuestionSqlDataProviderContext): Promise<any> {
+        // TODO: put some validation to check if the results of each SQL in each dataset returns the same number of rows 
+
+        // This is what we have to return.
+        // const data = {
+        //     labels: ['January', 'February', 'March', 'April', 'May', 'June', 'July'],
+        //     datasets: [
+        //         {
+        //             label: 'Dataset 1',
+        //             data: labels.map(() => faker.number.int({ min: 0, max: 1000 })),
+        //             backgroundColor: 'rgba(255, 99, 132, 0.5)',
+        //         },
+        //         {
+        //             label: 'Dataset 2',
+        //             data: labels.map(() => faker.number.int({ min: 0, max: 1000 })),
+        //             backgroundColor: 'rgba(53, 162, 235, 0.5)',
+        //         },
+        //         {
+        //             label: 'Dataset 3',
+        //             data: labels.map(() => faker.number.int({ min: 0, max: 1000 })),
+        //             backgroundColor: 'rgba(53, 235, 162, 0.5)',
+        //         },
+        //     ],
+        // };
+
+        // TODO: Load the set of labels by using a separate field on the question entity.
+
+        let datasetIdx = 0;
+        const datasets = [];
+
+        const labels: string[] = await getLabels(question, expressions, this.entityManager, this.sqlExpressionResolver);
+        const kpi: string = await getKpi(question, expressions, this.entityManager, this.sqlExpressionResolver);
+
+        // const question = context.question;
+        for (const questionSqlDatasetConfig of question.questionSqlDatasetConfigs) {
+
+            const sql = questionSqlDatasetConfig.sql;
+            if (!sql) {
+                throw new Error(`SQL dataset ${questionSqlDatasetConfig.datasetName} configuration does not contain a valid SQL query.`);
+            }
+
+            const sqlReplacementResult = this.sqlExpressionResolver.resolveSqlWithExpressions(sql, expressions || []);
+            this.logger.debug(`Final Sql query for dataset [${questionSqlDatasetConfig.datasetName}] is query=[${sqlReplacementResult.rawSql}]`);
+            this.logger.debug(`Final Sql query for dataset [${questionSqlDatasetConfig.datasetName}] is parameters=[${JSON.stringify(sqlReplacementResult.parameters)}]`);
+            const results = await this.entityManager.query(sqlReplacementResult.rawSql, sqlReplacementResult.parameters);
+
+            // Also for each data set we create the dataset object as is expected by ChartJs.
+            const data = [];
+            for (let i = 0; i < results.length; i++) {
+                const result = results[i];
+                data.push(result[questionSqlDatasetConfig.valueColumnName]);
+            }
+            datasets.push({
+                label: questionSqlDatasetConfig.datasetDisplayName,
+                data: data,
+                ...JSON.parse(questionSqlDatasetConfig.options || '{}'),
+            });
+
+            datasetIdx++;
+        }
+
+        return {
+            kpi,
+            visualizedAs: question.visualisedAs,
+            visualizationData: {
+                labels,
+                datasets
+            }
+        };
+
+    }
+
+
+}

package/src/services/question-data-providers/helpers.ts

@@ -0,0 +1,30 @@
+import { DashboardQuestion } from "src/entities/dashboard-question.entity";
+import { EntityManager } from "typeorm";
+import { SqlExpression } from "./chartjs-sql-data-provider.service";
+import { SqlExpressionResolverService } from "../sql-expression-resolver.service";
+
+export async function getLabels(question: DashboardQuestion, expressions: SqlExpression[], entityManager: EntityManager, sqlExpressionResolver: SqlExpressionResolverService): Promise<string[]> {
+    const sql = question.labelSql;
+    if (!sql) {
+        return [];
+    } 
+    const sqlReplacementResult = sqlExpressionResolver.resolveSqlWithExpressions(sql, expressions || []);
+   
+    const labelResults = await entityManager.query(sqlReplacementResult.rawSql, sqlReplacementResult.parameters);
+    
+    // Assuming labelResults has a single row with a 'label' field
+    // Map the label results to the labels array
+    const labels: string[] = labelResults.map((result: { [x: string]: string; }) => result['label']);
+    return labels;
+}
+
+export async function getKpi(question: DashboardQuestion, expressions: SqlExpression[], entityManager: EntityManager, sqlExpressionResolver: SqlExpressionResolverService): Promise<string> {
+    const sql = question.kpiSql;
+    if (!sql) {
+        return "";
+    }
+    const sqlReplacementResult = sqlExpressionResolver.resolveSqlWithExpressions(sql, expressions || []);
+    const result = await entityManager.query(sqlReplacementResult.rawSql, sqlReplacementResult.parameters);
+    const kpiResult = result.pop();
+    return kpiResult?.kpi || "";
+}

package/src/services/question-data-providers/prime-react-datatable-sql-data-provider.service.ts

@@ -0,0 +1,78 @@
+import { Injectable } from "@nestjs/common";
+import { DashboardQuestionDataProvider } from "src/decorators/dashboard-question-data-provider.decorator";
+import { DashboardQuestion } from "src/entities/dashboard-question.entity";
+import { IDashboardQuestionDataProvider } from "src/interfaces";
+import { EntityManager } from "typeorm";
+import { SqlExpressionResolverService } from "../sql-expression-resolver.service";
+import { Logger } from '@nestjs/common';
+import { SqlExpression } from "./chartjs-sql-data-provider.service";
+import { getKpi } from "./helpers";
+
+export interface QuestionSqlDataProviderContext {
+    // questionSqlDatasetConfig: QuestionSqlDatasetConfig;
+    // questionId: number;
+    // question: Question;
+}
+
+@DashboardQuestionDataProvider()
+@Injectable()
+export class PrimeReactDatatableSqlDataProvider implements IDashboardQuestionDataProvider<QuestionSqlDataProviderContext, any> {
+    private readonly logger = new Logger(PrimeReactDatatableSqlDataProvider.name);
+
+    constructor(private readonly entityManager: EntityManager, private readonly sqlExpressionResolver: SqlExpressionResolverService) { }
+
+    help(): string {
+        return "Provides data for dashboard questions using a SQL dataset configuration. Configure your SQL dataset in the admin panel, then reference it in your dashboard question to fetch data.";
+    }
+
+    name(): string {
+        return "PrimeReactDatatableSqlDataProvider";
+    }
+
+    async getData(question: DashboardQuestion, expressions?: SqlExpression[], context?: QuestionSqlDataProviderContext): Promise<any> {
+        // TODO: put some validation to check if the results of each SQL in each dataset returns the same number of rows 
+
+        // Check the expected response for prime react data tables to understand what is going on here...
+
+        const kpi: string = await getKpi(question, expressions, this.entityManager, this.sqlExpressionResolver);
+        // TODO: Load the set of labels by using a separate field on the question entity.
+        const labelSql = question.labelSql;
+        const labelResults = await this.entityManager.query(labelSql);
+        const columns = [];
+        for (let i = 0; i < labelResults.length; i++) {
+            const labelResult = labelResults[i];
+            columns.push({
+                field: labelResult['field'],
+                header: labelResult['header'],
+            });
+        }
+
+        // Load the chart options as a JSON 
+        // const chartOptions = JSON.parse(question.barChartLabelOptions || '{}');
+
+        const values = []
+
+        // For meter group we can assume that we only have one sql dataset config.
+        const questionSqlDatasetConfig = question.questionSqlDatasetConfigs[0];
+
+        const sql = questionSqlDatasetConfig.sql;
+        if (!sql) {
+            throw new Error(`SQL dataset ${questionSqlDatasetConfig.datasetName} configuration does not contain a valid SQL query.`);
+        }
+
+        const sqlReplacementResult = this.sqlExpressionResolver.resolveSqlWithExpressions(sql, expressions || []);
+        this.logger.debug(`Final Sql query for dataset [${questionSqlDatasetConfig.datasetName}] is query=[${sqlReplacementResult.rawSql}]`);
+        this.logger.debug(`Final Sql query for dataset [${questionSqlDatasetConfig.datasetName}] is parameters=[${JSON.stringify(sqlReplacementResult.parameters)}]`);
+        const results = await this.entityManager.query(sqlReplacementResult.rawSql, sqlReplacementResult.parameters);
+
+        return {
+            kpi,
+            visualisedAs: question.visualisedAs,
+            visualizationData: {
+                columns,
+                rows: results,
+            }
+        };
+
+    }
+}