@solidstarters/solid-core 1.2.200 → 1.2.201

package/src/services/crud.service.ts

@@ -1,4 +1,4 @@
-import { BadRequestException, NotFoundException } from "@nestjs/common";
+import { BadRequestException, Inject, NotFoundException } from "@nestjs/common";
 import { ConfigService } from "@nestjs/config";
 import { DiscoveryService, ModuleRef } from "@nestjs/core";
 import { isArray } from "class-validator";
@@ -33,13 +33,14 @@ import { SelectionStaticFieldCrudManager } from "../helpers/field-crud-managers/
 import { ShortTextFieldCrudManager } from "../helpers/field-crud-managers/ShortTextFieldCrudManager";
 import { UUIDFieldCrudManager } from "../helpers/field-crud-managers/UUIDFieldCrudManager";
 import { FieldCrudManager, MediaWithFullUrl } from "../interfaces";
-import { CrudHelperService, UserIdFields } from "./crud-helper.service";
+import { CrudHelperService, FilterCombinator, UserIdFields } from "./crud-helper.service";
 import { FileService } from "./file.service";
 import { HashingService } from "./hashing.service";
 import { getMediaStorageProvider } from "./mediaStorageProviders";
 import { ModelMetadataService } from "./model-metadata.service";
 import { ModuleMetadataService } from "./module-metadata.service";
 import { RequestContextService } from "./request-context.service";
+import { BasicGroupFilterDto } from "src/dtos/basic-group-filters.dto";
 
 export class CRUDService<T extends CommonEntity> { // Add two generic value i.e Person,CreatePersonDto, so we get the proper types in our service
 
@@ -55,6 +56,8 @@ export class CRUDService<T extends CommonEntity> { // Add two generic value i.e
         readonly modelName: string,
         readonly moduleName: string,
         readonly moduleRef: ModuleRef,
+        readonly defaultEntityManager? : EntityManager
+        
         //We can just have the Model Entity here
     ) { }
 
@@ -182,7 +185,12 @@ export class CRUDService<T extends CommonEntity> { // Add two generic value i.e
             throw new BadRequestException(`Cannot update a published record for model ${this.modelName}. Unpublish it first.`
             );
         }
-        updateDto.id = id;
+
+        // // In some instances for legacy tables sometimes id is mapped as a bigint. 
+        // // in these cases the update method ends up attempting to insert records due to some type orm type mismatch issue.
+        // const idFieldMetadata = model.fields.find(f => f.name === 'id');
+        // updateDto.id = idFieldMetadata?.type === 'bigint' ? BigInt(id) : id;
+
         // This class will be extended by the generated service class i.e PersonService
         // The data required to identify the model and module name will be passed from the generate CrudService subclass
         //TODO: Algorithm to create the entity
@@ -455,19 +463,34 @@ export class CRUDService<T extends CommonEntity> { // Add two generic value i.e
 
         // Create above query on pincode table using query builder
         var qb: SelectQueryBuilder<T> = await this.repo.createSecurityRuleAwareQueryBuilder(alias)
-        // qb = this.crudHelperService.buildFilterQuery(qb, basicFilterDto, alias);
-        if (internationalisation && draftPublishWorkflow) {
-            qb = this.crudHelperService.buildFilterQuery(qb, basicFilterDto, alias, internationalisation, draftPublishWorkflow, this.moduleRef);
-        }
-        else {
-            qb = this.crudHelperService.buildFilterQuery(qb, basicFilterDto, alias);
-        }
 
         if (basicFilterDto.groupBy) {
-            // Get the records and the count
-            const { groupMeta, groupRecords } = await this.handleGroupFind(qb, groupFilter, populateGroup, alias, populateUserIdFields, populateMedia);
-            const totalGroups = await this.crudHelperService.countGroupedRecords(qb, basicFilterDto, alias);
-            // qb = this.crudHelperService.buildFilterQuery(qb, basicFilterDto, alias);
+            const groupFilterQb = (internationalisation && draftPublishWorkflow)
+                ? this.crudHelperService.buildFilterQuery(qb, basicFilterDto, alias, internationalisation, draftPublishWorkflow, this.moduleRef, FilterCombinator.AND, false, false)
+                : this.crudHelperService.buildFilterQuery(qb, basicFilterDto, alias, undefined, undefined, undefined, FilterCombinator.AND, false, false);
+
+            const groupByFields = this.crudHelperService.normalize(basicFilterDto.groupBy);
+            if (!groupByFields.length) {
+                throw new BadRequestException(ERROR_MESSAGES.INVALID_GROUP_BY_COUNT);
+            }
+
+            if (basicFilterDto.populateGroup) {
+                const hasRelationGroup = groupByFields.some(field => field.includes('.'));
+                if (hasRelationGroup) {
+                    throw new BadRequestException('populateGroup is not supported when grouping on relation fields. Fetch group metadata first and retrieve records in a separate call.');
+                }
+            }
+
+            const { aliasMap: groupAliasMap, formatMap: groupFormatMap, expressionMap: groupExpressionMap } = this.crudHelperService.applyGroupBySelections(groupFilterQb, groupByFields, alias);
+            const aggregateAliasMap = this.crudHelperService.applyAggregates(groupFilterQb, basicFilterDto.aggregates, alias);
+            const sortAliasMap = { ...groupAliasMap, ...aggregateAliasMap };
+            this.crudHelperService.applyGroupSortingAndPagination(groupFilterQb, basicFilterDto.sort, sortAliasMap, limit, offset);
+
+            const groupByResult = await groupFilterQb.getRawMany();
+            const totalGroups = await this.crudHelperService.countGroups(groupFilterQb);
+
+            const groupByFieldsOrdered = this.crudHelperService.normalize(basicFilterDto.groupBy || []);
+            const { groupMeta, groupRecords } = await this.handleGroupFind(groupByResult, groupFilter, populateGroup, alias, populateUserIdFields, populateMedia, basicFilterDto, groupAliasMap, aggregateAliasMap, groupByFieldsOrdered, groupFormatMap, groupExpressionMap);
 
             return {
                 meta: {
@@ -478,7 +501,9 @@ export class CRUDService<T extends CommonEntity> { // Add two generic value i.e
             }
         }
         else {
-            // Get the records and the count
+            qb = (internationalisation && draftPublishWorkflow)
+                ? this.crudHelperService.buildFilterQuery(qb, basicFilterDto, alias, internationalisation, draftPublishWorkflow, this.moduleRef)
+                : this.crudHelperService.buildFilterQuery(qb, basicFilterDto, alias);
             const { meta, records } = await this.handleNonGroupFind(qb, populateUserIdFields, populateMedia, offset, limit, alias);
             return {
                 meta,
@@ -503,17 +528,38 @@ export class CRUDService<T extends CommonEntity> { // Add two generic value i.e
         return this.wrapFindResponse(offset, limit, count, entities);
     }
 
-    private async handleGroupFind(qb: SelectQueryBuilder<T>, groupFilter: BasicFilterDto, populateGroup: boolean, alias: string, populateUserIdFields: UserIdFields[], populateMedia: string[]) {
-        const groupByResult = await qb.getRawMany();
-
+    private async handleGroupFind(
+        groupByResult: any[],
+        groupFilter: BasicGroupFilterDto | undefined,
+        populateGroup: boolean,
+        alias: string,
+        populateUserIdFields: UserIdFields[],
+        populateMedia: string[],
+        baseFilterDto: BasicFilterDto,
+        groupAliasMap: Record<string, string>,
+        aggregateAliasMap: Record<string, string>,
+        groupByFieldsOrdered: string[],
+        groupFormatMap: Record<string, string | undefined>,
+        groupExpressionMap: Record<string, string>
+    ) {
         const groupMeta = [];
         const groupRecords = [];
+        const aggregateAliasSet = new Set(Object.values(aggregateAliasMap));
         // For each group, get the records and the count
         for (const group of groupByResult) {
             if (populateGroup) {
                 let groupByQb: SelectQueryBuilder<T> = await this.repo.createSecurityRuleAwareQueryBuilder(alias);
-                groupByQb = this.crudHelperService.buildFilterQuery(groupByQb, groupFilter, alias);
-                groupByQb = this.crudHelperService.buildGroupByRecordsQuery(groupByQb, group, alias);
+                const groupFilterDto: BasicFilterDto = {
+                    ...baseFilterDto,
+                    ...groupFilter,
+                    groupBy: undefined,
+                    aggregates: undefined,
+                    // Only use explicit groupFilter.sort for record ordering; group-level sorts can contain
+                    // group expressions (e.g. createdAt:day) that are invalid on record queries.
+                    sort: groupFilter?.sort,
+                };
+                groupByQb = this.crudHelperService.buildFilterQuery(groupByQb, groupFilterDto, alias);
+                groupByQb = this.crudHelperService.buildGroupByRecordsQuery(groupByQb, group, alias, groupAliasMap, aggregateAliasMap, groupExpressionMap);
                 const [entities, count] = await groupByQb.getManyAndCount();
 
                 // Populate the entity with the userId fields
@@ -525,20 +571,22 @@ export class CRUDService<T extends CommonEntity> { // Add two generic value i.e
                 if (populateMedia && populateMedia.length > 0) {
                     await this.handlePopulateMedia(populateMedia, entities);
                 }
-                const groupData = this.wrapFindResponse(groupFilter.offset, groupFilter.limit, count, entities);
-                groupRecords.push(this.crudHelperService.createGroupRecords(group, alias, groupData));
+                const groupData = this.wrapFindResponse(groupFilter?.offset, groupFilter?.limit, count, entities);
+                groupRecords.push(this.crudHelperService.createGroupRecords(group, aggregateAliasSet, groupData, groupByFieldsOrdered, groupAliasMap, groupFormatMap));
             }
-            groupMeta.push(this.crudHelperService.createGroupMeta(group, alias));
+            groupMeta.push(this.crudHelperService.createGroupMeta(group, aggregateAliasSet, groupByFieldsOrdered, groupAliasMap, groupFormatMap));
         }
         return { groupMeta, groupRecords };
     }
 
-    private wrapFindResponse(offset: number, limit: number, count: number, entities: T[]) {
-        const currentPage = Math.floor(offset / limit) + 1;
-        const totalPages = Math.ceil(count / limit);
+    private wrapFindResponse(offset: number | undefined, limit: number | undefined, count: number, entities: T[]) {
+        const safeLimit = limit ?? count ?? 0;
+        const safeOffset = offset ?? 0;
+        const currentPage = safeLimit ? Math.floor(safeOffset / safeLimit) + 1 : 1;
+        const totalPages = safeLimit ? Math.ceil(count / safeLimit) : 1;
 
-        const nextPage = currentPage < totalPages ? currentPage + 1 : null;
-        const prevPage = currentPage > 1 ? currentPage - 1 : null;
+        const nextPage = safeLimit && currentPage < totalPages ? currentPage + 1 : null;
+        const prevPage = safeLimit && currentPage > 1 ? currentPage - 1 : null;
 
         const r = {
             meta: {
@@ -547,7 +595,7 @@ export class CRUDService<T extends CommonEntity> { // Add two generic value i.e
                 nextPage: nextPage,
                 prevPage: prevPage,
                 totalPages: totalPages,
-                perPage: +limit,
+                perPage: safeLimit ? +safeLimit : 0,
             },
             records: entities
         };
@@ -574,7 +622,7 @@ export class CRUDService<T extends CommonEntity> { // Add two generic value i.e
     }
 
     private async handlePopulateMedia(populateMedia: string[], entities: T[]) {
-        const model = await this.entityManager.getRepository(ModelMetadata).findOne({
+        const model = await this.getDefaultEntityManager().getRepository(ModelMetadata).findOne({
             where: {
                 singularName: this.modelName,
             },
@@ -913,7 +961,7 @@ export class CRUDService<T extends CommonEntity> { // Add two generic value i.e
             throw new BadRequestException(`Field ${field.name} does not define a relationCoModelSingularName`);
         }
 
-        const relationCoModel = await this.entityManager.getRepository(ModelMetadata).findOne({
+        const relationCoModel = await this.getDefaultEntityManager().getRepository(ModelMetadata).findOne({
             where: { singularName: field.relationCoModelSingularName },
             relations: ['fields', 'fields.mediaStorageProvider', 'fields.model'],
         });
@@ -1016,5 +1064,8 @@ export class CRUDService<T extends CommonEntity> { // Add two generic value i.e
 
         return updatedEntity
     }
-}
 
+    private getDefaultEntityManager(){
+        return this.defaultEntityManager ?? this.entityManager;
+    }
+}

package/src/services/database/database-bootstrap.service.ts

@@ -0,0 +1,91 @@
+import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
+import { DataSource } from 'typeorm';
+import { readdir, readFile } from 'fs/promises';
+import * as path from 'path';
+import { InjectDataSource } from '@nestjs/typeorm';
+
+@Injectable()
+export class DatabaseBootstrapService implements OnModuleInit {
+  private readonly logger = new Logger(DatabaseBootstrapService.name);
+
+  constructor(
+    @InjectDataSource()
+    private readonly dataSource: DataSource,
+  ) { }
+
+  async onModuleInit() {
+    if (!this.dataSource.isInitialized) {
+      this.logger.warn(`[${this.dataSource.name}] DataSource not initialized. Skipping SQL bootstrap.`);
+      return;
+    }
+
+    this.logger.debug(`[${this.dataSource.name}] Bootstrapping stored procedures...`);
+
+    await this.applyAllSqlFiles();
+
+    this.logger.debug(`[${this.dataSource.name}] SQL bootstrap completed`);
+  }
+
+  private resolveSqlDirectory(): string {
+    const datasourceName = this.dataSource.name || 'default';
+    const dbType = this.dataSource.options.type;
+
+    const sqlFilePath = path.resolve(__dirname, `../../../sql/${datasourceName}/${dbType}`);
+
+    return sqlFilePath
+  }
+
+  private async applyAllSqlFiles() {
+    const sqlDir = this.resolveSqlDirectory();
+
+    this.logger.debug(`[${this.dataSource.name}] SQL directory: ${sqlDir}`);
+
+    let files: string[];
+    try {
+      files = await readdir(sqlDir);
+    } catch {
+      this.logger.warn(
+        `[${this.dataSource.name}] No SQL directory found. Skipping.`,
+      );
+      return;
+    }
+
+    const sqlFiles = files
+      .filter(file => file.endsWith('.sql'))
+      .sort();
+
+    if (!sqlFiles.length) {
+      this.logger.warn(
+        `[${this.dataSource.name}] No SQL files found`,
+      );
+      return;
+    }
+
+    for (const file of sqlFiles) {
+      await this.applySqlFileSafely(
+        path.join(sqlDir, file),
+        file,
+      );
+    }
+  }
+
+  private async applySqlFileSafely(
+    filePath: string,
+    fileName: string,
+  ) {
+    this.logger.debug(`[${this.dataSource.name}] Applying ${fileName}`);
+
+    try {
+      const sql = await readFile(filePath, 'utf8');
+      await this.dataSource.query(sql);
+
+      this.logger.debug(`[${this.dataSource.name}] Applied ${fileName}`);
+    } catch (error) {
+      // DO NOT THROW — continue with next file
+      this.logger.error(
+        `[${this.dataSource.name}] Failed ${fileName}`,
+        error instanceof Error ? error.stack : String(error),
+      );
+    }
+  }
+}

package/src/services/email-template.service.ts

@@ -1,25 +1,23 @@
-import { Injectable, NotFoundException } from '@nestjs/common';
+import { forwardRef, Inject, Injectable } from '@nestjs/common';
 
-import { InjectEntityManager, InjectRepository } from '@nestjs/typeorm';
-import { EntityManager, Repository } from 'typeorm';
-import { PaginationQueryDto } from 'src/dtos/pagination-query.dto';
+import { ConfigService } from '@nestjs/config';
+import { DiscoveryService, ModuleRef } from '@nestjs/core';
+import { InjectEntityManager } from '@nestjs/typeorm';
+import { EmailTemplateRepository } from 'src/repository/email-template.repository';
+import { EntityManager } from 'typeorm';
 import { EmailTemplate } from '../entities/email-template.entity';
-import { CreateEmailTemplateDto } from '../dtos/create-email-template.dto';
-import { UpdateEmailTemplateDto } from '../dtos/update-email-template.dto';
+import { CrudHelperService } from './crud-helper.service';
 import { CRUDService } from './crud.service';
-import { ModelMetadataService } from './model-metadata.service';
-import { ModuleMetadataService } from './module-metadata.service';
-import { MediaStorageProviderMetadataService } from './media-storage-provider-metadata.service';
-import { ConfigService } from '@nestjs/config';
 import { FileService } from './file.service';
+import { MediaStorageProviderMetadataService } from './media-storage-provider-metadata.service';
 import { MediaService } from './media.service';
-import { DiscoveryService, ModuleRef } from '@nestjs/core';
-import { CrudHelperService } from './crud-helper.service';
-import { EmailTemplateRepository } from 'src/repository/email-template.repository';
+import { ModelMetadataService } from './model-metadata.service';
+import { ModuleMetadataService } from './module-metadata.service';
 
 @Injectable()
 export class EmailTemplateService extends CRUDService<EmailTemplate>{
     constructor(
+    @Inject(forwardRef(() => ModelMetadataService))
     readonly modelMetadataService: ModelMetadataService,
     readonly moduleMetadataService: ModuleMetadataService,
     readonly mediaStorageProviderService: MediaStorageProviderMetadataService,

package/src/services/excel.service.ts

@@ -3,7 +3,6 @@ import * as ExcelJS from 'exceljs';
 import { ERROR_MESSAGES } from 'src/constants/error-messages';
 import { PassThrough, Readable } from 'stream';
 
-
 export interface ExcelReadOptions {
   pageSize?: number; // Number of records per page
   hasHeaderRow?: boolean; // Whether the first row contains headers
@@ -17,6 +16,11 @@ export interface ExcelReadResult {
   data: Record<string, any>[]; // Data records in the current page
 }
 
+export interface ExcelReadAllResult {
+  headers: string[];
+  rows: Record<string, any>[];
+}
+
 @Injectable()
 export class ExcelService {
   private logger = new Logger(ExcelService.name);
@@ -59,7 +63,7 @@ export class ExcelService {
         //   headers.reduce((acc, header) => ({ ...acc, [header]: '' }), {})
         // ).commit(); // Write a dummy record with headers
 
-         workbook.commit();
+        workbook.commit();
         return passThrough;
       }
 
@@ -86,7 +90,7 @@ export class ExcelService {
         this.logger.debug(`✅ Chunk ${chunkIndex} written to Excel`);
       }
 
-       workbook.commit();
+      workbook.commit();
       // passThrough.end(); // ✅ Properly close the stream
     } catch (error) {
       this.logger.error(`❌ Error writing Excel: ${error.message}`);
@@ -159,4 +163,143 @@ export class ExcelService {
     }
   }
 
+  private cleanString(value: any): string {
+    return (value === null || value === undefined ? '' : String(value))
+      .replace(/\uFEFF/g, '') // BOM
+      .replace(/\u00A0/g, ' ') // NBSP
+      .replace(/\s+/g, ' ')
+      .trim();
+  }
+
+  private normalizeCellValue(value: any, sharedStrings?: any[]): any {
+    if (value === undefined || value === null) return null;
+
+    // ExcelJS streaming shared string ref: { sharedString: number }
+    if (typeof value === 'object' && value && 'sharedString' in value) {
+      const idx = (value as any).sharedString;
+      const resolved = sharedStrings?.[idx];
+      // sharedStrings may store objects or plain strings depending on ExcelJS internals
+      if (resolved === undefined || resolved === null) return null;
+      if (typeof resolved === 'string') return resolved;
+      if (typeof resolved === 'object') {
+        if ('text' in resolved && typeof (resolved as any).text === 'string') return (resolved as any).text;
+        if ('richText' in resolved && Array.isArray((resolved as any).richText)) {
+          return (resolved as any).richText.map((item: any) => item?.text ?? '').join('');
+        }
+        if ('value' in resolved) return (resolved as any).value;
+      }
+      return resolved;
+    }
+
+    // ExcelJS can return rich objects for styled cells; unwrap to plain text/primitive
+    if (typeof value === 'object' && value) {
+      // Plain rich cell: { text: '...' }
+      if ('text' in value && typeof (value as any).text === 'string') return (value as any).text;
+
+      // Rich text: { richText: [{text:'a'}, ...] }
+      if ('richText' in value && Array.isArray((value as any).richText)) {
+        return (value as any).richText.map((item: any) => item?.text ?? '').join('');
+      }
+
+      // Formula cells: { formula: '...', result: ... }
+      if ('result' in value) return (value as any).result;
+      if ('formula' in value) return (value as any).formula;
+
+      // Hyperlinks: { text: '...', hyperlink: '...' }
+      if ('hyperlink' in value && typeof (value as any).hyperlink === 'string') {
+        return (value as any).text ?? (value as any).hyperlink;
+      }
+
+      // Sometimes primitive nested under .value
+      if ('value' in value) return this.normalizeCellValue((value as any).value, sharedStrings);
+    }
+
+    return value;
+  }
+
+  public async readExcelFromStreamNonStreaming(
+    stream: Readable,
+    options?: ExcelReadOptions & { worksheetIndex?: number; maxRows?: number }
+  ): Promise<ExcelReadAllResult> {
+    const {
+      hasHeaderRow = true,
+      providedHeaders = [],
+      worksheetIndex = 0, // 0-based
+      maxRows,
+    } = options || {};
+
+    // 1) Read entire stream into a Buffer
+    const chunks: Buffer[] = [];
+    for await (const chunk of stream) {
+      chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+    }
+    const buffer = Buffer.concat(chunks);
+
+    // 2) Load workbook (non-streaming)
+    const workbook = new ExcelJS.Workbook();
+    // @ts-ignore
+    await workbook.xlsx.load(buffer);
+
+    const worksheet = workbook.worksheets?.[worksheetIndex];
+    if (!worksheet) {
+      return { headers: [], rows: [] };
+    }
+
+    // 3) Determine headers
+    let headers: string[] = [];
+
+    const firstRow = worksheet.getRow(1);
+    const firstRowValues = Array.isArray(firstRow.values) ? (firstRow.values as any[]).slice(1) : [];
+
+    const normalizeNonStreamingCell = (v: any) => {
+      // In non-streaming ExcelJS, cell.value can be:
+      // - string/number/boolean/date
+      // - {richText}, {text}, {hyperlink}, {formula,result}, etc.
+      // We'll reuse your normalizeCellValue but without sharedStrings
+      return this.normalizeCellValue(v);
+    };
+
+    if (hasHeaderRow) {
+      headers = firstRowValues.map((v) => this.cleanString(normalizeNonStreamingCell(v)));
+    } else if (providedHeaders.length) {
+      headers = providedHeaders.map((h) => this.cleanString(h));
+    } else {
+      headers = firstRowValues.map((_, idx) => `${idx}`);
+    }
+
+    // If headers are all blank and hasHeaderRow=true, treat as no headers (avoid mapping everything to "")
+    if (hasHeaderRow && headers.length > 0 && headers.every((h) => !h)) {
+      this.logger.warn(`ExcelService.readExcelFromStreamNonStreaming: header row appears blank`);
+    }
+
+    // 4) Read rows
+    const rows: Record<string, any>[] = [];
+
+    const startRowNumber = hasHeaderRow ? 2 : 1;
+    const lastRowNumber = worksheet.rowCount || 0;
+
+    for (let r = startRowNumber; r <= lastRowNumber; r++) {
+      if (maxRows && rows.length >= maxRows) break;
+
+      const row = worksheet.getRow(r);
+      const rawValues = Array.isArray(row.values) ? (row.values as any[]).slice(1) : [];
+      const values = rawValues.map((v) => normalizeNonStreamingCell(v));
+
+      // Align row width to header width
+      while (values.length < headers.length) values.push(null);
+      if (values.length > headers.length) values.length = headers.length;
+
+      const record = headers.reduce((acc, key, i) => {
+        acc[key] = values[i] ?? null;
+        return acc;
+      }, {} as Record<string, any>);
+
+      // Skip fully empty rows
+      if (Object.values(record).every((v) => v === null || this.cleanString(v) === '')) continue;
+
+      rows.push(record);
+    }
+
+    return { headers, rows };
+  }
 }