@solidstarters/solid-core 1.2.88 → 1.2.90

package/src/services/excel.service.ts

@@ -3,87 +3,69 @@ import * as ExcelJS from 'exceljs';
 import { PassThrough, Readable } from 'stream';
 
 
+export interface ExcelReadOptions {
+  pageSize?: number; // Number of records per page
+  hasHeaderRow?: boolean; // Whether the first row contains headers
+  providedHeaders?: string[]; // Custom headers if hasHeaderRow is false
+}
+
+const DEFAULT_PAGE_SIZE = 100; // Default page size if not provided
+
+export interface ExcelReadResult {
+  headers: string[]; // Headers of the Excel file
+  data: Record<string, any>[]; // Data records in the current page
+}
+
 @Injectable()
 export class ExcelService {
   private logger = new Logger(ExcelService.name);
-  // Sample JSON data
-  // const jsonData = [
-  //   { id: 1, name: 'John Doe', age: 25, email: 'john@example.com' },
-  //   { id: 2, name: 'Jane Doe', age: 28, email: 'jane@example.com' },
-  //   { id: 3, name: 'Alice Smith', age: 30, email: 'alice@example.com' }
-  // ];
-
-  // public async createExcelFromJson(data: any[], fileName: string) {
-  //   const workbook = new ExcelJS.Workbook();
-  //   const worksheet = workbook.addWorksheet('Data');
-
-  //   // Define Columns (Header)
-  //   worksheet.columns = Object.keys(data[0]).map((key) => ({
-  //     header: key.toUpperCase(), // Convert header names to uppercase
-  //     key: key,
-  //     width: 20, // Set column width
-  //   }));
-
-  //   // Add Data Rows
-  //   data.forEach((item) => {
-  //     worksheet.addRow(item);
-  //   });
-
-  //   // Apply basic formatting
-  //   worksheet.getRow(1).font = { bold: true }; // Make headers bold
-
-  //   // Save file
-  //   await workbook.xlsx.writeFile(fileName);
-  //   this.logger.log(`✅ Excel file "${fileName}" created successfully!`);
-  //   // console.log(`✅ Excel file "${fileName}" created successfully!`);
-  // }
-
-  // public async createExcelStreamFromJson(data: any[]): Promise<Readable> {
-  //   const passThrough = new PassThrough(); // Stream to pipe data
-  //   const workbook = new ExcelJS.stream.xlsx.WorkbookWriter({ stream: passThrough });
-  //   const worksheet = workbook.addWorksheet('Data');
-  //   worksheet.columns = Object.keys(data[0]).map((key) => ({
-  //     header: key.toUpperCase(),
-  //     key: key,
-  //     width: 20,
-  //   }));
-
-  //   data.forEach((item) => {
-  //     worksheet.addRow(item);
-  //   });
-
-  //   worksheet.getRow(1).font = { bold: true };
-
-  //   await workbook.commit();
-  //   return passThrough;
-  // }
-
 
   public async createExcelStream(
     getDataRecords: (chunkIndex: number, chunkSize: number) => Promise<any[]>,
-    chunkSize: number
+    chunkSize: number = 100,
+    headers: string[] = []
   ): Promise<Readable> {
+    // Validations
+    // If neither headers nor data records function is provided, throw an error
+    if (headers.length === 0 && typeof getDataRecords !== 'function') {
+      throw new Error('Either headers or data records function must be provided.');
+    }
+
+    // If data records function is provided, chunkSize must be greater than 0
+    if (getDataRecords && chunkSize <= 0) {
+      throw new Error('Chunk size must be greater than 0 when data records function is provided.');
+    }
+
     const passThrough = new PassThrough(); // Create streaming pipe
     try {
       const workbook = new ExcelJS.stream.xlsx.WorkbookWriter({ stream: passThrough });
       const worksheet = workbook.addWorksheet('Data');
 
-      let chunkIndex = 0;
-      let isHeaderWritten = false;
+      // If headers are provided, use them;
+      if (headers.length > 0) {
+        worksheet.columns = headers.map((header) => ({
+          header: header, // Convert header names to uppercase
+          key: header,
+          width: 20, // Set column width
+        }));
+      }
+
+      // ✅ If no data loader provided, write only headers and finish
+      if (typeof getDataRecords !== 'function') {
+        // worksheet.addRow(
+        //   headers.reduce((acc, header) => ({ ...acc, [header]: '' }), {})
+        // ).commit(); // Write a dummy record with headers
+
+        await workbook.commit();
+        return passThrough;
+      }
 
+      // Write the data records in chunks
+      let chunkIndex = 0;
       while (true) {
         const records = await getDataRecords(chunkIndex, chunkSize); // Fetch chunked data
         if (records.length === 0) break; // Stop if no more records
 
-        if (!isHeaderWritten) {
-          worksheet.columns = Object.keys(records[0]).map((key) => ({
-            header: key.toUpperCase(),
-            key: key,
-            width: 20,
-          }));
-          isHeaderWritten = true;
-        }
-
         records.forEach((item) => {
           worksheet.addRow(item).commit(); // Commit each row immediately
         });
@@ -102,4 +84,67 @@ export class ExcelService {
     return passThrough; // Return streaming response
   }
 
+  public async *readExcelInPagesFromStream(
+    stream: Readable,
+    options?: ExcelReadOptions
+  ): AsyncGenerator<ExcelReadResult> {
+    const { pageSize = DEFAULT_PAGE_SIZE, hasHeaderRow = true, providedHeaders = [] } = options || {};
+    const workbookReader = new ExcelJS.stream.xlsx.WorkbookReader(stream, {});
+
+    let headers: string[] = [];
+    let page: Record<string, any>[] = [];
+    let isFirstRow = true;
+    let hasYieldedData = false;
+
+    for await (const worksheet of workbookReader) {
+      for await (const row of worksheet) {
+        const values = Array.isArray(row.values) ? row.values.slice(1) : [];
+
+        if (isFirstRow) {
+          isFirstRow = false;
+
+          if (hasHeaderRow) {
+            headers = values.map(v => v?.toString().trim() || '');
+            continue;
+          } else if (providedHeaders.length) {
+            headers = providedHeaders;
+          } else {
+            headers = values.map((_, idx) => `${idx}`);
+          }
+        }
+
+        while (values.length < headers.length) values.push(null);
+        if (values.length > headers.length) values.length = headers.length;
+
+        const record = headers.reduce((acc, key, i) => {
+          acc[key] = values[i] ?? null;
+          return acc;
+        }, {} as Record<string, any>);
+
+        if (Object.values(record).every(v => v === null || v === '')) continue;
+
+        page.push(record);
+
+        if (page.length === pageSize) {
+          yield { headers, data: page };
+          hasYieldedData = true;
+          page = [];
+        }
+      }
+
+      // Optional: break if only processing first worksheet
+      // break;
+    }
+
+    if (page.length > 0) {
+      yield { headers, data: page };
+      hasYieldedData = true;
+    }
+
+    // ✅ Yield headers with empty data if only headers were found
+    if (!hasYieldedData && headers.length > 0) {
+      yield { headers, data: [] };
+    }
+  }
+
 }

package/src/services/export-template.service.ts

@@ -18,35 +18,36 @@ import { ExportTransaction } from 'src/entities/export-transaction.entity';
 import { Readable } from 'stream';
 import { ExportTemplate } from '../entities/export-template.entity';
 import { ExportTransactionFileInfo, ExportTransactionService } from './export-transaction.service';
+import { UpdateExportTemplateDto } from 'src/dtos/update-export-template.dto';
 
 @Injectable()
 export class ExportTemplateService extends CRUDService<ExportTemplate>{
-  async startExportSync(id: number): Promise<ExportTransactionFileInfo> {
+  async startExportSync(updateDto: UpdateExportTemplateDto, filters:any): Promise<ExportTransactionFileInfo> {
     // Create the export transaction entry, with status 'started'
     const exportTransaction: CreateExportTransactionDto =  await this.exportTransactionService.toDto({
       datetime: new Date(),
       status: 'started',
-      exportTemplateId: id,
+      exportTemplateId: updateDto?.id ? updateDto?.id : null,
     });
     const exportTransactionEntity = await this.exportTransactionService.create(exportTransaction);
 
     // Trigger the export process
-    const exportFileInfo = await this.exportTransactionService.triggerExportSync(exportTransactionEntity.id);
+    const exportFileInfo = await this.exportTransactionService.triggerExportSync(exportTransactionEntity.id, exportTransactionEntity, updateDto, filters);
     // It should return the export transaction id
     return exportFileInfo;
   }
 
-  async startExportAsync(id: number): Promise<ExportTransaction>{
+  async startExportAsync(updateDto: UpdateExportTemplateDto,  filters:any): Promise<ExportTransaction>{
     // Create the export transaction entry, with status 'started'
     const exportTransaction: CreateExportTransactionDto =  await this.exportTransactionService.toDto({
       datetime: new Date(),
       status: 'started',
-      exportTemplateId: id,
+      exportTemplateId: updateDto?.id ? updateDto?.id : null,
     });
     const exportTransactionEntity = await this.exportTransactionService.create(exportTransaction);
 
     // Trigger the export process
-    this.exportTransactionService.triggerExportAsync(exportTransactionEntity.id);
+    this.exportTransactionService.triggerExportAsync(exportTransactionEntity.id, exportTransactionEntity, updateDto, filters);
 
     // It should return the export transaction id, so client can use this to check the status
     return exportTransactionEntity;

package/src/services/export-transaction.service.ts

@@ -26,6 +26,8 @@ import { CsvService } from './csv.service';
 import { ExcelService } from './excel.service';
 import { getMediaStorageProvider } from './mediaStorageProviders';
 import { SolidIntrospectService } from './solid-introspect.service';
+import { ModelMetadata } from 'src/entities/model-metadata.entity';
+import { UpdateExportTemplateDto } from 'src/dtos/update-export-template.dto';
 
 const EXPORT_CHUNK_SIZE = 100;
 enum ExportStatus {
@@ -67,19 +69,30 @@ export class ExportTransactionService extends CRUDService<ExportTransaction> {
     // readonly fieldMetadataService: FieldMetadataService,
     @InjectRepository(FieldMetadata, 'default')
     readonly fieldRepo: Repository<FieldMetadata>,
+    @InjectRepository(ModelMetadata, 'default')
+    readonly ModelMetadataRepo : Repository<ModelMetadata>,
     readonly moduleRef: ModuleRef
   ) {
     super(modelMetadataService, moduleMetadataService, configService, fileService, discoveryService, crudHelperService, entityManager, repo, 'exportTransaction', 'solid-core',moduleRef);
   }
 
   // Return the export stream
-  async triggerExportSync(id: number): Promise<ExportTransactionFileInfo> {
+  async triggerExportSync(id: number, exportTransactionEntity: any, updateDto: UpdateExportTemplateDto , filters: any): Promise<ExportTransactionFileInfo> {
     try {
-      const loadedExportTransaction = await this.loadExportTransaction(id);
-      const { exportStream, templateName, uuid, exportTransaction } = await this.getExportStreamDetails(loadedExportTransaction);
+      // const loadedExportTransaction = await this.loadExportTransaction(id);
+      // from updateDto, get modelId and get modelMetadata
+      const modeldata = await this.ModelMetadataRepo.findOne({
+        where: { id:  updateDto?.modelMetadataId},
+        relations: { fields: true},
+      })
+      const modelName = modeldata?.singularName;
+      const modelTemplateName = modelName;
+      const fields = JSON.parse(updateDto?.fields);
+      const templateFormat = updateDto?.templateFormat;
+      const { exportStream, templateName, uuid, exportTransaction } = await this.getExportStreamDetails(modelName, modelTemplateName, fields, modeldata, templateFormat, id, exportTransactionEntity, filters);
       this.updateExportTransaction(id, ExportStatus.COMPLETED);
-      const fileName = this.getFileName(templateName, uuid, loadedExportTransaction.exportTemplate.templateFormat);
-      const mimeType = this.getMimeType(loadedExportTransaction.exportTemplate.templateFormat);
+      const fileName = this.getFileName(templateName, uuid, templateFormat);
+      const mimeType = this.getMimeType(templateFormat);
       return { exportStream, fileName, mimeType, exportTransaction };
     } catch (error) {
       this.updateExportTransaction(id, ExportStatus.FAILED, error.message);
@@ -88,13 +101,22 @@ export class ExportTransactionService extends CRUDService<ExportTransaction> {
   }
 
   // Store the export stream using the appropriate storage provider
-  async triggerExportAsync(id: number): Promise<void> {
+  async triggerExportAsync(id: number, exportTransactionEntity: any, updateDto: UpdateExportTemplateDto, filters:any): Promise<void> {
     try {
-      const loadedExportTransaction = await this.loadExportTransaction(id)
-      const { exportStream, templateName, uuid, exportTransaction } = await this.getExportStreamDetails(loadedExportTransaction);
-      const fileFormat = loadedExportTransaction.exportTemplate.templateFormat;
+      // const loadedExportTransaction = await this.loadExportTransaction(id)
+      // from updateDto, get modelId and get modelMetadata
+      const modeldata = await this.ModelMetadataRepo.findOne({
+        where: { id:  updateDto?.modelMetadataId},
+        relations: { fields: true},
+      })
+      const modelName = modeldata?.singularName;
+      const modelTemplateName = modelName;
+      const fields = JSON.parse(updateDto?.fields);
+      const templateFormat = updateDto?.templateFormat;
+      const { exportStream, templateName, uuid, exportTransaction } = await this.getExportStreamDetails(modelName, modelTemplateName, fields, modeldata, templateFormat, id, exportTransactionEntity, filters);
+      // const fileFormat = loadedExportTransaction.exportTemplate.templateFormat;
       // Store the file using the appropriate storage provider
-      await this.storeExportStream(exportStream, exportTransaction, this.getFileName(templateName, uuid, fileFormat));
+      await this.storeExportStream(exportStream, exportTransaction, this.getFileName(templateName, uuid, templateFormat));
       this.updateExportTransaction(id, ExportStatus.COMPLETED);
     } catch (error) {
       this.updateExportTransaction(id, ExportStatus.FAILED, error.message);
@@ -106,7 +128,7 @@ export class ExportTransactionService extends CRUDService<ExportTransaction> {
   private async loadExportTransaction(id: number) {
     return await this.repo.findOne({
       where: { id: id },
-      relations: { exportTemplate: { modelMetadata: true } },
+      relations: { exportTemplate: { modelMetadata: {fields: true} }},
     }
     );
   }
@@ -115,22 +137,26 @@ export class ExportTransactionService extends CRUDService<ExportTransaction> {
     await this.repo.update(id, { status, error });
   }
 
-  private async getExportStreamDetails(exportTransaction: ExportTransaction) {
+  private async getExportStreamDetails(modelName: string, templateName: string, fields:any, modelData:any, templateFormat:string, id:number, exportTransaction: any, filters: any) {
     // Get the columns which need to be exported & the model id
-    const fields = JSON.parse(exportTransaction.exportTemplate.fields);
-
-    // Get the appropriate service for the model by trying to fetch a model service matching a particular name
-    const modelName = exportTransaction.exportTemplate.modelMetadata.singularName;
-    const modelService = this.introspectService.getProvider(`${classify(modelName)}Service`);
-    const templateName = exportTransaction.exportTemplate.templateName;
-    const uuid = exportTransaction.exportTransactionId; //TODO can be renamed to exportTransactionUUID
+    // const fields = JSON.parse(exportTransaction.exportTemplate.fields);
 
+    // // Get the appropriate service for the model by trying to fetch a model service matching a particular name
+    // const modelName = exportTransaction.exportTemplate.modelMetadata.singularName;
+     const modelService = this.introspectService.getProvider(`${classify(modelName)}Service`);
+    // const templateName = exportTransaction.exportTemplate.templateName;
+     const uuid = String(id); //TODO can be renamed to exportTransactionUUID
+    // const modelData = exportTransaction.exportTemplate.modelMetadata;
 
     // Get the data records function
-    const dataRecordsFunc = await this.getDataRecordsFunc(fields, modelService);
+    //const dataRecordsFunc = await this.getDataRecordsFunc(fields, modelService,modelData, filters);
+    const dataRecordsFunc = await this.getDataRecordsFunc(fields, modelService, modelData, filters);
 
     // Get the export passthru stream (since it is a passthru stream, nothing is stored in memory & it is streamed directly when the stream is read)
-    let exportStream = await this.getExportStream(exportTransaction.exportTemplate.templateFormat, dataRecordsFunc);
+    // let exportStream = await this.getExportStream(exportTransaction.exportTemplate.templateFormat, dataRecordsFunc);
+    // return { exportStream, templateName, uuid, exportTransaction };
+
+    let exportStream = await this.getExportStream(templateFormat, dataRecordsFunc);
     return { exportStream, templateName, uuid, exportTransaction };
   }
 
@@ -181,20 +207,78 @@ export class ExportTransactionService extends CRUDService<ExportTransaction> {
     return (fileFormat === ExportFormat.EXCEL) ? 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' : 'text/csv';
   }
 
-  private async getDataRecordsFunc(fields: any, modelService: InstanceWrapper<any>): Promise<(chunkIndex: number, chunkSize: number) => Promise<any[]>> {
+  private async getDataRecordsFunc(fields: any, modelService: InstanceWrapper<any>, modelMetadata: any, filters:any): Promise<(chunkIndex: number, chunkSize: number) => Promise<any[]>> {
     // Return a function which will take the chunkIndex & chunkSize and return the data
+    // Get the relation fields to populate
+    const relatedFieldNames = modelMetadata?.fields
+    .filter((field: { relationType: any; }) => field.relationType !== null)
+    .map((field: { name: any; }) => field.name);
+
+    //Get the model metadata of relation field with userKey details
+    const relatedModelsUserKeyMap = new Map<string, string>();
+    for (const field of modelMetadata?.fields || []) {
+      if (field.relationType && field.relationCoModelSingularName && fields.includes(field.name)) {
+        const relatedModelMetadata = await this.ModelMetadataRepo.findOne({
+          where: { singularName: field.relationCoModelSingularName },
+          relations: ['userKeyField'],
+        });
+
+        if (relatedModelMetadata?.userKeyField?.name) {
+          relatedModelsUserKeyMap.set(field.name, relatedModelMetadata.userKeyField.name);
+        }
+      }
+    }
+
     return async (chunkIndex: number, chunkSize: number) => {
       const offset = chunkIndex * chunkSize;
       const recordFilterDto: BasicFilterDto = {
-        fields,
         limit: chunkSize,
         offset,
+        populate: relatedFieldNames
       };
+      const cleanedFilters = cleanNullsFromObject(filters);
+
+      if (cleanedFilters && Object.keys(cleanedFilters).length > 0) {
+        recordFilterDto.filters = cleanedFilters;
+      }
+
+      //Get the non relation fields which are in fields array passed to this function
+      const nonRelationalFieldSet = new Set(
+        modelMetadata?.fields
+          .filter((field: { name: any; relationType: any; }) => fields.includes(field.name) && field.relationType === null)
+          .map((field: { name: any; }) => field.name)
+      );
       const data = await modelService.instance.find(recordFilterDto);
       const records = data.records ?? [];
-      return records;
-    }
+    const cleanedRecords = records.map((record: Record<string, any>) => {
+      const newRecord: Record<string, any> = {};
+    
+      // Include non-relational fields
+      for (const key of nonRelationalFieldSet as Set<string>) {
+        newRecord[key] = record[key];
+      }
+    
+      // Include userKey from each related field
+      for (const [relatedFieldName, userKeyFieldName] of relatedModelsUserKeyMap.entries()) {
+        const relatedData = record[relatedFieldName];
+    
+        if (Array.isArray(relatedData)) {
+          // For many-to-many or one-to-many
+          const values = relatedData.map(item => item?.[userKeyFieldName]).filter(Boolean);
+          newRecord[relatedFieldName] = values.join(', ');
+        } else if (relatedData && typeof relatedData === 'object') {
+          // For many-to-one or one-to-one
+          newRecord[relatedFieldName] = relatedData?.[userKeyFieldName] ?? null;
+        } else {
+          newRecord[relatedFieldName] = null;
+        }
+      }
+    
+      return newRecord;
+    });
+    return cleanedRecords
   }
+}
 
   async toDto(data: Partial<CreateExportTransactionDto>): Promise<CreateExportTransactionDto> {
     const dto = new CreateExportTransactionDto(data);
@@ -206,3 +290,30 @@ export class ExportTransactionService extends CRUDService<ExportTransaction> {
     return dto;
   }
 }
+
+function cleanNullsFromObject(obj: any): any {
+  if (Array.isArray(obj)) {
+    return obj
+      .filter(item => item !== null && item !== undefined)
+      .map(cleanNullsFromObject);
+  } else if (typeof obj === 'object' && obj !== null) {
+    const newObj: any = {};
+    for (const key in obj) {
+      const value = obj[key];
+      if (value !== null && value !== undefined) {
+        const cleanedValue = cleanNullsFromObject(value);
+        // Only assign non-empty objects/arrays or non-null primitives
+        if (
+          (typeof cleanedValue === 'object' && Object.keys(cleanedValue).length > 0) ||
+          (Array.isArray(cleanedValue) && cleanedValue.length > 0) ||
+          typeof cleanedValue !== 'object'
+        ) {
+          newObj[key] = cleanedValue;
+        }
+      }
+    }
+    return newObj;
+  }
+  return obj;
+}
+

package/src/services/export_issues.txt

@@ -3,4 +3,8 @@
 2. queues handling -> if queues is enabled by default, i.e triggerExport(exportTransactionEntity.id).
     - startExport should either return the data or return the transaction id
 3. How to handle scenarios wherein, nested related exist.(do i need to only get the userkey)
-    - show the userKey
+    - show the userKey
+4. Exclude the media fields from the fields shown in the template creation
+5. Allow provision to export the data without creating a template
+6. Also for template creation i.e do we want the option to keep the templates at a user level instead of at the application level.
+7. Do we want to save the filter against the template.

package/src/services/import-transaction-error-log.service.ts

@@ -0,0 +1,34 @@
+import { Injectable } from '@nestjs/common';
+import { InjectEntityManager, InjectRepository } from '@nestjs/typeorm';
+import { DiscoveryService, ModuleRef  } from "@nestjs/core";
+import { EntityManager, Repository } from 'typeorm';
+
+import { CRUDService } from 'src/services/crud.service';
+import { ModelMetadataService } from 'src/services/model-metadata.service';
+import { ModuleMetadataService } from 'src/services/module-metadata.service';
+import { ConfigService } from '@nestjs/config';
+import { FileService } from 'src/services/file.service';
+import { CrudHelperService } from 'src/services/crud-helper.service';
+
+
+import { ImportTransactionErrorLog } from '../entities/import-transaction-error-log.entity';
+
+@Injectable()
+export class ImportTransactionErrorLogService extends CRUDService<ImportTransactionErrorLog>{
+  constructor(
+    readonly modelMetadataService: ModelMetadataService,
+    readonly moduleMetadataService: ModuleMetadataService,
+    readonly configService: ConfigService,
+    readonly fileService: FileService,
+    readonly discoveryService: DiscoveryService,
+    readonly crudHelperService: CrudHelperService,
+    @InjectEntityManager()
+    readonly entityManager: EntityManager,
+    @InjectRepository(ImportTransactionErrorLog, 'default')
+    readonly repo: Repository<ImportTransactionErrorLog>,
+    readonly moduleRef: ModuleRef
+
+ ) {
+   super(modelMetadataService, moduleMetadataService,  configService, fileService,  discoveryService, crudHelperService,entityManager, repo, 'importTransactionErrorLog', 'solid-core', moduleRef);
+ }
+}