@solidstarters/solid-core 1.2.89 → 1.2.90

package/src/services/excel.service.ts

@@ -3,87 +3,69 @@ import * as ExcelJS from 'exceljs';
 import { PassThrough, Readable } from 'stream';
 
 
+export interface ExcelReadOptions {
+  pageSize?: number; // Number of records per page
+  hasHeaderRow?: boolean; // Whether the first row contains headers
+  providedHeaders?: string[]; // Custom headers if hasHeaderRow is false
+}
+
+const DEFAULT_PAGE_SIZE = 100; // Default page size if not provided
+
+export interface ExcelReadResult {
+  headers: string[]; // Headers of the Excel file
+  data: Record<string, any>[]; // Data records in the current page
+}
+
 @Injectable()
 export class ExcelService {
   private logger = new Logger(ExcelService.name);
-  // Sample JSON data
-  // const jsonData = [
-  //   { id: 1, name: 'John Doe', age: 25, email: 'john@example.com' },
-  //   { id: 2, name: 'Jane Doe', age: 28, email: 'jane@example.com' },
-  //   { id: 3, name: 'Alice Smith', age: 30, email: 'alice@example.com' }
-  // ];
-
-  // public async createExcelFromJson(data: any[], fileName: string) {
-  //   const workbook = new ExcelJS.Workbook();
-  //   const worksheet = workbook.addWorksheet('Data');
-
-  //   // Define Columns (Header)
-  //   worksheet.columns = Object.keys(data[0]).map((key) => ({
-  //     header: key.toUpperCase(), // Convert header names to uppercase
-  //     key: key,
-  //     width: 20, // Set column width
-  //   }));
-
-  //   // Add Data Rows
-  //   data.forEach((item) => {
-  //     worksheet.addRow(item);
-  //   });
-
-  //   // Apply basic formatting
-  //   worksheet.getRow(1).font = { bold: true }; // Make headers bold
-
-  //   // Save file
-  //   await workbook.xlsx.writeFile(fileName);
-  //   this.logger.log(`✅ Excel file "${fileName}" created successfully!`);
-  //   // console.log(`✅ Excel file "${fileName}" created successfully!`);
-  // }
-
-  // public async createExcelStreamFromJson(data: any[]): Promise<Readable> {
-  //   const passThrough = new PassThrough(); // Stream to pipe data
-  //   const workbook = new ExcelJS.stream.xlsx.WorkbookWriter({ stream: passThrough });
-  //   const worksheet = workbook.addWorksheet('Data');
-  //   worksheet.columns = Object.keys(data[0]).map((key) => ({
-  //     header: key.toUpperCase(),
-  //     key: key,
-  //     width: 20,
-  //   }));
-
-  //   data.forEach((item) => {
-  //     worksheet.addRow(item);
-  //   });
-
-  //   worksheet.getRow(1).font = { bold: true };
-
-  //   await workbook.commit();
-  //   return passThrough;
-  // }
-
 
   public async createExcelStream(
     getDataRecords: (chunkIndex: number, chunkSize: number) => Promise<any[]>,
-    chunkSize: number
+    chunkSize: number = 100,
+    headers: string[] = []
   ): Promise<Readable> {
+    // Validations
+    // If neither headers nor data records function is provided, throw an error
+    if (headers.length === 0 && typeof getDataRecords !== 'function') {
+      throw new Error('Either headers or data records function must be provided.');
+    }
+
+    // If data records function is provided, chunkSize must be greater than 0
+    if (getDataRecords && chunkSize <= 0) {
+      throw new Error('Chunk size must be greater than 0 when data records function is provided.');
+    }
+
     const passThrough = new PassThrough(); // Create streaming pipe
     try {
       const workbook = new ExcelJS.stream.xlsx.WorkbookWriter({ stream: passThrough });
       const worksheet = workbook.addWorksheet('Data');
 
-      let chunkIndex = 0;
-      let isHeaderWritten = false;
+      // If headers are provided, use them;
+      if (headers.length > 0) {
+        worksheet.columns = headers.map((header) => ({
+          header: header, // Convert header names to uppercase
+          key: header,
+          width: 20, // Set column width
+        }));
+      }
+
+      // ✅ If no data loader provided, write only headers and finish
+      if (typeof getDataRecords !== 'function') {
+        // worksheet.addRow(
+        //   headers.reduce((acc, header) => ({ ...acc, [header]: '' }), {})
+        // ).commit(); // Write a dummy record with headers
+
+        await workbook.commit();
+        return passThrough;
+      }
 
+      // Write the data records in chunks
+      let chunkIndex = 0;
       while (true) {
         const records = await getDataRecords(chunkIndex, chunkSize); // Fetch chunked data
         if (records.length === 0) break; // Stop if no more records
 
-        if (!isHeaderWritten) {
-          worksheet.columns = Object.keys(records[0]).map((key) => ({
-            header: key.toUpperCase(),
-            key: key,
-            width: 20,
-          }));
-          isHeaderWritten = true;
-        }
-
         records.forEach((item) => {
           worksheet.addRow(item).commit(); // Commit each row immediately
         });
@@ -102,4 +84,67 @@ export class ExcelService {
     return passThrough; // Return streaming response
   }
 
+  public async *readExcelInPagesFromStream(
+    stream: Readable,
+    options?: ExcelReadOptions
+  ): AsyncGenerator<ExcelReadResult> {
+    const { pageSize = DEFAULT_PAGE_SIZE, hasHeaderRow = true, providedHeaders = [] } = options || {};
+    const workbookReader = new ExcelJS.stream.xlsx.WorkbookReader(stream, {});
+
+    let headers: string[] = [];
+    let page: Record<string, any>[] = [];
+    let isFirstRow = true;
+    let hasYieldedData = false;
+
+    for await (const worksheet of workbookReader) {
+      for await (const row of worksheet) {
+        const values = Array.isArray(row.values) ? row.values.slice(1) : [];
+
+        if (isFirstRow) {
+          isFirstRow = false;
+
+          if (hasHeaderRow) {
+            headers = values.map(v => v?.toString().trim() || '');
+            continue;
+          } else if (providedHeaders.length) {
+            headers = providedHeaders;
+          } else {
+            headers = values.map((_, idx) => `${idx}`);
+          }
+        }
+
+        while (values.length < headers.length) values.push(null);
+        if (values.length > headers.length) values.length = headers.length;
+
+        const record = headers.reduce((acc, key, i) => {
+          acc[key] = values[i] ?? null;
+          return acc;
+        }, {} as Record<string, any>);
+
+        if (Object.values(record).every(v => v === null || v === '')) continue;
+
+        page.push(record);
+
+        if (page.length === pageSize) {
+          yield { headers, data: page };
+          hasYieldedData = true;
+          page = [];
+        }
+      }
+
+      // Optional: break if only processing first worksheet
+      // break;
+    }
+
+    if (page.length > 0) {
+      yield { headers, data: page };
+      hasYieldedData = true;
+    }
+
+    // ✅ Yield headers with empty data if only headers were found
+    if (!hasYieldedData && headers.length > 0) {
+      yield { headers, data: [] };
+    }
+  }
+
 }

package/src/services/export_issues.txt

@@ -3,4 +3,8 @@
 2. queues handling -> if queues is enabled by default, i.e triggerExport(exportTransactionEntity.id).
     - startExport should either return the data or return the transaction id
 3. How to handle scenarios wherein, nested related exist.(do i need to only get the userkey)
-    - show the userKey
+    - show the userKey
+4. Exclude the media fields from the fields shown in the template creation
+5. Allow provision to export the data without creating a template
+6. Also for template creation i.e do we want the option to keep the templates at a user level instead of at the application level.
+7. Do we want to save the filter against the template.

package/src/services/import-transaction-error-log.service.ts

@@ -0,0 +1,34 @@
+import { Injectable } from '@nestjs/common';
+import { InjectEntityManager, InjectRepository } from '@nestjs/typeorm';
+import { DiscoveryService, ModuleRef  } from "@nestjs/core";
+import { EntityManager, Repository } from 'typeorm';
+
+import { CRUDService } from 'src/services/crud.service';
+import { ModelMetadataService } from 'src/services/model-metadata.service';
+import { ModuleMetadataService } from 'src/services/module-metadata.service';
+import { ConfigService } from '@nestjs/config';
+import { FileService } from 'src/services/file.service';
+import { CrudHelperService } from 'src/services/crud-helper.service';
+
+
+import { ImportTransactionErrorLog } from '../entities/import-transaction-error-log.entity';
+
+@Injectable()
+export class ImportTransactionErrorLogService extends CRUDService<ImportTransactionErrorLog>{
+  constructor(
+    readonly modelMetadataService: ModelMetadataService,
+    readonly moduleMetadataService: ModuleMetadataService,
+    readonly configService: ConfigService,
+    readonly fileService: FileService,
+    readonly discoveryService: DiscoveryService,
+    readonly crudHelperService: CrudHelperService,
+    @InjectEntityManager()
+    readonly entityManager: EntityManager,
+    @InjectRepository(ImportTransactionErrorLog, 'default')
+    readonly repo: Repository<ImportTransactionErrorLog>,
+    readonly moduleRef: ModuleRef
+
+ ) {
+   super(modelMetadataService, moduleMetadataService,  configService, fileService,  discoveryService, crudHelperService,entityManager, repo, 'importTransactionErrorLog', 'solid-core', moduleRef);
+ }
+}

package/src/services/import-transaction.service.ts

@@ -0,0 +1,281 @@
+import { Injectable, Logger } from '@nestjs/common';
+import { DiscoveryService, ModuleRef } from "@nestjs/core";
+import { InjectEntityManager, InjectRepository } from '@nestjs/typeorm';
+import { EntityManager, Repository } from 'typeorm';
+
+import { ConfigService } from '@nestjs/config';
+import { CrudHelperService } from 'src/services/crud-helper.service';
+import { CRUDService } from 'src/services/crud.service';
+import { FileService } from 'src/services/file.service';
+import { ModelMetadataService } from 'src/services/model-metadata.service';
+import { ModuleMetadataService } from 'src/services/module-metadata.service';
+
+
+import { HttpService } from '@nestjs/axios';
+import { SolidFieldType } from 'src/dtos/create-field-metadata.dto';
+import { ImportInstructionsResponseDto, StandardImportInstructionsResponseDto } from 'src/dtos/import-instructions.dto';
+import { FieldMetadata } from 'src/entities/field-metadata.entity';
+import { MediaWithFullUrl } from 'src/interfaces';
+import { Readable } from 'stream';
+import { ImportTransaction } from '../entities/import-transaction.entity';
+import { CsvService } from './csv.service';
+import { ExcelService } from './excel.service';
+
+interface ImportTemplateFileInfo {
+  stream: NodeJS.ReadableStream;
+  fileName: string;
+  mimeType: string;
+}
+
+export enum ImportFormat {
+  CSV = 'csv',
+  EXCEL = 'excel',
+}
+export interface ImportMappingInfo {
+  sampleImportedRecordInfo: SampleImportedRecordInfo[];
+  importableFields: ImportableFieldInfo[];
+}
+export interface SampleImportedRecordInfo {
+  cellHeader: string; // The header of the cell in the import file
+  cellValue: string; // The value of the cell in the import file
+  defaultMappedFieldName: string; // The default mapped field name in the model metadata
+}
+export interface ImportableFieldInfo {
+  name: string;
+  displayName: string;
+}
+
+export interface ImportReadResult {
+  headers: string[]; // Headers of the CSV file
+  data: Record<string, any>[]; // Data records in the current page
+}
+
+@Injectable()
+export class ImportTransactionService extends CRUDService<ImportTransaction> {
+  constructor(
+    readonly modelMetadataService: ModelMetadataService,
+    readonly moduleMetadataService: ModuleMetadataService,
+    readonly configService: ConfigService,
+    readonly fileService: FileService,
+    readonly discoveryService: DiscoveryService,
+    readonly crudHelperService: CrudHelperService,
+    @InjectEntityManager()
+    readonly entityManager: EntityManager,
+    @InjectRepository(ImportTransaction, 'default')
+    readonly repo: Repository<ImportTransaction>,
+    readonly moduleRef: ModuleRef,
+    readonly excelService: ExcelService,
+    readonly csvService: CsvService,
+    readonly httpService: HttpService
+
+  ) {
+    super(modelMetadataService, moduleMetadataService, configService, fileService, discoveryService, crudHelperService, entityManager, repo, 'importTransaction', 'solid-core', moduleRef);
+  }
+
+  private readonly logger = new Logger(ImportTransactionService.name);
+  saveImportMapping(arg0: number) {
+    throw new Error('Method not implemented.');
+  }
+
+  async getImportMappingInfo(importTransactionId: number): Promise<ImportMappingInfo> {
+    // Load the import transaction for the given ID
+    const importTransaction = await this.findOne(importTransactionId, {
+      populate: ['modelMetadata', 'modelMetadata.fields'],
+      populateMedia: ['fileLocation'],
+    });
+    if (!importTransaction) {
+      throw new Error(`Import transaction with ID ${importTransactionId} not found.`);
+    }
+
+    // Get all the importable fields from the model metadata
+    const importableFields: ImportableFieldInfo[] = this.fieldsAllowedForImport(importTransaction.modelMetadata.fields).map(field => ({
+      name: field.name,
+      displayName: field.displayName,
+    }));
+
+    // Get the import file stream for the import transaction
+    const importFileMediaObject = importTransaction['_media']['fileLocation'][0] as MediaWithFullUrl; // Since there can be only one fileLocation, we can safely access the first element
+    if (!importFileMediaObject) {
+      throw new Error(`Import file for transaction ID ${importTransactionId} not found.`);
+    }
+    const importFileStream = await this.getImportFileStream(importFileMediaObject);
+
+    // Get a sample of records from the import file
+    const sampleRecord = await this.getFileRecordsSample(importFileStream, importFileMediaObject.mimeType);
+
+    // Convert sampleRecord to the format required for SampleImportedRecordInfo
+    const wrappedRecords: SampleImportedRecordInfo[] = sampleRecord.data.map((record: Record<string, any>) => {
+      return Object.entries(record).map(([key, value]) => ({
+        cellHeader: key,
+        cellValue: value,
+        defaultMappedFieldName: importableFields.find(field => field.displayName === key)?.name || '',
+      }));
+    }).flat();
+
+    //     for await (const page of this.csvService.readCsvInPagesFromStream(importFileStream)) {
+    //   // await dbService.bulkInsert(page);
+    // }
+
+    return {
+      sampleImportedRecordInfo: wrappedRecords, // This will hold the sample data from the file
+      importableFields: importableFields, // This will hold the fields that can be imported
+    };
+  }
+
+  private async getFileRecordsSample(importFileStream: Readable, mimeType: string): Promise<ImportReadResult> {
+    // Depending upon the mime type of the file, read the file in pages
+    // For CSV files, use the csvService to read the file in pages
+    // For Excel files, use the excelService to read the file in pages
+    if (mimeType === 'text/csv') {
+      const generator = this.csvService.readCsvInPagesFromStream(importFileStream, { pageSize: 1 });
+      const firstRecord = await generator.next(); // Get the first record to extract headers and sample data
+      return firstRecord.value;
+    } else if (mimeType === 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet') {
+      const generator = this.excelService.readExcelInPagesFromStream(importFileStream, { pageSize: 1 });
+      const firstRecord = await generator.next(); // Get the first record to extract headers and sample data
+      return firstRecord.value;
+    }
+    else { // If the file is neither CSV nor Excel, throw an error
+      throw new Error(`Unsupported file type: ${mimeType}`);
+    }
+  }
+
+  private async getImportFileStream(importFileMediaObject: MediaWithFullUrl): Promise<Readable> {
+    const fileUrl = importFileMediaObject['_full_url'];
+    const mimeType = importFileMediaObject['mimeType'];
+    if (!fileUrl) {
+      throw new Error(`File URL  ${fileUrl} not found.`);
+    }
+    // From the file URL, convert the file URL to a readable stream using nestjs http service and axios
+    const fileUrlResponse = await this.httpService.axiosRef.get(fileUrl, {
+      responseType: 'stream',
+    });
+
+    if (!fileUrlResponse || !fileUrlResponse.data) {
+      throw new Error(`Failed to read file from URL: ${fileUrl}`);
+    }
+    // fileUrlResponse.data is a Node.js Readable stream
+    return fileUrlResponse.data;
+  }
+
+  async getImportInstructions(modelMetadataId: number): Promise<ImportInstructionsResponseDto> {
+    // Load the model metadata for the given ID
+    const modelMetadata = await this.modelMetadataService.findOne(modelMetadataId, {
+      populate: ['fields'],
+    });
+    if (!modelMetadata) {
+      throw new Error(`Model metadata with ID ${modelMetadataId} not found.`);
+    }
+
+    // Create the standard import instructions
+    const standardInstructions: StandardImportInstructionsResponseDto = {
+      requiredFields: [],
+      dateFields: [],
+      dateTimeFields: [],
+      numberFields: [],
+      emailFields: [],
+      regexFields: [],
+      jsonFields: [],
+      booleanFields: [],
+    };
+
+    // Iterate through the fields and populate the standard instructions
+    for (const field of modelMetadata.fields) {
+      if (field.isSystem) continue; // Skip system fields
+      if (field.required) {
+        standardInstructions.requiredFields.push(field.displayName);
+      }
+      if (field.type === SolidFieldType.date) {
+        standardInstructions.dateFields.push(field.displayName);
+      }
+      if (field.type === SolidFieldType.datetime) {
+        standardInstructions.dateTimeFields.push(field.displayName);
+      }
+      if ([SolidFieldType.bigint, SolidFieldType.int, SolidFieldType.decimal].includes(field.type as SolidFieldType)) {
+        standardInstructions.numberFields.push(field.displayName);
+      }
+      if (field.type === SolidFieldType.email) {
+        standardInstructions.emailFields.push(field.displayName);
+      }
+      if (field.regexPattern) {
+        standardInstructions.regexFields.push({
+          fieldName: field.displayName,
+          regexPattern: field.regexPattern,
+        });
+      }
+      if (field.type === SolidFieldType.json) {
+        standardInstructions.jsonFields.push(field.displayName);
+      }
+      if (field.type === SolidFieldType.boolean) {
+        standardInstructions.booleanFields.push(field.displayName);
+      }
+    }
+
+    // Create the custom instructions
+    const customInstructions: string[] = [];
+
+    return {
+      standard: standardInstructions,
+      custom: customInstructions,
+    };
+  }
+
+  /**
+   * This method is used to return a csv / excel template for the import transaction
+   * It will contain the display names of the fields in the header row
+   * @param modelMetadataId 
+   */
+  async getImportTemplate(modelMetadataId: number, format: ImportFormat = ImportFormat.CSV): Promise<ImportTemplateFileInfo> {
+    // Load the model metadata for the given ID
+    const modelMetadata = await this.modelMetadataService.findOne(modelMetadataId, {
+      populate: ['fields'],
+    });
+    if (!modelMetadata) {
+      throw new Error(`Model metadata with ID ${modelMetadataId} not found.`);
+    }
+    // Create a header row with the display names of the fields, excluding the media fields,computed fields
+    const headers = this.fieldsAllowedForImport(modelMetadata.fields)
+      .map(field => field.displayName);
+
+    // Depending on the format, generate the template
+    if (format === ImportFormat.CSV) {
+      const stream = await this.csvService.createCsvStream(null, 0, headers); // Create a CSV stream with the header row
+      const fileName = `${modelMetadata.singularName}-import-template.csv`;
+      const mimeType = 'text/csv';
+      return {
+        stream,
+        fileName,
+        mimeType,
+      };
+    } else if (format === ImportFormat.EXCEL) {
+      const stream = await this.excelService.createExcelStream(null, 0, headers); // Create an Excel stream with the header row
+      const fileName = `${modelMetadata.singularName}-import-template.xlsx`;
+      // Set the MIME type for Excel files
+      const mimeType = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet';
+      return {
+        stream,
+        fileName,
+        mimeType,
+      };
+    } else {
+      throw new Error(`Unsupported import format: ${format}`);
+    }
+
+  }
+
+  private fieldsAllowedForImport(fields: FieldMetadata[]): FieldMetadata[] {
+    // Filter out fields that are not allowed for import
+    return fields.filter(field =>
+      field.type !== SolidFieldType.mediaMultiple && // Exclude media multiple fields
+      field.type !== SolidFieldType.mediaSingle &&
+      field.type !== SolidFieldType.computed && // Exclude computed fields
+      field.type !== SolidFieldType.password &&
+      field.type !== SolidFieldType.richText &&
+      field.type !== SolidFieldType.uuid &&
+      field.isSystem !== true // Exclude system fields
+    );
+  }
+
+
+
+}

package/src/solid-core.module.ts

@@ -171,6 +171,12 @@ import { ExcelService } from './services/excel.service';
 import { CsvService } from './services/csv.service';
 import { ClsModule } from 'nestjs-cls';
 import { FieldRepository } from './repository/field.repository';
+import { ImportTransaction } from './entities/import-transaction.entity';
+import { ImportTransactionService } from './services/import-transaction.service';
+import { ImportTransactionController } from './controllers/import-transaction.controller';
+import { ImportTransactionErrorLog } from './entities/import-transaction-error-log.entity';
+import { ImportTransactionErrorLogService } from './services/import-transaction-error-log.service';
+import { ImportTransactionErrorLogController } from './controllers/import-transaction-error-log.controller';
 
 
 @Global()
@@ -229,6 +235,8 @@ import { FieldRepository } from './repository/field.repository';
       middleware: {
         mount: true,
       }}),
+    TypeOrmModule.forFeature([ImportTransaction]),
+    TypeOrmModule.forFeature([ImportTransactionErrorLog]),
   ],
   controllers: [
     ModuleMetadataController,
@@ -262,6 +270,8 @@ import { FieldRepository } from './repository/field.repository';
     ChatterMessageDetailsController,
     ExportTemplateController,
     ExportTransactionController,
+    ImportTransactionController,
+    ImportTransactionErrorLogController,
   ],
   providers: [
     {
@@ -370,7 +380,9 @@ import { FieldRepository } from './repository/field.repository';
     ExportTransactionService,
     ExcelService,
     CsvService,
-    FieldRepository
+    FieldRepository,
+    ImportTransactionService,
+    ImportTransactionErrorLogService,
   ],
   exports: [
     ModuleMetadataService,