@solidstarters/solid-core 1.2.115 → 1.2.116

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@solidstarters/solid-core",
-  "version": "1.2.115",
+  "version": "1.2.116",
   "description": "This module is a NestJS module containing all the required core providers required by a Solid application",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

package/src/controllers/import-transaction.controller.ts

@@ -107,6 +107,21 @@ export class ImportTransactionController {
     return this.service.getImportMappingInfo(+id);
   }
 
+  @ApiBearerAuth("jwt")
+  @Get(':id/export-failed-import-records')
+  async exportFailedImportedImports(@Param('id') id: string, @Res() res: Response) {
+    const {stream, fileName, mimeType}  = await this.service.exportFailedImportedImports(+id);
+    if (stream === null) {
+      throw new InternalServerErrorException("Failed records stream is null");
+    }
+    // ✅ Set response headers for streaming
+    res.setHeader('Content-Disposition', `attachment; filename="${fileName}"`);
+    res.setHeader('Content-Type', mimeType);
+    res.setHeader('Access-Control-Expose-Headers', 'Content-Disposition, Content-Type');
+    // Pipe the strea to the response as an excel file
+    stream.pipe(res);
+  }
+
   @ApiBearerAuth("jwt")
   @Post(':id/start-import/sync')
   async startImportSync(@Param('id') id: string) {

package/src/services/export-transaction.service.ts

@@ -167,7 +167,7 @@ export class ExportTransactionService extends CRUDService<ExportTransaction> {
         exportStream = await this.excelService.createExcelStream(dataRecordsFunc, EXPORT_CHUNK_SIZE);
         break;
       case ExportFormat.CSV:
-        exportStream = await await this.csvService.createCsvStream(dataRecordsFunc, EXPORT_CHUNK_SIZE);
+        exportStream = await this.csvService.createCsvStream(dataRecordsFunc, EXPORT_CHUNK_SIZE);
         break;
       default:
         throw new Error('Invalid export format');

package/src/services/import-transaction.service.ts

@@ -1,4 +1,4 @@
-import { Injectable, Logger } from '@nestjs/common';
+import { BadRequestException, Injectable, Logger } from '@nestjs/common';
 import { DiscoveryService, ModuleRef } from "@nestjs/core";
 import { InjectEntityManager, InjectRepository } from '@nestjs/typeorm';
 import { EntityManager, Repository } from 'typeorm';
@@ -16,9 +16,11 @@ import { HttpService } from '@nestjs/axios';
 import { RelationFieldsCommand, RelationType, SolidFieldType } from 'src/dtos/create-field-metadata.dto';
 import { ImportInstructionsResponseDto, StandardImportInstructionsResponseDto } from 'src/dtos/import-instructions.dto';
 import { FieldMetadata } from 'src/entities/field-metadata.entity';
+import { ImportTransactionErrorLog } from 'src/entities/import-transaction-error-log.entity';
 import { ModelMetadata } from 'src/entities/model-metadata.entity';
 import { MediaWithFullUrl } from 'src/interfaces';
 import { Readable } from 'stream';
+import { v4 as uuidv4 } from 'uuid';
 import { ImportTransaction } from '../entities/import-transaction.entity';
 import { CsvService } from './csv.service';
 import { ExcelService } from './excel.service';
@@ -34,6 +36,11 @@ export enum ImportFormat {
   CSV = 'csv',
   EXCEL = 'excel',
 }
+
+export enum ImportMimeTypes {
+  CSV = 'text/csv',
+  EXCEL = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+}
 export interface ImportMappingInfo {
   sampleImportedRecordInfo: SampleImportedRecordInfo[];
   importableFields: ImportableFieldInfo[];
@@ -63,6 +70,11 @@ export interface ImportSyncResult {
   importedIds: Array<number>; // The IDs of the records created during the import
 }
 
+interface ImportRecordsResult {
+  ids: Array<number>; // The IDs of the records created during the import
+  errorLogIds: Array<number>; // The IDs of the error log entries created during the import
+}
+
 @Injectable()
 export class ImportTransactionService extends CRUDService<ImportTransaction> {
   constructor(
@@ -242,25 +254,106 @@ export class ImportTransactionService extends CRUDService<ImportTransaction> {
     // Get the import file stream for the import transaction
     const importFileStream = await this.getImportFileStream(importFileMediaObject);
 
-    const ids = await this.writeFileRecordsToDb(
+    const { ids, errorLogIds } = await this.importFromFileToDB(
+      importTransaction,
       importFileStream,
       importFileMediaObject.mimeType,
-      JSON.parse(importTransaction.mapping) as ImportMapping[], // Parse the mapping from the import transaction
-      importTransaction.modelMetadata,
     );
 
     // Update the import transaction status to 'completed'
-    importTransaction.status = 'import_succeeded';
+    (errorLogIds.length > 0) ? importTransaction.status = 'import_failed' : importTransaction.status = 'import_succeeded'; //FIXME: We can probably have import_partially_failed status to differentiate
     // Save the import transaction
     await this.repo.save(importTransaction);
 
-    return {status: importTransaction.status, importedIds: ids}; // Return the IDs of the created records
+    return { status: importTransaction.status, importedIds: ids }; // Return the IDs of the created records
   }
 
   startImportAsync(importTransactionId: number): Promise<void> {
     throw new Error('Method not implemented.');
   }
 
+  async exportFailedImportedImports(importTransactionId: number) {
+    // Get the 1st error log entry to determine the headers for the export file
+    const firstErrorLogEntry = await this.entityManager.getRepository(ImportTransactionErrorLog).findOne({
+      where: {
+        importTransaction: { id: importTransactionId },
+      },
+    });
+
+    if (!firstErrorLogEntry) {
+      throw new BadRequestException(`No error log entries found for import transaction ID ${importTransactionId}.`);
+    }
+
+    // Create the headers for the export file
+    const headers = [
+      'rowNumber', // Row number in the import file
+      'errorMessage', // Error message for the failed record
+      'errorTrace', // Error trace for debugging
+      ...Object.keys(firstErrorLogEntry.rowData ? JSON.parse(firstErrorLogEntry.rowData) : {}), // Include all keys from the rowData JSON
+    ];
+
+
+    // Depending upon the format of the import tranaction file, create a readable stream of the error log entries
+    const importTransaction = await this.loadImportTransaction(importTransactionId);
+    const importFileMediaObject = this.getImportFileObject(importTransaction);
+    const mimeType = importFileMediaObject.mimeType;
+    const templateFormat = mimeType === ImportMimeTypes.CSV ? "csv" : "excel";
+    const dataRecordsFunc = async (chunkIndex: number, chunkSize: number): Promise<any[]> => {
+      // Get the error log entries for the import transaction
+      const errorLogEntries = await this.entityManager.getRepository(ImportTransactionErrorLog).find({
+        where: {
+          importTransaction: { id: importTransactionId },
+        },
+        skip: chunkIndex * chunkSize,
+        take: chunkSize,
+      });
+
+      if (!errorLogEntries || errorLogEntries.length === 0) {
+        return []; // Return an empty array if no error log entries found
+      }
+
+      // Read the row data json from the error log entry, parse it and write it as a record to the stream
+      return errorLogEntries.map(entry => {
+        const rowData = entry.rowData ? JSON.parse(entry.rowData) : {};
+        return {
+          rowNumber: entry.rowNumber,
+          errorMessage: entry.errorMessage,
+          errorTrace: entry.errorTrace,
+          ...rowData, // Spread the row data into the record
+        };
+      });
+    };
+
+    // Get the export stream for the failed records
+    const exportStream = await this.getFailedRecordsStream(templateFormat, headers, dataRecordsFunc);
+    if (!exportStream) {
+      throw new BadRequestException(`Failed to create export stream for import transaction ID ${importTransactionId}.`);
+    }
+    // Return the export stream
+    return {
+      stream: exportStream,
+      fileName: `${importTransaction.modelMetadata.singularName}-failed-imports.${templateFormat}`,
+      mimeType: templateFormat === "excel" ? ImportMimeTypes.EXCEL : ImportMimeTypes.CSV,
+    };
+
+  }
+
+  private async getFailedRecordsStream(templateFormat: string, headers: string[], dataRecordsFunc: (chunkIndex: number, chunkSize: number) => Promise<any[]>) {
+    let exportStream = null;
+    switch (templateFormat) {
+      case "excel":
+        exportStream = await this.excelService.createExcelStream(dataRecordsFunc, 100, headers);
+        break;
+      case "csv":
+        exportStream = await this.csvService.createCsvStream(dataRecordsFunc, 100, headers);
+        break;
+      default:
+        throw new Error('Invalid export format');
+    }
+    return exportStream;
+  }
+
+
   private async loadImportTransaction(importTransactionId: number) {
     const importTransaction = await this.findOne(importTransactionId, {
       populate: ['modelMetadata', 'modelMetadata.fields'],
@@ -329,48 +422,96 @@ export class ImportTransactionService extends CRUDService<ImportTransaction> {
     return fileUrlResponse.data;
   }
 
-  private async writeFileRecordsToDb(
+  private async importFromFileToDB(
+    importTransaction: ImportTransaction,
     importFileStream: Readable,
     mimeType: string,
-    mapping: ImportMapping[],
-    modelMetadataWithFields: ModelMetadata,
-  ): Promise<Array<number>> {
-    // Get the model service for the model metadata name
-    const modelService = this.getModelService(modelMetadataWithFields.singularName);
+  ): Promise<ImportRecordsResult> {
+    if (!importTransaction.modelMetadata) {
+      throw new Error(`Model metadata for import transaction ID ${importTransaction.id} not found.`);
+    }
+
     const createdRecordIds = [];
-    // Depending upon the mime type of the file, read the file in pages
-    // For CSV files, use the csvService to read the file in pages
-    // For Excel files, use the excelService to read the file in pages
-    if (mimeType === 'text/csv') {
-      // Read the csv file in pages
+    const createdErrorLogIds = [];
+
+    // Get the model service for the model metadata name
+    const modelService = this.getModelService(importTransaction.modelMetadata.singularName);
+
+    // Depending upon the mime type of the file, read the file in pages and insert the records into the database
+    if (mimeType === ImportMimeTypes.CSV) {
       for await (const page of this.csvService.readCsvInPagesFromStream(importFileStream)) {
-        // Convert the paginated result to DTOs
-        const dtos = await this.convertPaginatedResultToDtos(page, modelMetadataWithFields, mapping);
-        // Use the model service to create the records in the database
-        const createdRecords = await modelService.insertMany(dtos, [], {});
-        // Set the solidRequestContext to null, as this is a background job;
-        // Return the IDs of the created records
-        const newIds = createdRecords.map(record => record.id);
-        // Add the new IDs to the createdRecordIds array
-        createdRecordIds.push(...newIds);
+        const { ids, errorLogIds } = await this.importRecords(page, importTransaction, modelService);
+        createdRecordIds.push(...ids);
+        createdErrorLogIds.push(...errorLogIds);
       }
     }
-    else if (mimeType === 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet') {
-      // Read the excel file in pages
+    else if (mimeType === ImportMimeTypes.EXCEL) {
       for await (const page of this.excelService.readExcelInPagesFromStream(importFileStream)) {
-        // Convert the paginated result to DTOs
-        const dtos = await this.convertPaginatedResultToDtos(page, modelMetadataWithFields, mapping);
-        // Use the model service to create the records in the database
-        const createdRecords = await modelService.insertMany(dtos, [], {});
-        // Set the solidRequestContext to null, as this is a background job;
-        // Return the IDs of the created records
-        const newIds = createdRecords.map(record => record.id);
-        createdRecordIds.push(...newIds);
+        const { ids, errorLogIds } = await this.importRecords(page, importTransaction, modelService);
+        createdRecordIds.push(...ids);
+        createdErrorLogIds.push(...errorLogIds);
       }
     } else { // If the file is neither CSV nor Excel, throw an error
       throw new Error(`Unsupported file type: ${mimeType}`);
     }
-    return createdRecordIds; // Return the IDs of the created records
+
+    return {
+      ids: createdRecordIds, // Return the IDs of the created records
+      errorLogIds: createdErrorLogIds, // Return the IDs of the error log entries created during the import
+    }
+  }
+
+  private async importRecords(page: ImportPaginatedReadResult, importTransaction: ImportTransaction, modelService: CRUDService<any>): Promise<ImportRecordsResult> {
+    if (!importTransaction.modelMetadata || !importTransaction.modelMetadata.fields) {
+      throw new Error(`Model metadata with fields for import transaction ID ${importTransaction.id} not found.`);
+    }
+
+    const ids: Array<number> = [];
+    const errorLogIds: Array<number> = [];
+    for (const record of page.data) {
+      try {
+        const createdRecord = await this.insertRecord(record, JSON.parse(importTransaction.mapping) as ImportMapping[], importTransaction.modelMetadata, modelService);
+        ids.push(createdRecord.id); // Add the ID of the created record to the ids array
+      }
+      catch (error) {
+        this.logger.debug(`Error inserting record: ${JSON.stringify(record)}. Error: ${error.message}`);
+        // Get the Import transaction error log repo
+        const errorLog = await this.createErrorLogEntry(importTransaction, record, error);
+        errorLogIds.push(errorLog.id); // Add the ID of the error log entry to the errorLogIds array
+      }
+    }
+    return {
+      ids: ids, // Return the IDs of the created records
+      errorLogIds: errorLogIds, // Return the IDs of the error log entries created during the import
+    };
+  }
+
+  private async createErrorLogEntry(importTransaction: ImportTransaction, record: Record<string, any>, error: any) {
+    const importTransactionRepo = this.entityManager.getRepository(ImportTransactionErrorLog);
+    // Create a new ImportTransactionErrorLog entry
+    const rowNumber = uuidv4(); // Generate a unique row number or use page.rowNumber if available 
+
+    const errorLogEntry = {
+      importTransactionErrorLogId: `${importTransaction.id}-${rowNumber}`, // FIXME pending to retrieve the row number from the page
+      rowNumber: 1, // FIXME pending to retrieve the row number from the page
+      rowData: JSON.stringify(record), // Store the row data
+      importTransaction: importTransaction, // Link to the import transaction
+      errorMessage: error.message, // Store the error message
+      errorTrace: error.stack || '', // Store the error stack trace if available
+    } as ImportTransactionErrorLog;
+
+    // Save the error log entry to the database
+    const savedEntry = await importTransactionRepo.save(errorLogEntry);
+    return savedEntry; // Return the ID of the saved error log entry
+  }
+
+  //FIXME Currently below method fails if any field in the record is not valid or if the record is not valid. It does not collect the errors for all fields in a record
+  private async insertRecord(record: Record<string, any>, mapping: ImportMapping[], modelMetadataWithFields: ModelMetadata, modelService: CRUDService<any>): Promise<any> {
+    // Convert the imported record to a DTO
+    const dto = await this.convertImportedRecordToDto(record, mapping, modelMetadataWithFields);
+    // Use the model service to create the record in the database
+    const createdRecord = await modelService.create(dto, [], {}); //FIXME: Need to handle this part alongwith the refactoring of the CRUDService for permissions
+    return createdRecord; // Return the created record
   }
 
   private getModelService(modelSingularName: string): CRUDService<any> {
@@ -383,22 +524,11 @@ export class ImportTransactionService extends CRUDService<ImportTransaction> {
     return modelService;
   }
 
-  // This method will 
-  private async convertPaginatedResultToDtos(importPaginatedResult: ImportPaginatedReadResult, modelMetadataWithFields: ModelMetadata, mapping: ImportMapping[]) {
-    const dtos = [];
-    // Iterate through the data records in the importPaginatedResult
-    for (const record of importPaginatedResult.data) {
-      // For every key in the record, get the corresponding field from the mapping, if the field is not found in mapping, skip the field
-      const dto = await this.convertImportedRecordToDto(record, mapping, modelMetadataWithFields);
-      dtos.push(dto);
-    }
-    return dtos;
-  }
-
   private async convertImportedRecordToDto(record: Record<string, any>, mapping: ImportMapping[], modelMetadataWithFields: ModelMetadata) {
     // Create a new record object
     const dtoRecord: Record<string, any> = {};
 
+    // Iterate through every cell in the record
     // Using the saved mapping, populate the dtoRecord w.r.t the record and fields
     for (const key in record) {
       const mappedField = mapping.find(m => m.header === key);
@@ -408,7 +538,7 @@ export class ImportTransactionService extends CRUDService<ImportTransaction> {
         // const userKeyField = modelMetadataWithFields.fields.find(f => f.isUserKey === true); // Assuming userKey is a field in the model metadata
         if (fieldMetadata) {
           // If the field is found in the model metadata, set the value in the dtoRecord
-          await this.populateDto(dtoRecord, fieldMetadata, record, key);
+          await this.populateDtoForACell(dtoRecord, fieldMetadata, record, key);
         } else {
           this.logger.warn(`Field ${mappedField.fieldName} not found in model metadata ${modelMetadataWithFields.singularName}`);
         }
@@ -417,7 +547,7 @@ export class ImportTransactionService extends CRUDService<ImportTransaction> {
     return dtoRecord;
   }
 
-  private async populateDto(dtoRecord: Record<string, any>, fieldMetadata: FieldMetadata, record: Record<string, any>, key: string): Promise<Record<string, any>> {
+  private async populateDtoForACell(dtoRecord: Record<string, any>, fieldMetadata: FieldMetadata, record: Record<string, any>, key: string): Promise<Record<string, any>> {
     const fieldType = fieldMetadata.type;
     // const userKeyFieldName = userKeyField?.name || 'id'; // Default to 'id' if not found
 
@@ -426,14 +556,14 @@ export class ImportTransactionService extends CRUDService<ImportTransaction> {
       case SolidFieldType.relation: {
         return await this.populateDtoForRelations(fieldMetadata, record, key, dtoRecord);
       }
-      case SolidFieldType.date: 
+      case SolidFieldType.date:
       case SolidFieldType.datetime: return this.populateDtoForDate(record, key, fieldMetadata, dtoRecord);
       case SolidFieldType.int:
       case SolidFieldType.bigint:
       case SolidFieldType.decimal:
-        return this.populateDtoForNumber(dtoRecord, fieldMetadata, record, key); 
+        return this.populateDtoForNumber(dtoRecord, fieldMetadata, record, key);
       case SolidFieldType.boolean:
-        return this.populateDtoForBoolean(dtoRecord, fieldMetadata, record, key);  
+        return this.populateDtoForBoolean(dtoRecord, fieldMetadata, record, key);
       default:
         dtoRecord[fieldMetadata.name] = record[key];
         return dtoRecord;
@@ -443,7 +573,7 @@ export class ImportTransactionService extends CRUDService<ImportTransaction> {
   private populateDtoForBoolean(dtoRecord: Record<string, any>, fieldMetadata: FieldMetadata, record: Record<string, any>, key: string) {
     const booleanValue = Boolean(record[key]);
     if (typeof booleanValue !== 'boolean') {
-      throw new Error(`Invalid boolean value for field ${fieldMetadata.name}: ${record[key]}`);
+      throw new Error(`Invalid boolean value for cell ${key} with value ${record[key]}`);
     }
     dtoRecord[fieldMetadata.name] = booleanValue;
     return dtoRecord;
@@ -452,7 +582,7 @@ export class ImportTransactionService extends CRUDService<ImportTransaction> {
   private populateDtoForNumber(dtoRecord: Record<string, any>, fieldMetadata: FieldMetadata, record: Record<string, any>, key: string) {
     const numberValue = Number(record[key]);
     if (isNaN(numberValue)) {
-      throw new Error(`Invalid number value for field ${fieldMetadata.name}: ${record[key]}`);
+      throw new Error(`Invalid number value for cell ${key} with value ${record[key]}`);
     }
     dtoRecord[fieldMetadata.name] = numberValue;
     return dtoRecord;
@@ -462,7 +592,7 @@ export class ImportTransactionService extends CRUDService<ImportTransaction> {
     {
       const dateValue = new Date(record[key]);
       if (isNaN(dateValue.getTime())) {
-        throw new Error(`Invalid date value for field ${fieldMetadata.name}: ${record[key]}`);
+        throw new Error(`Invalid date value for cell ${key} with value ${record[key]}`);
       }
       dtoRecord[fieldMetadata.name] = dateValue;
       return dtoRecord;
@@ -511,7 +641,7 @@ export class ImportTransactionService extends CRUDService<ImportTransaction> {
     // From the userKeys, we will get the IDs of the related records using the userKeyFieldName and throw an error if any of the userKeys is not found
     const relatedRecordsResult = await coModelService.find(relationFilterDto);
     if (!relatedRecordsResult || !relatedRecordsResult.records || relatedRecordsResult.records.length === 0 || relatedRecordsResult.records.length !== relationUserKeys.length) {
-      throw new Error(`Missing related records found for userKeys: ${relationUserKeys.join(', ')} in model ${fieldMetadata.relationCoModelSingularName}`);
+      throw new Error(`Invalid related records userKey values found for cell ${key} with value ${record[key]}`);
     }
     const relatedRecordsIds = relatedRecordsResult.records.map(record => record.id);
     return relatedRecordsIds;