@solidstarters/solid-core 1.2.89 → 1.2.91

package/src/seeders/seed-data/solid-core-metadata.json

@@ -3414,7 +3414,7 @@
             "ormType": "varchar",
             "length": 256,
             "required": true,
-            "unique": true,  
+            "unique": true,
             "index": true,
             "private": false,
             "encrypt": false,
@@ -3506,7 +3506,7 @@
           {
             "name": "exportTransactionId",
             "displayName": "Transaction Id",
-            "type":"computed",
+            "type": "computed",
             "ormType": "varchar",
             "length": 128,
             "required": true,
@@ -3577,6 +3577,201 @@
             "isSystem": true
           }
         ]
+      },
+      {
+        "singularName": "importTransaction",
+        "tableName": "ss_import_transaction",
+        "pluralName": "importTransactions",
+        "displayName": "Import Transactions",
+        "description": "Model to capture all information related to a file import transaction",
+        "dataSource": "default",
+        "dataSourceType": "postgres",
+        "isSystem": true,
+        "userKeyFieldUserKey": "importTransactionId",
+        "fields": [
+          {
+            "name": "status",
+            "displayName": "Status",
+            "type": "selectionStatic",
+            "ormType": "varchar",
+            "length": 25,
+            "required": false,
+            "index": true,
+            "isSystem": true,
+            "selectionValueType": "string",
+            "selectionStaticValues": [
+              "draft:draft",
+              "mapping_created:mapping_created",
+              "import_started:import_started",
+              "import_succeeded:import_succeeded",
+              "import_failed:import_failed"
+            ],
+            "defaultValue": "draft"
+          },
+          {
+            "name": "importTransactionId",
+            "displayName": "Transaction Id",
+            "type": "shortText",
+            "ormType": "varchar",
+            "length": 128,
+            "required": false,
+            "unique": true,
+            "index": true,
+            "private": false,
+            "encrypt": false,
+            "isSystem": true
+          },
+          {
+            "name": "fileLocation",
+            "displayName": "File Location",
+            "type": "mediaSingle",
+            "required": true,
+            "unique": true,
+            "index": false,
+            "private": false,
+            "encrypt": false,
+            "isSystem": true,
+            "mediaStorageProviderUserKey": "default-filesystem"
+          },
+          {
+            "name": "mapping",
+            "displayName": "Mapping",
+            "type": "json",
+            "ormType": "text",
+            "required": false,
+            "unique": false,
+            "index": false,
+            "private": false,
+            "encrypt": false,
+            "isSystem": true
+          },
+          {
+            "name": "modelMetadata",
+            "displayName": "Related Model Metadata",
+            "type": "relation",
+            "ormType": "int",
+            "required": false,
+            "unique": false,
+            "index": true,
+            "private": false,
+            "encrypt": false,
+            "relationType": "many-to-one",
+            "relationCoModelSingularName": "modelMetadata",
+            "relationCreateInverse": false,
+            "relationCascade": "cascade",
+            "relationModelModuleName": "solid-core",
+            "isSystem": true
+          },
+          {
+            "name": "importTransactionErrorLog",
+            "displayName": "Related Import Transaction Error Log",
+            "type": "relation",
+            "required": false,
+            "unique": false,
+            "index": true,
+            "private": false,
+            "encrypt": false,
+            "relationType": "one-to-many",
+            "relationCoModelFieldName": "importTransaction",
+            "relationCreateInverse": true,
+            "relationCoModelSingularName": "importTransactionErrorLog",
+            "relationCoModelColumnName": "",
+            "relationModelModuleName": "solid-core"
+          }
+        ]
+      },
+      {
+        "singularName": "importTransactionErrorLog",
+        "tableName": "ss_import_transaction_error_log",
+        "pluralName": "importTransactionErrorLogs",
+        "displayName": "Import Transaction Error Logs",
+        "description": "Model to capture all information related to a file import transaction error log",
+        "dataSource": "default",
+        "dataSourceType": "postgres",
+        "isSystem": true,
+        "userKeyFieldUserKey": "importTransactionErrorLogId",
+        "fields": [
+          {
+            "name": "importTransactionErrorLogId",
+            "displayName": "Import Transaction Error Log Id",
+            "type": "shortText",
+            "ormType": "varchar",
+            "length": 256,
+            "required": true,
+            "unique": true,
+            "index": true,
+            "private": false,
+            "encrypt": false,
+            "isSystem": true
+          },
+          {
+            "name": "rowNumber",
+            "displayName": "Row Number",
+            "type": "int",
+            "ormType": "integer",
+            "required": true,
+            "unique": false,
+            "index": false,
+            "private": false,
+            "encrypt": false,
+            "isSystem": true
+          },
+          {
+            "name": "rowData",
+            "displayName": "Row Data",
+            "type": "json",
+            "ormType": "text",
+            "required": false,
+            "unique": false,
+            "index": false,
+            "private": false,
+            "encrypt": false,
+            "isSystem": true
+          },
+          {
+            "name": "importTransaction",
+            "displayName": "Related Import Transaction",
+            "type": "relation",
+            "ormType": "int",
+            "required": true,
+            "unique": false,
+            "index": true,
+            "private": false,
+            "encrypt": false,
+            "relationType": "many-to-one",
+            "relationCoModelFieldName": "importTransactionErrorLogs",
+            "relationCreateInverse": true,
+            "relationCoModelSingularName": "importTransaction",
+            "relationCoModelColumnName": "",
+            "relationModelModuleName": "solid-core",
+            "relationCascade": "cascade"
+          },
+          {
+            "name": "errorMessage",
+            "displayName": "Error Message",
+            "type": "shortText",
+            "ormType": "varchar",
+            "length": 512,
+            "required": true,
+            "unique": false,
+            "index": false,
+            "private": false,
+            "encrypt": false,
+            "isSystem": true
+          },
+          {
+            "name": "errorTrace",
+            "displayName": "Error Trace",
+            "type": "longText",
+            "ormType": "text",
+            "required": false,
+            "unique": false,
+            "index": false,
+            "private": false,
+            "encrypt": false,
+            "isSystem": true
+          }
+        ]
       }
     ]
   },
@@ -3915,6 +4110,19 @@
       "viewUserKey": "listOfValues-list-view",
       "moduleUserKey": "solid-core",
       "modelUserKey": "listOfValues"
+    },
+    {
+      "displayName": "Import Transactions List View",
+      "name": "importTransaction-list-view",
+      "type": "solid",
+      "domain": "",
+      "context": "",
+      "customComponent": "/admin/address-master/importTransaction/all",
+      "customIsModal": true,
+      "serverEndpoint": "",
+      "viewUserKey": "importTransaction-list-view",
+      "moduleUserKey": "solid-core",
+      "modelUserKey": "importTransaction"
     }
   ],
   "menus": [
@@ -4149,6 +4357,14 @@
       "actionUserKey": "chatter-message-details-list-action",
       "moduleUserKey": "solid-core",
       "parentMenuItemUserKey": "other-menu-item"
+    },
+    {
+      "displayName": "Import Transactions",
+      "name": "importTransaction-menu-item",
+      "sequenceNumber": 1,
+      "actionUserKey": "importTransaction-list-view",
+      "moduleUserKey": "solid-core",
+      "parentMenuItemUserKey": ""
     }
   ],
   "views": [
@@ -8750,6 +8966,106 @@
           }
         ]
       }
+    },
+    {
+      "name": "importTransaction-list-view",
+      "displayName": "Import Transactions",
+      "type": "list",
+      "context": "{}",
+      "moduleUserKey": "solid-core",
+      "modelUserKey": "importTransaction",
+      "layout": {
+        "type": "list",
+        "attrs": {
+          "pagination": true,
+          "pageSizeOptions": [
+            10,
+            25,
+            50
+          ],
+          "enableGlobalSearch": true,
+          "create": true,
+          "edit": true,
+          "delete": true
+        },
+        "children": [
+          {
+            "type": "field",
+            "attrs": {
+              "name": "id",
+              "sortable": true,
+              "filterable": true
+            }
+          },
+          {
+            "type": "field",
+            "attrs": {
+              "name": "importTransactionErrorLog",
+              "sortable": true,
+              "filterable": true
+            }
+          }
+        ]
+      }
+    },
+    {
+      "name": "importTransaction-form-view",
+      "displayName": "Import Transactions",
+      "type": "form",
+      "context": "{}",
+      "moduleUserKey": "solid-core",
+      "modelUserKey": "importTransaction",
+      "layout": {
+        "type": "form",
+        "attrs": {
+          "name": "form-1",
+          "label": "Import Transactions",
+          "className": "grid"
+        },
+        "children": [
+          {
+            "type": "sheet",
+            "attrs": {
+              "name": "sheet-1"
+            },
+            "children": [
+              {
+                "type": "row",
+                "attrs": {
+                  "name": "sheet-1"
+                },
+                "children": [
+                  {
+                    "type": "column",
+                    "attrs": {
+                      "name": "group-1",
+                      "label": "",
+                      "className": "col-6"
+                    },
+                    "children": [
+                      {
+                        "type": "field",
+                        "attrs": {
+                          "name": "importTransactionErrorLog"
+                        }
+                      }
+                    ]
+                  },
+                  {
+                    "type": "column",
+                    "attrs": {
+                      "name": "group-2",
+                      "label": "",
+                      "className": "col-6"
+                    },
+                    "children": []
+                  }
+                ]
+              }
+            ]
+          }
+        ]
+      }
     }
   ],
   "emailTemplates": [

package/src/services/csv.service.ts

@@ -1,22 +1,59 @@
-import { PassThrough, Readable } from 'stream';
-import { format } from 'fast-csv';
 import { Injectable, Logger } from '@nestjs/common';
+import { format, parse } from 'fast-csv';
+import { PassThrough, Readable } from 'stream';
+
+export interface CsvReadOptions {
+  pageSize?: number; // Number of records per page
+  hasHeaderRow?: boolean;
+  providedHeaders?: string[]
+};
+const DEFAULT_PAGE_SIZE = 100; // Default page size if not provided
+export interface CsvReadResult {
+  headers: string[]; // Headers of the CSV file
+  data: Record<string, any>[]; // Data records in the current page
+}
 
 @Injectable()
 export class CsvService {
   private logger = new Logger(CsvService.name);
   public async createCsvStream(
-    getDataRecords: (chunkIndex: number, chunkSize: number) => Promise<any[]>,
-    chunkSize: number
+    getDataRecords: (chunkIndex: number, chunkSize: number) => Promise<any[]> = null,
+    chunkSize: number = 100,
+    headers: string[] = []
   ): Promise<Readable> {
+    // Validations
+    // If neither headers nor data records function is provided, throw an error
+    if (headers.length === 0 && typeof getDataRecords !== 'function') {
+      throw new Error('Either headers or data records function must be provided.');
+    }
+
+    // If data records function is provided, chunkSize must be greater than 0
+    if (getDataRecords && chunkSize <= 0) {
+      throw new Error('Chunk size must be greater than 0 when data records function is provided.');
+    }
+
     const passThrough = new PassThrough(); // ✅ Create a streaming pipe
-    const csvStream = format({ headers: true });
+    const csvStream = headers.length
+      ? format({ headers })
+      : format({ headers: true });
 
     csvStream.pipe(passThrough); // ✅ Pipe CSV output to PassThrough stream
 
-    let chunkIndex = 0;
-
     try {
+      // 🧠 If no data retrieval logic is provided, just write headers and close
+      if (typeof getDataRecords !== 'function') {
+
+        const dummyRow = headers.reduce((acc, header) => {
+          acc[header] = '';
+          return acc;
+        }, {} as Record<string, string>);
+        csvStream.write(dummyRow);
+        csvStream.end();
+        return passThrough;
+      }
+
+      // Write the data records in chunks
+      let chunkIndex = 0;
       while (true) {
         const records = await getDataRecords(chunkIndex, chunkSize); // ✅ Fetch chunked data
         if (records.length === 0) break; // ✅ Stop if no more records
@@ -38,4 +75,76 @@ export class CsvService {
 
     return passThrough; // ✅ Return the streaming response
   }
+
+  public async  *readCsvInPagesFromStream(
+    stream: Readable,
+    options?: CsvReadOptions
+  ): AsyncGenerator<CsvReadResult> {
+    const { pageSize = DEFAULT_PAGE_SIZE, hasHeaderRow = true, providedHeaders = [] } = options || {};
+    let headers: string[] = [];
+    let page: Record<string, any>[] = [];
+    let isFirstRow = true;
+    let hasYieldedData = false;
+
+    // Create parser
+    const parser = parse({ headers: hasHeaderRow, renameHeaders: false, trim: true });
+
+    // Pipe the input stream into the parser
+    const parsingStream = stream.pipe(parser);
+
+    for await (const row of parsingStream) {
+      if (isFirstRow && !hasHeaderRow) {
+        isFirstRow = false;
+
+        if (providedHeaders.length) {
+          headers = providedHeaders;
+        } else {
+          // If no header row and no provided headers, generate index-based headers
+          headers = Object.keys(row).length > 0 ? Object.keys(row).map((_, i) => i.toString()) : [];
+        }
+      }
+
+      // If hasHeaderRow = true, fast-csv already assigns keys as headers, so capture once
+      if (hasHeaderRow && isFirstRow) {
+        headers = Object.keys(row);
+        isFirstRow = false;
+      }
+
+      // When headers are not set yet (edge case), set them now
+      if (!headers.length) {
+        headers = providedHeaders.length ? providedHeaders : Object.keys(row);
+      }
+
+      // Map row fields to headers - if keys mismatch, fallback to index-based mapping
+      const record: Record<string, any> = {};
+      for (let i = 0; i < headers.length; i++) {
+        // For safety, access by header name or fallback by index
+        const key = headers[i];
+        const value = row[key] ?? Object.values(row)[i] ?? null;
+        record[key] = value;
+      }
+
+      // Skip empty rows
+      if (Object.values(record).every(v => v === null || v === '')) continue;
+
+      page.push(record);
+
+      if (page.length === pageSize) {
+        yield { headers, data: page };
+        hasYieldedData = true;
+        page = [];
+      }
+    }
+
+    if (page.length > 0) {
+      yield { headers, data: page };
+      hasYieldedData = true;
+    }
+
+    // If only headers present but no data, yield headers with empty data array
+    if (!hasYieldedData && headers.length > 0) {
+      yield { headers, data: [] };
+    }
+  }
+
 }

package/src/services/excel.service.ts

@@ -3,79 +3,72 @@ import * as ExcelJS from 'exceljs';
 import { PassThrough, Readable } from 'stream';
 
 
+export interface ExcelReadOptions {
+  pageSize?: number; // Number of records per page
+  hasHeaderRow?: boolean; // Whether the first row contains headers
+  providedHeaders?: string[]; // Custom headers if hasHeaderRow is false
+}
+
+const DEFAULT_PAGE_SIZE = 100; // Default page size if not provided
+
+export interface ExcelReadResult {
+  headers: string[]; // Headers of the Excel file
+  data: Record<string, any>[]; // Data records in the current page
+}
+
 @Injectable()
 export class ExcelService {
   private logger = new Logger(ExcelService.name);
-  // Sample JSON data
-  // const jsonData = [
-  //   { id: 1, name: 'John Doe', age: 25, email: 'john@example.com' },
-  //   { id: 2, name: 'Jane Doe', age: 28, email: 'jane@example.com' },
-  //   { id: 3, name: 'Alice Smith', age: 30, email: 'alice@example.com' }
-  // ];
-
-  // public async createExcelFromJson(data: any[], fileName: string) {
-  //   const workbook = new ExcelJS.Workbook();
-  //   const worksheet = workbook.addWorksheet('Data');
-
-  //   // Define Columns (Header)
-  //   worksheet.columns = Object.keys(data[0]).map((key) => ({
-  //     header: key.toUpperCase(), // Convert header names to uppercase
-  //     key: key,
-  //     width: 20, // Set column width
-  //   }));
-
-  //   // Add Data Rows
-  //   data.forEach((item) => {
-  //     worksheet.addRow(item);
-  //   });
-
-  //   // Apply basic formatting
-  //   worksheet.getRow(1).font = { bold: true }; // Make headers bold
-
-  //   // Save file
-  //   await workbook.xlsx.writeFile(fileName);
-  //   this.logger.log(`✅ Excel file "${fileName}" created successfully!`);
-  //   // console.log(`✅ Excel file "${fileName}" created successfully!`);
-  // }
-
-  // public async createExcelStreamFromJson(data: any[]): Promise<Readable> {
-  //   const passThrough = new PassThrough(); // Stream to pipe data
-  //   const workbook = new ExcelJS.stream.xlsx.WorkbookWriter({ stream: passThrough });
-  //   const worksheet = workbook.addWorksheet('Data');
-  //   worksheet.columns = Object.keys(data[0]).map((key) => ({
-  //     header: key.toUpperCase(),
-  //     key: key,
-  //     width: 20,
-  //   }));
-
-  //   data.forEach((item) => {
-  //     worksheet.addRow(item);
-  //   });
-
-  //   worksheet.getRow(1).font = { bold: true };
-
-  //   await workbook.commit();
-  //   return passThrough;
-  // }
-
 
   public async createExcelStream(
     getDataRecords: (chunkIndex: number, chunkSize: number) => Promise<any[]>,
-    chunkSize: number
+    chunkSize: number = 100,
+    headers: string[] = []
   ): Promise<Readable> {
+    // Validations
+    // If neither headers nor data records function is provided, throw an error
+    if (headers.length === 0 && typeof getDataRecords !== 'function') {
+      throw new Error('Either headers or data records function must be provided.');
+    }
+
+    // If data records function is provided, chunkSize must be greater than 0
+    if (getDataRecords && chunkSize <= 0) {
+      throw new Error('Chunk size must be greater than 0 when data records function is provided.');
+    }
+
     const passThrough = new PassThrough(); // Create streaming pipe
     try {
       const workbook = new ExcelJS.stream.xlsx.WorkbookWriter({ stream: passThrough });
       const worksheet = workbook.addWorksheet('Data');
 
-      let chunkIndex = 0;
+      // If headers are provided, use them;
       let isHeaderWritten = false;
+      if (headers.length > 0) {
+        worksheet.columns = headers.map((header) => ({
+          header: header, // Convert header names to uppercase
+          key: header,
+          width: 20, // Set column width
+        }));
+        isHeaderWritten = true; // Mark headers as written
+      }
 
+      // ✅ If no data loader provided, write only headers and finish
+      if (typeof getDataRecords !== 'function') {
+        // worksheet.addRow(
+        //   headers.reduce((acc, header) => ({ ...acc, [header]: '' }), {})
+        // ).commit(); // Write a dummy record with headers
+
+        await workbook.commit();
+        return passThrough;
+      }
+
+      // Write the data records in chunks
+      let chunkIndex = 0;
       while (true) {
         const records = await getDataRecords(chunkIndex, chunkSize); // Fetch chunked data
         if (records.length === 0) break; // Stop if no more records
 
-        if (!isHeaderWritten) {
+        if (!isHeaderWritten) { // Falback because without columns being set, ExcelJS won't write data correctly
           worksheet.columns = Object.keys(records[0]).map((key) => ({
             header: key.toUpperCase(),
             key: key,
@@ -102,4 +95,67 @@ export class ExcelService {
     return passThrough; // Return streaming response
   }
 
+  public async *readExcelInPagesFromStream(
+    stream: Readable,
+    options?: ExcelReadOptions
+  ): AsyncGenerator<ExcelReadResult> {
+    const { pageSize = DEFAULT_PAGE_SIZE, hasHeaderRow = true, providedHeaders = [] } = options || {};
+    const workbookReader = new ExcelJS.stream.xlsx.WorkbookReader(stream, {});
+
+    let headers: string[] = [];
+    let page: Record<string, any>[] = [];
+    let isFirstRow = true;
+    let hasYieldedData = false;
+
+    for await (const worksheet of workbookReader) {
+      for await (const row of worksheet) {
+        const values = Array.isArray(row.values) ? row.values.slice(1) : [];
+
+        if (isFirstRow) {
+          isFirstRow = false;
+
+          if (hasHeaderRow) {
+            headers = values.map(v => v?.toString().trim() || '');
+            continue;
+          } else if (providedHeaders.length) {
+            headers = providedHeaders;
+          } else {
+            headers = values.map((_, idx) => `${idx}`);
+          }
+        }
+
+        while (values.length < headers.length) values.push(null);
+        if (values.length > headers.length) values.length = headers.length;
+
+        const record = headers.reduce((acc, key, i) => {
+          acc[key] = values[i] ?? null;
+          return acc;
+        }, {} as Record<string, any>);
+
+        if (Object.values(record).every(v => v === null || v === '')) continue;
+
+        page.push(record);
+
+        if (page.length === pageSize) {
+          yield { headers, data: page };
+          hasYieldedData = true;
+          page = [];
+        }
+      }
+
+      // Optional: break if only processing first worksheet
+      // break;
+    }
+
+    if (page.length > 0) {
+      yield { headers, data: page };
+      hasYieldedData = true;
+    }
+
+    // ✅ Yield headers with empty data if only headers were found
+    if (!hasYieldedData && headers.length > 0) {
+      yield { headers, data: [] };
+    }
+  }
+
 }