@servicelabsco/slabs-access-manager 0.1.244 → 0.1.246

package/dist/accessUtility/libraries/read.xls.file.js

@@ -4,175 +4,320 @@ exports.ReadXlsFile = void 0;
 const nestjs_utility_services_1 = require("@servicelabsco/nestjs-utility-services");
 const csv = require("csv-parser");
 const date_fns_1 = require("date-fns");
-const fs_1 = require("fs");
-const path_1 = require("path");
-const bulk_upload_item_entity_1 = require("../entities/bulk.upload.item.entity");
+const http = require("http");
+const https = require("https");
+const stream_1 = require("stream");
+const url_1 = require("url");
 const ExcelJS = require("exceljs");
+var FileExtension;
+(function (FileExtension) {
+    FileExtension["CSV"] = "csv";
+    FileExtension["XLSX"] = "xlsx";
+})(FileExtension || (FileExtension = {}));
+var ColumnType;
+(function (ColumnType) {
+    ColumnType[ColumnType["STRING"] = 1] = "STRING";
+    ColumnType[ColumnType["NUMBER"] = 2] = "NUMBER";
+    ColumnType[ColumnType["DATE"] = 3] = "DATE";
+    ColumnType[ColumnType["DATETIME"] = 4] = "DATETIME";
+    ColumnType[ColumnType["BOOLEAN"] = 5] = "BOOLEAN";
+})(ColumnType || (ColumnType = {}));
 class ReadXlsFile {
     constructor() {
-        this.promises = [];
         this.rowNumber = 0;
         this.totalRows = 0;
-        this.throughCache = true;
         this.sheetCount = {};
     }
     async handle(types) {
-        await this.setStringExclusions();
-        await this.readFiles(types);
-        return this.totalRows || this.rowNumber;
-    }
-    async readFiles(types) {
-        const extension = this.upload.file_url.split('.').pop().toLowerCase();
-        if (extension === 'csv')
-            return this.processCsv(types);
-        if (extension === 'xlsx')
-            return this.processXls(types);
-        throw new nestjs_utility_services_1.OperationException(`Cannot read this extension file`);
-    }
-    getFiles() {
-        const upload = this.upload;
-        const url = decodeURIComponent(upload.file_url);
-        const size = upload.attributes?.size;
-        if (!size)
-            return [url];
-        if (size && size <= 20000000)
-            return [url];
-        const count = upload.attributes?.file_count;
-        if (!count)
-            return;
-        const extension = url.split('.').pop();
-        const baseFile = url.replace(`.${extension}`, '');
-        const urls = [];
-        for (let i = 0; i < count; ++i) {
-            const url = `${baseFile}_${i}.${extension}`;
-            urls.push(url);
+        try {
+            await this.initialize();
+            await this.processFiles(types);
+            await this.flushBatch();
+            return this.totalRows || this.rowNumber;
+        }
+        catch (error) {
+            throw new nestjs_utility_services_1.OperationException(`Failed to process files: ${error.message}`);
         }
-        return urls;
     }
-    async processXls(types) {
-        const files = this.getFiles();
+    async initialize() {
+        if (!this.upload?.file_url) {
+            throw new nestjs_utility_services_1.OperationException('Upload entity or file URL is missing');
+        }
+        this.config = await this.loadConfiguration();
+        this.resetCounters();
+    }
+    async loadConfiguration() {
+        const [stringExclusions, batchSize, maxFileLength] = await Promise.all([
+            this.loadStringExclusions(),
+            this.propertyService.get('bulk.item.batch.size', '100'),
+            this.propertyService.get('bulk.item.max.file.length', '1000000000'),
+        ]);
+        const redisKey = `bulk-upload-${this.upload.id}`;
+        await this.redisService.del(redisKey);
+        return {
+            stringExclusions,
+            batchSize: parseInt(batchSize, 10),
+            maxFileLength: parseInt(maxFileLength, 10),
+            redisKey,
+        };
+    }
+    async loadStringExclusions() {
+        const defaultValue = JSON.stringify(["'", '`', '"']);
+        const value = await this.propertyService.get('bulk.item.string.exclusions', defaultValue);
+        return JSON.parse(value);
+    }
+    resetCounters() {
+        this.rowNumber = 0;
+        this.totalRows = 0;
+        this.sheetCount = {};
+    }
+    async processFiles(types) {
+        const extension = this.getFileExtension();
+        const files = this.getFileUrls();
         for (const type of types) {
             this.rowNumber = 0;
-            for (const file of files) {
-                await this.excelInit(file);
-                await this.readWorksheet(type);
+            for (const fileUrl of files) {
+                await this.processFile(fileUrl, type, extension);
             }
             this.totalRows += this.rowNumber;
         }
     }
-    async processCsv(types) {
-        const sheet = types[0].name.toLowerCase();
-        const files = this.getFiles();
-        const tmpDirectory = await this.ensureTempDirectory();
-        for (const file of files) {
-            const buffer = await this.uploadService.getBufferFromUrl(file);
-            const tempPath = (0, path_1.join)(tmpDirectory, `./${this.upload.uuid}.${nestjs_utility_services_1.PlatformUtility.generateRandomAlpha(6).toLowerCase()}.${sheet}.csv`);
-            (0, fs_1.writeFileSync)(tempPath, buffer);
-            await this.readCsv(tempPath, types[0], sheet);
-            await (0, fs_1.unlink)(tempPath, () => { });
+    getFileExtension() {
+        const extension = this.upload.file_url.split('.').pop()?.toLowerCase();
+        if (!extension || !Object.values(FileExtension).includes(extension)) {
+            throw new nestjs_utility_services_1.OperationException(`Unsupported file extension: ${extension}`);
         }
+        return extension;
     }
-    async getDictionaryOfType(type) {
-        return this.getDictionary(type.columns);
+    getFileUrls() {
+        const url = decodeURIComponent(this.upload.file_url);
+        const size = this.upload.attributes?.size;
+        if (!size || size <= this.config.maxFileLength) {
+            return [url];
+        }
+        const count = this.upload.attributes?.file_count;
+        if (!count) {
+            throw new nestjs_utility_services_1.OperationException('File count not specified for large file');
+        }
+        return this.generateSplitFileUrls(url, count);
     }
-    async excelInit(url) {
-        this.workbook = new ExcelJS.Workbook();
-        const buffer = await this.uploadService.getBufferFromUrl(url);
-        return this.workbook.xlsx.load(buffer);
+    generateSplitFileUrls(baseUrl, count) {
+        const extension = baseUrl.split('.').pop();
+        const baseFile = baseUrl.replace(`.${extension}`, '');
+        return Array.from({ length: count }, (_, i) => `${baseFile}_${i}.${extension}`);
     }
-    async readWorksheet(type) {
-        if (type.sheet?.all)
-            return this.handleAvailableSheets(type);
-        if (type.sheet?.index)
-            return this.handleSheetIndex(type);
-        if (type.sheet?.name)
-            return this.handleSheetName(type);
-        return this.handleAvailableSheets(type);
+    async processFile(fileUrl, type, extension) {
+        try {
+            switch (extension) {
+                case FileExtension.CSV:
+                    await this.processCsvFile(fileUrl, type);
+                    break;
+                case FileExtension.XLSX:
+                    await this.processExcelFile(fileUrl, type);
+                    break;
+                default:
+                    throw new nestjs_utility_services_1.OperationException(`Unsupported file extension: ${extension}`);
+            }
+        }
+        catch (error) {
+            throw new nestjs_utility_services_1.OperationException(`Failed to process file ${fileUrl}: ${error.message}`);
+        }
     }
-    async handleSheetName(type) {
-        return this.setWorksheet(type, type.sheet.name);
+    async processCsvFile(fileUrl, type) {
+        const sheet = type.name.toLowerCase();
+        await this.readCsvFromStream(fileUrl, type, sheet);
+        await this.flushBatch();
     }
-    async handleSheetIndex(type) {
-        const worksheet = this.workbook.getWorksheet(type.sheet.index);
-        return this.setWorksheet(type, worksheet.name);
+    async processExcelFile(fileUrl, type) {
+        await this.loadExcelWorkbook(fileUrl);
+        await this.processWorksheet(type);
     }
-    async handleAvailableSheets(type) {
-        for (let i = 1; i < 100; ++i) {
+    async loadExcelWorkbook(fileUrl) {
+        try {
+            this.workbook = new ExcelJS.Workbook();
+            const buffer = await this.uploadService.getBufferFromUrl(fileUrl);
+            await this.workbook.xlsx.load(buffer);
+        }
+        catch (error) {
+            throw new nestjs_utility_services_1.OperationException(`Failed to load Excel workbook: ${error.message}`);
+        }
+    }
+    async processWorksheet(type) {
+        if (type.sheet?.all) {
+            await this.processAllSheets(type);
+        }
+        else if (type.sheet?.index) {
+            await this.processSheetByIndex(type);
+        }
+        else if (type.sheet?.name) {
+            await this.processSheetByName(type);
+        }
+        else {
+            await this.processAllSheets(type);
+        }
+    }
+    async processAllSheets(type) {
+        const maxSheets = 100;
+        for (let i = 1; i <= maxSheets; i++) {
             const worksheet = this.workbook.getWorksheet(i);
-            const sheet = worksheet?.name;
-            if (!sheet)
+            if (!worksheet?.name)
                 continue;
+            const sheetName = worksheet.name;
             this.totalRows += this.rowNumber;
-            this.rowNumber = this.sheetCount[sheet] || 0;
-            await this.setWorksheet(type, sheet);
-            this.setSheetCount(sheet);
+            this.rowNumber = this.sheetCount[sheetName] || 0;
+            await this.processSheet(type, sheetName);
+            this.sheetCount[sheetName] = this.rowNumber;
+        }
+    }
+    async processSheetByIndex(type) {
+        const worksheet = this.workbook.getWorksheet(type.sheet.index);
+        if (!worksheet) {
+            throw new nestjs_utility_services_1.OperationException(`Worksheet at index ${type.sheet.index} not found`);
         }
+        await this.processSheet(type, worksheet.name);
     }
-    setSheetCount(sheet) {
-        if (!this.sheetCount[sheet])
-            this.sheetCount[sheet] = 0;
-        this.sheetCount[sheet] = this.rowNumber;
-    }
-    async setWorksheet(type, sheet) {
-        const tmpDirectory = await this.ensureTempDirectory();
-        const tempPath = (0, path_1.join)(tmpDirectory, `./${this.upload.uuid}.${nestjs_utility_services_1.PlatformUtility.generateRandomAlpha(6).toLowerCase()}.${sheet}.csv`);
-        const writeStream = (0, fs_1.createWriteStream)(tempPath);
-        await this.workbook.csv.write(writeStream, { sheetName: sheet });
-        await this.readCsv(tempPath, type, sheet);
-        return (0, fs_1.unlink)(tempPath, () => { });
-    }
-    async readCsv(tempPath, type, sheet) {
-        const fn = this;
-        const columns = await this.getDictionaryOfType(type);
+    async processSheetByName(type) {
+        await this.processSheet(type, type.sheet.name);
+    }
+    async processSheet(type, sheetName) {
+        const csvBuffer = await this.convertSheetToCsvBuffer(sheetName);
+        await this.readCsvFromBuffer(csvBuffer, type, sheetName);
+        await this.flushBatch();
+    }
+    async convertSheetToCsvBuffer(sheetName) {
         return new Promise((resolve, reject) => {
-            this.promises = [];
-            (0, fs_1.createReadStream)(tempPath)
-                .pipe(csv({
-                mapHeaders: ({ header, index }) => fn.mapHeader(header, columns.headers),
-                mapValues: ({ header, index, value }) => fn.castValue(columns, header, value),
-            }))
-                .on('data', (data) => {
-                ++fn.rowNumber;
-                fn.promises.push(fn.processRow(sheet, fn.rowNumber, data, type, columns));
+            const chunks = [];
+            const writeStream = new stream_1.Writable({
+                write(chunk, encoding, callback) {
+                    chunks.push(chunk);
+                    callback();
+                },
+            });
+            writeStream.on('finish', () => {
+                resolve(Buffer.concat(chunks));
+            });
+            writeStream.on('error', (error) => {
+                reject(error);
+            });
+            this.workbook.csv.write(writeStream, { sheetName });
+        });
+    }
+    async readCsvFromStream(fileUrl, type, sheetName) {
+        const columns = await this.buildColumnDictionary(type);
+        const stream = await this.createStreamFromUrl(fileUrl);
+        return new Promise((resolve, reject) => {
+            const csvStream = stream.pipe(csv({
+                mapHeaders: ({ header }) => this.mapHeader(header, columns.headers),
+                mapValues: ({ header, value }) => this.castValue(columns, header, value),
+            }));
+            csvStream
+                .on('data', async (data) => {
+                try {
+                    this.rowNumber++;
+                    await this.processRow(sheetName, this.rowNumber, data, type, columns);
+                }
+                catch (error) {
+                    reject(new Error(`Error processing row ${this.rowNumber}: ${error.message}`));
+                }
             })
-                .on('end', () => {
-                resolve(fn.rowNumber);
+                .on('end', () => resolve())
+                .on('error', (error) => reject(new Error(`CSV parsing error: ${error.message}`)));
+        });
+    }
+    async createStreamFromUrl(fileUrl) {
+        const url = new url_1.URL(fileUrl);
+        const protocol = url.protocol === 'https:' ? https : http;
+        return new Promise((resolve, reject) => {
+            const request = protocol.get(fileUrl, (response) => {
+                if (response.statusCode !== 200) {
+                    reject(new Error(`Failed to fetch file: ${response.statusCode}`));
+                    return;
+                }
+                resolve(response);
             });
+            request.on('error', (error) => {
+                reject(error);
+            });
+        });
+    }
+    async readCsvFromBuffer(buffer, type, sheetName) {
+        const columns = await this.buildColumnDictionary(type);
+        return new Promise((resolve, reject) => {
+            const bufferStream = new stream_1.Readable();
+            bufferStream.push(buffer);
+            bufferStream.push(null);
+            const csvStream = bufferStream.pipe(csv({
+                mapHeaders: ({ header }) => this.mapHeader(header, columns.headers),
+                mapValues: ({ header, value }) => this.castValue(columns, header, value),
+            }));
+            csvStream
+                .on('data', async (data) => {
+                try {
+                    this.rowNumber++;
+                    await this.processRow(sheetName, this.rowNumber, data, type, columns);
+                }
+                catch (error) {
+                    reject(new Error(`Error processing row ${this.rowNumber}: ${error.message}`));
+                }
+            })
+                .on('end', () => resolve())
+                .on('error', (error) => reject(new Error(`CSV parsing error: ${error.message}`)));
         });
     }
-    async processRow(sheet, row, data, type, columns) {
-        this.setData(type, data);
-        const shouldIgnore = this.shouldIgnore(data, columns.mandatory);
-        if (shouldIgnore)
-            data = { ...data, ignore_data: shouldIgnore };
-        if (this.throughCache) {
-            const obj = { sheet, row, data, bulk_upload_id: this.upload.id, type_id: type.id };
-            return this.pushToBulkItemJob.dispatch(obj);
+    async processRow(sheet, row_num, data, type, columns) {
+        const processedData = this.applyDataTransformations(type, data);
+        const shouldIgnore = this.shouldIgnoreRow(processedData, columns.mandatory);
+        if (shouldIgnore) {
+            processedData.ignore_data = shouldIgnore;
         }
-        const r = await bulk_upload_item_entity_1.BulkUploadItemEntity.firstOrNew({ bulk_upload_id: this.upload.id, sheet, row_num: row, type_id: type.id });
-        r.data = data;
-        await r.save();
+        const rowData = {
+            sheet,
+            row_num,
+            data: processedData,
+            bulk_upload_id: this.upload.id,
+            type_id: type.id,
+        };
+        return this.addToBatch(rowData);
     }
-    setData(type, data) {
-        if (!type?.script?.script)
+    applyDataTransformations(type, data) {
+        const script = type?.attributes?.script;
+        if (!script?.script) {
             return data;
-        eval(type.script.script);
-        return data;
+        }
+        try {
+            eval(script.script);
+            return data;
+        }
+        catch (error) {
+            throw new nestjs_utility_services_1.OperationException(`Data transformation script failed: ${error.message}`);
+        }
     }
-    castValue(columns, header, value) {
-        if (columns.stringColumns.includes(header))
-            return this.getStringValue(value);
-        if (columns.numberColumns.includes(header))
-            return this.getNumberValue(value);
-        if (columns.booleanColumns.includes(header))
-            return this.getBooleanValue(value);
-        if (columns.dateColumns.includes(header)) {
-            const column = columns.dateObject[header];
-            return this.getDateValue(value, column);
+    shouldIgnoreRow(data, mandatoryFields) {
+        return mandatoryFields.some((field) => !data[field]);
+    }
+    async addToBatch(rowData) {
+        await this.redisService.lpush(this.config.redisKey, JSON.stringify(rowData));
+    }
+    async flushBatch() {
+        const key = this.config.redisKey;
+        let items = [];
+        while (true) {
+            const item = await this.redisService.lpop(key);
+            if (!item)
+                return this.pushToQueue(items);
+            items.push(JSON.parse(item));
+            if (items.length >= this.config.batchSize) {
+                await this.pushToQueue(items);
+                items = [];
+            }
         }
-        return this.getStringValue(value);
     }
-    getDictionary(columns) {
+    async pushToQueue(items) {
+        if (!items?.length)
+            return;
+        return this.pushToBulkItemJob.dispatch(items);
+    }
+    async buildColumnDictionary(type) {
         const headers = {};
         const stringColumns = [];
         const numberColumns = [];
@@ -180,95 +325,91 @@ class ReadXlsFile {
         const dateColumns = [];
         const dateObject = {};
         const mandatory = [];
-        columns.forEach((column) => {
+        type.columns.forEach((column) => {
             const header = column.identifier;
             headers[column.name] = header;
-            if (column.column_type_id === 1)
-                stringColumns.push(header);
-            if (column.column_type_id === 2)
-                numberColumns.push(header);
-            if (column.column_type_id === 5)
-                booleanColumns.push(header);
-            if ([3, 4].includes(column.column_type_id)) {
-                dateColumns.push(header);
-                dateObject[header] = column;
+            switch (column.column_type_id) {
+                case ColumnType.STRING:
+                    stringColumns.push(header);
+                    break;
+                case ColumnType.NUMBER:
+                    numberColumns.push(header);
+                    break;
+                case ColumnType.BOOLEAN:
+                    booleanColumns.push(header);
+                    break;
+                case ColumnType.DATE:
+                case ColumnType.DATETIME:
+                    dateColumns.push(header);
+                    dateObject[header] = column;
+                    break;
             }
-            if (column.is_mandatory && column.column_type_id !== 5)
+            if (column.is_mandatory && column.column_type_id !== ColumnType.BOOLEAN) {
                 mandatory.push(header);
+            }
         });
         return { headers, stringColumns, numberColumns, booleanColumns, dateColumns, dateObject, mandatory };
     }
     mapHeader(header, headers) {
-        const str = header
+        const normalizedHeader = header
             .replace(/[^a-zA-Z0-9_ ]/g, ' ')
             .trim()
             .replace(/\s+/g, '_')
             .trim();
-        return headers[str] || str.toLowerCase().replaceAll(' ', '_');
+        return headers[normalizedHeader] || normalizedHeader.toLowerCase().replaceAll(' ', '_');
     }
-    getStringValue(str) {
-        if (!str)
+    castValue(columns, header, value) {
+        if (columns.stringColumns.includes(header)) {
+            return this.castToString(value);
+        }
+        if (columns.numberColumns.includes(header)) {
+            return this.castToNumber(value);
+        }
+        if (columns.booleanColumns.includes(header)) {
+            return this.castToBoolean(value);
+        }
+        if (columns.dateColumns.includes(header)) {
+            const column = columns.dateObject[header];
+            return this.castToDate(value, column);
+        }
+        return this.castToString(value);
+    }
+    castToString(value) {
+        if (!value)
             return null;
-        return str.replace(new RegExp(`[${this.stringExclusions.join('')}]`, 'g'), '').trim();
+        return value.replace(new RegExp(`[${this.config.stringExclusions.join('')}]`, 'g'), '').trim();
     }
-    getNumberValue(str) {
-        if (!str)
+    castToNumber(value) {
+        if (!value)
             return null;
-        str = str.replace(/[^a-zA-Z0-9.-]/g, '').trim();
-        if (str.length)
-            return +str.trim();
-        return null;
+        const cleanedValue = value.replace(/[^a-zA-Z0-9.-]/g, '').trim();
+        const parsed = parseFloat(cleanedValue);
+        return isNaN(parsed) ? null : parsed;
     }
-    getBooleanValue(str) {
-        if (!str)
-            return false;
-        if (!str.length)
+    castToBoolean(value) {
+        if (!value || !value.trim())
             return false;
-        str = str.replace(/[^a-zA-Z0-9]/g, '').trim();
-        if (str.toLowerCase() === 'true')
-            return true;
-        return false;
+        const cleanedValue = value
+            .replace(/[^a-zA-Z0-9]/g, '')
+            .trim()
+            .toLowerCase();
+        return cleanedValue === 'true';
     }
-    getDateValue(str, column) {
-        if (!str)
+    castToDate(value, column) {
+        if (!value)
             return null;
         try {
-            const d = this.getDate(str, column);
+            const date = this.parseDate(value, column);
             const offset = column.attributes?.offset || 0;
-            return d instanceof Date && isFinite(d.getSeconds()) ? (0, date_fns_1.subMinutes)(d, offset) : null;
+            return date instanceof Date && isFinite(date.getTime()) ? (0, date_fns_1.subMinutes)(date, offset) : null;
         }
         catch (error) {
             return null;
         }
     }
-    shouldIgnore(data, mandatory) {
-        for (const str of mandatory) {
-            if (!data[str])
-                return true;
-        }
-        return false;
-    }
-    getDate(str, column) {
+    parseDate(value, column) {
         const format = column.attributes?.format;
-        if (!format)
-            return new Date(str);
-        return (0, date_fns_1.parse)(str, format, new Date());
-    }
-    async ensureTempDirectory() {
-        const appRootPath = (0, path_1.dirname)(require.main.filename);
-        const tempDir = (0, path_1.resolve)(appRootPath, '..', 'tmp');
-        if (!(0, fs_1.existsSync)(tempDir)) {
-            (0, fs_1.mkdirSync)(tempDir);
-        }
-        return tempDir;
-    }
-    async setStringExclusions() {
-        if (this.stringExclusions)
-            return this.stringExclusions;
-        const defaultValue = JSON.stringify(["'", '`', '"']);
-        const value = await this.propertyService.get('bulk.item.string.exclusions', defaultValue);
-        this.stringExclusions = JSON.parse(value);
-        return this.stringExclusions;
+        return format ? (0, date_fns_1.parse)(value, format, new Date()) : new Date(value);
     }
 }
 exports.ReadXlsFile = ReadXlsFile;