@devassure/tpp-engine 0.0.0-stage → 1.8.1

package/src/library/excel/excel-utils.js

@@ -0,0 +1,922 @@
+const fs = require('fs');
+const path = require('path');
+const Excel = require('exceljs');
+const xlsx = require('xlsx');
+
+const { Logger } = require('../../utils/logger');
+const { FileUtils } = require('../file/file-utils');
+const { TableUtils } = require('../table/table-utils');
+const { processExcel } = require('../../utils/vectordb-utils');
+
+const {
+  getLastElements,
+  getElementsInRange,
+  getFirstElements,
+  getElementsAtIndices,
+  getAnyElements,
+  sleep,
+  getBoolean,
+  isNumber
+} = require('../../utils/common-utils');
+
+class ExcelUtils {
+  constructor(caller, blacklistedLogs, customConfig) {
+    this.caller = caller;
+    this.blacklistedLogs = blacklistedLogs;
+    this.customConfig = customConfig;
+    this.fileUtils = new FileUtils();
+    this.dependenciesPath = customConfig?.dependenciesPath;
+  }
+
+  setCaller(caller) {
+    this.caller = caller;
+  }
+
+  setBlacklistedLogs(blacklistedLogs) {
+    this.blacklistedLogs = blacklistedLogs;
+  }
+
+  // ---------- Small internal helpers ----------
+
+  _getLogger(logLabel) {
+    return new Logger(this.caller, this.blacklistedLogs);
+  }
+
+  _ensureFileExists(rawPath, logger, logLabel) {
+    const filePath = this.fileUtils.getFilePath(rawPath);
+    if (!fs.existsSync(filePath)) {
+      logger.failureLog(logLabel);
+      throw { message: `Excel file ${filePath} does not exists!` };
+    }
+    return filePath;
+  }
+
+  _cleanupTempFile(filePath) {
+    if (!filePath) return;
+    try {
+      if (fs.existsSync(filePath)) {
+        fs.unlinkSync(filePath);
+      }
+    } catch {
+      // ignore cleanup errors
+    }
+  }
+
+  _isSheetsReadError(err) {
+    return err?.message?.includes("reading 'sheets'");
+  }
+
+  async _fixFileWithXlsxAndSleep(filePath, logLabel, logger, sleepMs = 1000) {
+    const fixed = await this.fixExcelFileWithXlsx(filePath, logLabel, logger);
+    await sleep(sleepMs);
+    return fixed;
+  }
+
+  _compareAndLogRowDifference(row1, row2, rowNumber, sheetName, allSheets, logLabel, logger, failures) {
+    if (JSON.stringify(row1) !== JSON.stringify(row2)) {
+      let message =
+        !row1?.length
+          ? `Row ${rowNumber} is empty in first file`
+          : !row2?.length
+          ? `Row ${rowNumber} is empty in second file`
+          : `❌ Row ${rowNumber} is different!`;
+
+      if (allSheets && sheetName) {
+        message = `${sheetName}: ${message}`;
+      }
+      failures.push(message);
+      logger.log(logLabel, message);
+
+      if (row1?.length && row2?.length) {
+        logger.log(logLabel, `row in 1st File: ${JSON.stringify(row1)}`);
+        logger.log(logLabel, `row in 2nd File: ${JSON.stringify(row2)}`);
+      }
+      return true;
+    }
+    return false;
+  }
+
+  // ---------- Public wrappers (unchanged behavior) ----------
+
+  async compare_all_sheets(filePath1, filePath2) {
+    const logLabel = `Excel.Compare data of all sheets of ${filePath1} and ${filePath2}`;
+    const logger = this._getLogger(logLabel);
+    logger.startLog(logLabel);
+    await this.compareExcelFiles(filePath1, filePath2, undefined, undefined, true, logLabel);
+    logger.endLog(logLabel);
+  }
+
+  async compare_first_sheet(filePath1, filePath2) {
+    const logLabel = `Excel.Compare data of first sheet of ${filePath1} and ${filePath2}`;
+    const logger = this._getLogger(logLabel);
+    logger.startLog(logLabel);
+    await this.compareExcelFiles(filePath1, filePath2, undefined, undefined, false, logLabel);
+    logger.endLog(logLabel);
+  }
+
+  async compare_given_sheet(filePath1, sheetName1, filePath2, sheetName2) {
+    const logLabel = `Excel.Compare data of file ${filePath1} sheet ${sheetName1} and file ${filePath2} sheet ${sheetName2}`;
+    const logger = this._getLogger(logLabel);
+    logger.startLog(logLabel);
+    await this.compareExcelFiles(filePath1, filePath2, sheetName1, sheetName2, false, logLabel);
+    logger.endLog(logLabel);
+  }
+
+  async read_first_sheet_rows_as_array(filePath) {
+    return await this.read_rows_as_array(filePath);
+  }
+
+  async read_rows_as_array(filePath, sheetName) {
+    const logLabel = sheetName
+      ? `Excel.Read sheet ${sheetName} rows as array ${filePath}`
+      : `Excel.Read first sheet rows as array ${filePath}`;
+
+    const logger = this._getLogger(logLabel);
+    logger.startLog(logLabel);
+    const data = await this.readSheetAsArray(filePath, sheetName, logLabel);
+    logger.endLog(logLabel);
+    return data;
+  }
+
+  async read_first_sheet_rows_as_object(filePath) {
+    return await this.read_rows_as_object(filePath);
+  }
+
+  async read_rows_as_object(filePath, sheetName) {
+    const logLabel = sheetName
+      ? `Excel.Read sheet ${sheetName} rows as objects ${filePath}`
+      : `Excel.Read first sheet rows as objects ${filePath}`;
+
+    const logger = this._getLogger(logLabel);
+    logger.startLog(logLabel);
+
+    const dataArray = await this.readSheetAsArray(filePath, sheetName, logLabel);
+    const fields = dataArray[0] || [];
+    const data = dataArray.slice(1).map(cells => {
+      const row = {};
+      for (let i = 0; i < fields.length; i++) {
+        if (!fields[i]) continue;
+        row[`${fields[i]}`] = i < cells.length ? cells[i] || '' : '';
+      }
+      return row;
+    });
+
+    logger.endLog(logLabel);
+    return data;
+  }
+
+  // ---------- Core reading helpers ----------
+
+  async readFixedFileAsArray(filePath, sheetName, logLabel, rowCount, ignoreFirstRow) {
+    const workbook = new Excel.Workbook();
+    await workbook.xlsx.readFile(filePath);
+
+    let sheet = sheetName
+      ? workbook.getWorksheet(sheetName)
+      : workbook.worksheets[0];
+
+    if (!sheet) {
+      const msg = sheetName
+        ? `Sheet with name ${sheetName} not found!`
+        : 'No sheet found in the file!';
+      throw { message: msg };
+    }
+
+    const result = [];
+    const maxRow = rowCount ? Math.min(rowCount, sheet.rowCount) : sheet.rowCount;
+
+    for (let i = 1; i <= maxRow; i++) {
+      const row = sheet.getRow(i);
+      if (!row || row.values.length <= 1) continue;
+      const cleanedRow = row.values.slice(1); // first index unused
+      result.push(cleanedRow);
+    }
+
+    if (ignoreFirstRow && result.length > 0) {
+      return result.slice(1);
+    }
+
+    return result;
+  }
+
+  async fixExcelFileWithXlsx(filePath, logLabel, loggerFromCaller) {
+    const logger = loggerFromCaller || this._getLogger(logLabel);
+    logger.log(
+      logLabel,
+      'Excel file appears corrupted or incompatible with ExcelJS stream reader. Attempting to fix using xlsx library...'
+    );
+
+    try {
+      const workbook = xlsx.readFile(filePath);
+
+      if (!workbook.SheetNames || workbook.SheetNames.length === 0) {
+        throw { message: 'No sheets found in the Excel file' };
+      }
+
+      const newWorkbook = new Excel.Workbook();
+
+      workbook.SheetNames.forEach(sheetName => {
+        const worksheet = workbook.Sheets[sheetName];
+        const data = xlsx.utils.sheet_to_json(worksheet, { header: 1, defval: '' });
+        const newWorksheet = newWorkbook.addWorksheet(sheetName);
+        data.forEach(row => newWorksheet.addRow(row));
+      });
+
+      const tempDir = path.join(path.dirname(filePath), '.tpp-excel-temp');
+      if (!fs.existsSync(tempDir)) {
+        fs.mkdirSync(tempDir, { recursive: true });
+      }
+
+      const fixedFilePath = path.join(
+        tempDir,
+        `fixed_${Date.now()}_${path.basename(filePath)}`
+      );
+      await newWorkbook.xlsx.writeFile(fixedFilePath);
+
+      logger.log(logLabel, `Fixed Excel file created at: ${fixedFilePath}`);
+      return fixedFilePath;
+    } catch (error) {
+      const msg = error?.message || String(error);
+      logger.log(logLabel, `Failed to fix Excel file: ${msg}`);
+      throw { message: `Failed to fix Excel file: ${msg}` };
+    }
+  }
+
+  async readSheetAsArray(
+    filePath,
+    sheetName,
+    logLabel,
+    rowCount,
+    ignoreFirstRow,
+    retryCount,
+    fixedFilePath
+  ) {
+    const logger = this._getLogger(logLabel);
+    filePath = this.fileUtils.getFilePath(filePath);
+
+    if (!fs.existsSync(filePath)) {
+      logger.failureLog(logLabel);
+      throw { message: `Excel file ${filePath} does not exists!` };
+    }
+
+    try {
+      const workbookReader = new Excel.stream.xlsx.WorkbookReader(filePath);
+      let sheetFound = false;
+      const result = [];
+
+      for await (const sheet of workbookReader) {
+        if (sheetName && sheet.name !== sheetName) continue;
+
+        sheetFound = true;
+
+        for await (const row of sheet) {
+          if (!row || !row.values) continue;
+          const cleanedRow = row.values.slice(1);
+          result.push(cleanedRow);
+
+          if (rowCount && row.number > rowCount) {
+            break;
+          }
+        }
+        break;
+      }
+
+      if (!sheetFound) {
+        const message = sheetName
+          ? `Sheet with name ${sheetName} not found!`
+          : 'No sheet found in the file!';
+        logger.failureLog(logLabel);
+        throw { message };
+      }
+
+      // Sheet found but nothing read -> try auto-fix
+      if (result.length === 0 && !retryCount && !fixedFilePath) {
+        logger.log(logLabel, 'Sheet found but no rows were read. Attempting to fix file...');
+        try {
+          const newFixedPath = await this._fixFileWithXlsxAndSleep(filePath, logLabel, logger);
+          const fixedResult = await this.readFixedFileAsArray(
+            newFixedPath,
+            sheetName,
+            logLabel,
+            rowCount,
+            ignoreFirstRow
+          );
+          this._cleanupTempFile(newFixedPath);
+          return fixedResult;
+        } catch (fixError) {
+          logger.log(logLabel, `File fix attempt failed: ${fixError.message}`);
+        }
+      }
+
+      if (ignoreFirstRow && result.length > 0) {
+        return result.slice(1);
+      }
+
+      return result;
+    } catch (err) {
+      const message = err?.message || '';
+
+      // Special invalid signature case
+      if (!retryCount && message.includes('invalid signature: 0x4a4')) {
+        const newPath = await this.extractAndRewriteCleanXlsx(filePath, logLabel);
+        await sleep(1000);
+        return await this.readSheetAsArray(
+          newPath,
+          sheetName,
+          logLabel,
+          rowCount,
+          ignoreFirstRow,
+          1,
+          fixedFilePath
+        );
+      }
+
+      // Stream "reading 'sheets'" error handling
+      if (this._isSheetsReadError(err)) {
+        // try auto-fix first
+        if (!retryCount && !fixedFilePath) {
+          logger.log(logLabel, 'Error reading sheets. Attempting to fix file...');
+          try {
+            const newFixedPath = await this._fixFileWithXlsxAndSleep(filePath, logLabel, logger);
+            const fixedResult = await this.readFixedFileAsArray(
+              newFixedPath,
+              sheetName,
+              logLabel,
+              rowCount,
+              ignoreFirstRow
+            );
+            this._cleanupTempFile(newFixedPath);
+            return fixedResult;
+          } catch (fixError) {
+            if (fixError.message?.includes('not found')) {
+              throw fixError;
+            }
+            logger.log(
+              logLabel,
+              `File fix attempt failed: ${fixError.message}. Retrying with original file...`
+            );
+          }
+        }
+
+        // Retry with original file
+        if (!fixedFilePath && (!retryCount || retryCount < 4)) {
+          console.log('Error in iterating sheets. Retrying...', message);
+          await sleep(5000);
+          return await this.readSheetAsArray(
+            filePath,
+            sheetName,
+            logLabel,
+            rowCount,
+            ignoreFirstRow,
+            (retryCount || 0) + 1,
+            fixedFilePath
+          );
+        }
+
+        throw err;
+      }
+
+      throw err;
+    }
+  }
+
+  // ---------- Compare files ----------
+
+  async compareExcelFiles(
+    filePath1,
+    filePath2,
+    sheetName1,
+    sheetName2,
+    allSheets,
+    logLabel,
+    retryCount,
+    fixedFilePath1,
+    fixedFilePath2
+  ) {
+    const logger = this._getLogger(logLabel);
+
+    filePath1 = this._ensureFileExists(filePath1, logger, logLabel);
+    filePath2 = this._ensureFileExists(filePath2, logger, logLabel);
+
+    if (filePath1.toLowerCase() === filePath2.toLowerCase()) {
+      logger.failureLog(logLabel);
+      throw { message: 'Given file paths are same!' };
+    }
+
+    // Check file sizes - if both are < 2MB, use xlsx library for comparison
+    const fileSize1 = fs.statSync(filePath1).size;
+    const fileSize2 = fs.statSync(filePath2).size;
+    const twoMB = 2 * 1024 * 1024; // 2MB in bytes
+
+    if (fileSize1 < twoMB && fileSize2 < twoMB) {
+      const workbook1 = xlsx.readFile(filePath1);
+      const workbook2 = xlsx.readFile(filePath2);
+
+      const failures = [];
+      let sheetFound = false;
+
+      for (const sheetName of workbook1.SheetNames) {
+        if (sheetName1 && sheetName !== sheetName1) {
+          continue;
+        }
+
+        sheetFound = true;
+
+        const targetSheetName = sheetName2 || sheetName;
+        if (!workbook2.SheetNames.includes(targetSheetName)) {
+          logger.failureLog(logLabel);
+          throw { message: `Sheet ${sheetName} not found in file2` };
+        }
+
+        const worksheet1 = workbook1.Sheets[sheetName];
+        const worksheet2 = workbook2.Sheets[targetSheetName];
+        const data1 = xlsx.utils.sheet_to_json(worksheet1, { header: 1, defval: '' });
+        const data2 = xlsx.utils.sheet_to_json(worksheet2, { header: 1, defval: '' });
+
+        const maxRows = Math.max(data1.length, data2.length);
+        for (let i = 0; i < maxRows; i++) {
+          const row1 = data1[i] || [];
+          const row2 = data2[i] || [];
+
+          this._compareAndLogRowDifference(row1, row2, i + 1, sheetName, allSheets, logLabel, logger, failures);
+        }
+
+        if (!sheetName1 && !allSheets) {
+          break;
+        }
+      }
+
+      if (!sheetFound) {
+        const message = sheetName1
+          ? `Sheet with name ${sheetName1} not found!`
+          : 'No sheet found in the first file!';
+        logger.failureLog(logLabel);
+        throw { message };
+      }
+
+      if (failures.length) {
+        logger.failureLog(logLabel);
+        throw { message: `There are ${failures.length} rows that do not match in the 2 files!` };
+      }
+
+      return;
+    }
+
+    try {
+      const workbookReader1 = new Excel.stream.xlsx.WorkbookReader(filePath1);
+      let sheetFound = false;
+      const failures = [];
+      let rowsRead = 0;
+
+      for await (const sheet1 of workbookReader1) {
+        console.log(`Reading sheet1: ${sheet1.name}`);
+
+        if (sheetName1 && sheet1.name !== sheetName1) {
+          continue;
+        }
+
+        sheetFound = true;
+
+        const sheet2 = await this.getMatchingSheet(
+          filePath2,
+          sheetName2 || sheet1.name,
+          undefined,
+          fixedFilePath2
+        );
+
+        if (!sheet2) {
+          logger.failureLog(logLabel);
+          throw { message: `Sheet ${sheet1.name} not found in file2` };
+        }
+
+        const iter1 = sheet1[Symbol.asyncIterator]();
+        const iter2 = sheet2[Symbol.asyncIterator]();
+
+        while (true) {
+          const [res1, res2] = await Promise.all([iter1.next(), iter2.next()]);
+
+          if (res1.done && res2.done) break;
+
+          const row1 = res1.value?.values?.slice(1) || [];
+          const row2 = res2.value?.values?.slice(1) || [];
+
+          if (res1.value || res2.value) {
+            rowsRead++;
+          }
+
+          const rowNumber = res1.value?.number || res2.value?.number || '';
+          this._compareAndLogRowDifference(row1, row2, rowNumber, sheet1.name, allSheets, logLabel, logger, failures);
+        }
+
+        if (!sheetName1 && !allSheets) {
+          break;
+        }
+      }
+
+      // If sheet found but nothing read -> try fix
+      if (sheetFound && rowsRead === 0 && !retryCount && !fixedFilePath1 && !fixedFilePath2) {
+        logger.log(logLabel, 'Sheets found but no rows were read. Attempting to fix files...');
+        try {
+          const newFixedPath1 = await this._fixFileWithXlsxAndSleep(filePath1, logLabel, logger);
+          const newFixedPath2 = await this._fixFileWithXlsxAndSleep(filePath2, logLabel, logger);
+          const result = await this.compareExcelFiles(
+            newFixedPath1,
+            newFixedPath2,
+            sheetName1,
+            sheetName2,
+            allSheets,
+            logLabel,
+            1,
+            newFixedPath1,
+            newFixedPath2
+          );
+          this._cleanupTempFile(newFixedPath1);
+          this._cleanupTempFile(newFixedPath2);
+          return result;
+        } catch (fixError) {
+          logger.log(logLabel, `File fix attempt failed: ${fixError.message}`);
+        }
+      }
+
+      if (!sheetFound) {
+        const message = sheetName1
+          ? `Sheet with name ${sheetName1} not found!`
+          : 'No sheet found in the first file!';
+        logger.failureLog(logLabel);
+        throw { message };
+      }
+
+      if (failures.length) {
+        logger.failureLog(logLabel);
+        throw { message: `There are ${failures.length} rows that do not match in the 2 files!` };
+      }
+    } catch (err) {
+      const message = err?.message || '';
+
+      if (!retryCount && message.includes('invalid signature: 0x4a4')) {
+        const newPath1 = await this.extractAndRewriteCleanXlsx(filePath1, logLabel);
+        await sleep(1000);
+        const newPath2 = await this.extractAndRewriteCleanXlsx(filePath2, logLabel);
+        await sleep(1000);
+        return await this.compareExcelFiles(
+          newPath1,
+          newPath2,
+          sheetName1,
+          sheetName2,
+          allSheets,
+          logLabel,
+          1,
+          fixedFilePath1,
+          fixedFilePath2
+        );
+      }
+
+      if (this._isSheetsReadError(err)) {
+        if (!retryCount && !fixedFilePath1 && !fixedFilePath2) {
+          logger.log(logLabel, 'Error reading sheets. Attempting to fix files...');
+          try {
+            const newFixedPath1 = await this._fixFileWithXlsxAndSleep(filePath1, logLabel, logger);
+            const newFixedPath2 = await this._fixFileWithXlsxAndSleep(filePath2, logLabel, logger);
+            const result = await this.compareExcelFiles(
+              newFixedPath1,
+              newFixedPath2,
+              sheetName1,
+              sheetName2,
+              allSheets,
+              logLabel,
+              1,
+              newFixedPath1,
+              newFixedPath2
+            );
+            this._cleanupTempFile(newFixedPath1);
+            this._cleanupTempFile(newFixedPath2);
+            return result;
+          } catch (fixError) {
+            logger.log(
+              logLabel,
+              `File fix attempt failed: ${fixError.message}. Retrying with original files...`
+            );
+          }
+        }
+
+        if (!retryCount || retryCount < 4) {
+          console.log('Error in iterating sheets. Retrying...', message);
+          await sleep(5000);
+          return await this.compareExcelFiles(
+            filePath1,
+            filePath2,
+            sheetName1,
+            sheetName2,
+            allSheets,
+            logLabel,
+            (retryCount || 0) + 1,
+            fixedFilePath1,
+            fixedFilePath2
+          );
+        }
+
+        throw err;
+      }
+
+      throw err;
+    }
+  }
+
+  async getMatchingSheet(filePath, targetSheetName, retry, fixedFilePath) {
+    try {
+      const workbookReader = new Excel.stream.xlsx.WorkbookReader(filePath);
+      let sheetFound = false;
+
+      for await (const sheet of workbookReader) {
+        if (sheet.name === targetSheetName) {
+          sheetFound = true;
+          let columnCount = sheet.dimensions?.model?.right || 0;
+
+          if (columnCount === 0 && !retry && !fixedFilePath) {
+            try {
+              const newFixedPath = await this._fixFileWithXlsxAndSleep(
+                filePath,
+                `Finding sheet ${targetSheetName}`,
+                this._getLogger(`Finding sheet ${targetSheetName}`)
+              );
+              return await this.getMatchingSheet(newFixedPath, targetSheetName, 1, newFixedPath);
+            } catch {
+              // fallback to original
+            }
+          }
+
+          return sheet;
+        }
+      }
+
+      if (!sheetFound && !retry && !fixedFilePath) {
+        try {
+          const newFixedPath = await this._fixFileWithXlsxAndSleep(
+            filePath,
+            `Finding sheet ${targetSheetName}`,
+            this._getLogger(`Finding sheet ${targetSheetName}`)
+          );
+          return await this.getMatchingSheet(newFixedPath, targetSheetName, 1, newFixedPath);
+        } catch {
+          // return null below
+        }
+      }
+    } catch (er) {
+      console.log('Error reading matching sheet!', er.message);
+      if ((!retry || retry < 2) && this._isSheetsReadError(er)) {
+        console.log('Retrying find matching sheet on iterator error...', er.message);
+        await sleep(5000);
+        return await this.getMatchingSheet(filePath, targetSheetName, (retry || 0) + 1, fixedFilePath);
+      } else if (!retry && !fixedFilePath && !this._isSheetsReadError(er)) {
+        try {
+          const newFixedPath = await this._fixFileWithXlsxAndSleep(
+            filePath,
+            `Finding sheet ${targetSheetName}`,
+            this._getLogger(`Finding sheet ${targetSheetName}`)
+          );
+          return await this.getMatchingSheet(newFixedPath, targetSheetName, 1, newFixedPath);
+        } catch (fixError) {
+          console.log(`Not able to find sheet ${targetSheetName}`, er);
+          throw er;
+        }
+      } else {
+        console.log(`Not able to find sheet ${targetSheetName}`, er);
+        throw er;
+      }
+    }
+
+    return null;
+  }
+
+  // ---------- Advanced read (modularized) ----------
+
+  async advanced_excel_read(params) {
+    const logLabel = 'Excel.Advanced Read';
+    const logger = this._getLogger(logLabel);
+    logger.startLog(logLabel);
+
+    if (!params.filePath?.trim()) {
+      logger.failureLog(logLabel);
+      throw { message: 'No file path provided!' };
+    }
+
+    const filePath = this._ensureFileExists(params.filePath, logger, logLabel);
+    const sheetName = params.sheetName?.trim() || undefined;
+    const ignore = getBoolean(params.ignoreMissingData);
+    const isFirstRowHeader = getBoolean(params.isFirstRowHeader);
+    const getRowsAsObjects = getBoolean(params.getRowsAsObjects);
+
+    // Read header row once
+    const sheetData = await this.readSheetAsArray(filePath, sheetName, logLabel, 1);
+    const header = sheetData?.at(0) || [];
+    const firstRowHeader = isFirstRowHeader && sheetData?.length > 0;
+
+    let fields = firstRowHeader
+      ? header.map(field => field.trim())
+      : header.map((_h, index) => index);
+
+    const allFields = fields.slice();
+
+    // Filter rows
+    const rowsFilter = params.rows || 'all';
+    const dataRows = await this._readRowsWithFilter({
+      filePath,
+      sheetName,
+      logLabel,
+      rowsFilter,
+      ignore,
+      isFirstRowHeader
+    });
+
+    // Filter columns
+    fields = new TableUtils(this.caller, this.blacklistedLogs, logLabel).filterColumns(
+      params,
+      fields,
+      ignore,
+      firstRowHeader
+    );
+
+    const dataObject = this._buildDataObject({
+      dataRows,
+      fields,
+      allFields,
+      getRowsAsObjects,
+      firstRowHeader
+    });
+
+    logger.endLog(logLabel);
+    return dataObject;
+  }
+
+  async _readRowsWithFilter({
+    filePath,
+    sheetName,
+    logLabel,
+    rowsFilter,
+    ignore,
+    isFirstRowHeader
+  }) {
+    const filter = rowsFilter?.toString().trim().toLowerCase() || 'all';
+
+    const readAll = () =>
+      this.readSheetAsArray(filePath, sheetName, logLabel, undefined, isFirstRowHeader);
+
+    if (filter === 'all') {
+      return await readAll();
+    }
+
+    if (filter.startsWith('last')) {
+      const countStr = filter.replace('last', '').trim();
+      if (!isNumber(countStr)) {
+        throw { message: `${rowsFilter} is not a valid rows filter!` };
+      }
+      const count = parseInt(countStr, 10);
+      let dataRows = await this.readSheetAsArray(
+        filePath,
+        sheetName,
+        logLabel,
+        undefined,
+        isFirstRowHeader
+      );
+      const data = getLastElements(dataRows, count, ignore);
+      if (!data.success) throw { message: data.error };
+      return data.data;
+    }
+
+    if (filter.startsWith('first')) {
+      const countStr = filter.replace('first', '').trim();
+      if (!isNumber(countStr)) {
+        throw { message: `${rowsFilter} is not a valid rows filter!` };
+      }
+      const count = parseInt(countStr, 10);
+      let dataRows = await this.readSheetAsArray(
+        filePath,
+        sheetName,
+        logLabel,
+        count,
+        isFirstRowHeader
+      );
+      const data = getFirstElements(dataRows, count, ignore);
+      if (!data.success) throw { message: data.error };
+      return data.data;
+    }
+
+    if (filter.startsWith('any')) {
+      const countStr = filter.replace('any', '').trim();
+      if (!isNumber(countStr)) {
+        throw { message: `${rowsFilter} is not a valid rows filter!` };
+      }
+      const count = parseInt(countStr, 10);
+      let dataRows = await this.readSheetAsArray(
+        filePath,
+        sheetName,
+        logLabel,
+        count + 10, // now correctly numeric
+        isFirstRowHeader
+      );
+      const data = getAnyElements(dataRows, count, ignore);
+      if (!data.success) throw { message: data.error };
+      return data.data;
+    }
+
+    if (filter.includes('-')) {
+      const [, y] = filter.split('-').map(Number);
+      let dataRows = await this.readSheetAsArray(
+        filePath,
+        sheetName,
+        logLabel,
+        y + 10,
+        isFirstRowHeader
+      );
+      const data = getElementsInRange(dataRows, filter, ignore);
+      if (!data.success) throw { message: data.error };
+      return data.data;
+    }
+
+    if (filter.includes(',')) {
+      const indexArray = filter.split(',').map(Number);
+      const max = Math.max(...indexArray);
+      let dataRows = await this.readSheetAsArray(
+        filePath,
+        sheetName,
+        logLabel,
+        max + 10,
+        isFirstRowHeader
+      );
+      const data = getElementsAtIndices(dataRows, filter, ignore);
+      if (!data.success) throw { message: data.error };
+      return data.data;
+    }
+
+    // single index
+    if (!isNumber(filter)) {
+      throw { message: `${rowsFilter} is not a valid rows filter!` };
+    }
+
+    const index = parseInt(filter.trim(), 10);
+    let dataRows = await this.readSheetAsArray(
+      filePath,
+      sheetName,
+      logLabel,
+      index + 2,
+      isFirstRowHeader
+    );
+
+    if (index < 0 || (index >= dataRows.length && !ignore)) {
+      throw { message: `${rowsFilter} - index is invalid. Data size - ${dataRows.length}` };
+    }
+
+    if (ignore && index >= dataRows.length) {
+      return [];
+    }
+
+    return [dataRows[index]];
+  }
+
+  _buildDataObject({ dataRows, fields, allFields, getRowsAsObjects, firstRowHeader }) {
+    return dataRows.map(cells => {
+      const row = getRowsAsObjects ? {} : [];
+      for (let i = 0; i < allFields.length; i++) {
+        const skip = firstRowHeader ? !fields.includes(allFields[i]) : !fields.includes(i);
+        if (skip) continue;
+        const val = i < cells.length ? cells[i] || '' : '';
+        if (getRowsAsObjects) {
+          const fieldName = firstRowHeader ? allFields[i] : `column_${i}`;
+          row[fieldName] = val;
+        } else {
+          row.push(val);
+        }
+      }
+      return row;
+    });
+  }
+
+  // ---------- Repair via external tool ----------
+
+  async extractAndRewriteCleanXlsx(filePath, parentLogLabel) {
+    const logLabel = `${parentLogLabel} :: Excel file ${filePath} is not properly formatted, fixing the file!`;
+    const logger = this._getLogger(logLabel);
+    logger.startLog(logLabel);
+
+    const inputPath = this.fileUtils.getFilePath(filePath);
+
+    if (!this.dependenciesPath) {
+      logger.failureLog(logLabel);
+      throw {
+        message: 'Dependencies path is not defined! Please set it in the config file.'
+      };
+    }
+
+    const result = await processExcel(this.dependenciesPath, inputPath);
+
+    if (result.status !== 0) {
+      logger.failureLog(logLabel);
+      throw { message: result.message };
+    }
+
+    logger.endLog(logLabel);
+    return result.message;
+  }
+}
+
+exports.ExcelUtils = ExcelUtils;