jtcsv 2.2.7 → 3.0.0

package/csv-to-json.js

@@ -1,3 +1,4 @@
+// @ts-nocheck
 /**
  * CSV to JSON Converter - Node.js Module
  * 
@@ -20,6 +21,7 @@ const {
 const { TransformHooks, predefinedHooks } = require('./src/core/transform-hooks');
 const DelimiterCache = require('./src/core/delimiter-cache');
 const FastPathEngine = require('./src/engines/fast-path-engine');
+const { stripBomFromString, normalizeCsvInput } = require('./src/utils/bom-utils');
 
 // Глобальный экземпляр кэша для авто-детектирования разделителя
 const globalDelimiterCache = new DelimiterCache(100);
@@ -73,19 +75,19 @@ function validateCsvInput(csv, options) {
     throw new ConfigurationError('cache must be an instance of DelimiterCache');
   }
 
-  if (options?.useFastPath !== undefined && typeof options.useFastPath !== 'boolean') {
-    throw new ConfigurationError('useFastPath must be a boolean');
-  }
-
-  if (options?.fastPathMode !== undefined
-    && options.fastPathMode !== 'objects'
-    && options.fastPathMode !== 'compact'
-    && options.fastPathMode !== 'stream') {
-    throw new ConfigurationError('fastPathMode must be "objects", "compact", or "stream"');
-  }
-
-  // Validate hooks
-  if (options?.hooks) {
+  if (options?.useFastPath !== undefined && typeof options.useFastPath !== 'boolean') {
+    throw new ConfigurationError('useFastPath must be a boolean');
+  }
+
+  if (options?.fastPathMode !== undefined
+    && options.fastPathMode !== 'objects'
+    && options.fastPathMode !== 'compact'
+    && options.fastPathMode !== 'stream') {
+    throw new ConfigurationError('fastPathMode must be "objects", "compact", or "stream"');
+  }
+
+  // Validate hooks
+  if (options?.hooks) {
     if (typeof options.hooks !== 'object') {
       throw new ConfigurationError('hooks must be an object');
     }
@@ -114,9 +116,9 @@ function validateCsvInput(csv, options) {
  * Parses a value based on options
  * @private
  */
-function parseCsvValue(value, options) {
-  /* istanbul ignore next */
-  const { trim = true, parseNumbers = false, parseBooleans = false } = options;
+function parseCsvValue(value, options) {
+  /* istanbul ignore next */
+  const { trim = true, parseNumbers = false, parseBooleans = false } = options;
   
   let result = value;
   
@@ -130,13 +132,23 @@ function parseCsvValue(value, options) {
   }
   
   // Parse numbers
-  if (parseNumbers && /^-?\d+(\.\d+)?$/.test(result)) {
-    const num = parseFloat(result);
-    /* istanbul ignore next */
-    if (!isNaN(num)) {
-      return num;
-    }
-  }
+  if (parseNumbers) {
+    // Fast numeric detection: check first character and use parseFloat
+    const trimmed = result.trim();
+    const firstChar = trimmed.charAt(0);
+    // Quick reject: if first character is not digit, minus, or dot, skip
+    if ((firstChar >= '0' && firstChar <= '9') || firstChar === '-' || firstChar === '.') {
+      const num = parseFloat(trimmed);
+      if (!isNaN(num) && isFinite(num)) {
+        // Ensure the whole string represents the same number (no extra characters)
+        // parseFloat ignores trailing non-numeric characters, so we need to verify
+        // that the string matches the parsed number when converted back
+        if (String(num) === trimmed || (trimmed.includes('.') && !isNaN(Number(trimmed)))) {
+          return num;
+        }
+      }
+    }
+  }
   
   // Parse booleans
   if (parseBooleans) {
@@ -150,17 +162,17 @@ function parseCsvValue(value, options) {
   }
   
   // Parse empty strings as null
-  if (result === '') {
-    return null;
-  }
-  
-  return result;
-}
-
-/**
- * Auto-detect CSV delimiter from content with caching support
- * @private
- */
+  if (result === '') {
+    return null;
+  }
+  
+  return result;
+}
+
+/**
+ * Auto-detect CSV delimiter from content with caching support
+ * @private
+ */
 function autoDetectDelimiter(csv, candidates = [';', ',', '\t', '|'], cache = null) {
   // Используем статический метод DelimiterCache с поддержкой кэширования
   return DelimiterCache.autoDetectDelimiter(csv, candidates, cache);
@@ -225,19 +237,21 @@ function csvToJson(csv, options = {}) {
       hasHeaders = true,
       renameMap = {},
       trim = true,
-      parseNumbers = false,
-      parseBooleans = false,
-      maxRows,
-      useCache = true,
-      cache: customCache,
-      useFastPath = true,
-      fastPathMode = 'objects',
-      hooks = {}
-    } = opts;
-
-    if (fastPathMode === 'stream') {
-      return csvToJsonIterator(csv, { ...opts, useFastPath, fastPathMode: 'objects' });
-    }
+      parseNumbers = false,
+      parseBooleans = false,
+      maxRows,
+      useCache = true,
+      cache: customCache,
+      useFastPath = true,
+      fastPathMode = 'objects',
+      hooks = {},
+      stripBom = true,  // New option: strip BOM by default
+      normalizeEncoding = true  // New option: normalize encoding
+    } = opts;
+
+    if (fastPathMode === 'stream') {
+      return csvToJsonIterator(csv, { ...opts, useFastPath, fastPathMode: 'objects' });
+    }
 
     // Выбираем кэш для использования
     const cacheToUse = useCache ? (customCache || globalDelimiterCache) : null;
@@ -261,10 +275,19 @@ function csvToJson(csv, options = {}) {
     // Use provided TransformHooks instance if available
     const finalHooks = hooks.transformHooks || transformHooks;
 
+    // Handle BOM and encoding normalization
+    let normalizedCsv = csv;
+    if (stripBom) {
+      normalizedCsv = stripBomFromString(normalizedCsv);
+    }
+    if (normalizeEncoding) {
+      normalizedCsv = normalizeCsvInput(normalizedCsv, { normalizeLineEndings: true });
+    }
+
     // Apply beforeConvert hooks to CSV string
-    const processedCsv = finalHooks.applyBeforeConvert(csv, { 
+    const processedCsv = finalHooks.applyBeforeConvert(normalizedCsv, {
       operation: 'csvToJson',
-      options: opts 
+      options: opts
     });
 
     // Determine delimiter with caching support
@@ -285,13 +308,13 @@ function csvToJson(csv, options = {}) {
     const result = [];
 
     try {
-      const parseOptions = { delimiter: finalDelimiter };
-      if (useFastPath === false) {
-        parseOptions.forceEngine = 'STANDARD';
-      }
-
-      globalFastPathEngine.parseRows(processedCsv, parseOptions, (fields) => {
-        totalRows++;
+      const parseOptions = { delimiter: finalDelimiter };
+      if (useFastPath === false) {
+        parseOptions.forceEngine = 'STANDARD';
+      }
+
+      globalFastPathEngine.parseRows(processedCsv, parseOptions, (fields) => {
+        totalRows++;
 
         if (!headers) {
           if (hasHeaders) {
@@ -308,10 +331,10 @@ function csvToJson(csv, options = {}) {
           return;
         }
 
-        if (maxRows && totalRows > maxRows) {
-          throw new LimitError(
-            `CSV size exceeds maximum limit of ${maxRows} rows`,
-            maxRows,
+        if (maxRows && totalRows > maxRows) {
+          throw new LimitError(
+            `CSV size exceeds maximum limit of ${maxRows} rows`,
+            maxRows,
             totalRows
           );
         }
@@ -344,26 +367,26 @@ function csvToJson(csv, options = {}) {
           console.warn(`[jtcsv] Line ${totalRows}: ${fields.length - headers.length} extra fields ignored`);
         }
       });
-    } catch (error) {
-      if (error && error.code === 'FAST_PATH_UNCLOSED_QUOTES') {
-        const lineInfo = error.lineNumber ? ` at line ${error.lineNumber}` : '';
-        throw new ParsingError(`Unclosed quotes in CSV${lineInfo}`, error.lineNumber);
-      }
-      throw error;
-    }
+    } catch (error) {
+      if (error && error.code === 'FAST_PATH_UNCLOSED_QUOTES') {
+        const lineInfo = error.lineNumber ? ` at line ${error.lineNumber}` : '';
+        throw new ParsingError(`Unclosed quotes in CSV${lineInfo}`, error.lineNumber);
+      }
+      throw error;
+    }
 
     if (!headers) {
       return [];
     }
 
-    if (totalRows > 1000000 && !maxRows && process.env.NODE_ENV !== 'test') {
-      console.warn(
-        'Warning: Processing >1M records in memory may be slow.\n' +
-        'Consider using createCsvToJsonStream() for better performance with large files.\n' +
-        'Current size: ' + totalRows.toLocaleString() + ' rows\n' +
-        'Tip: Use { maxRows: N } option to set a custom limit if needed.'
-      );
-    }
+    if (totalRows > 1000000 && !maxRows && process.env.NODE_ENV !== 'test') {
+      console.warn(
+        'Warning: Processing >1M records in memory may be slow.\n' +
+        'Consider using createCsvToJsonStream() for better performance with large files.\n' +
+        'Current size: ' + totalRows.toLocaleString() + ' rows\n' +
+        'Tip: Use { maxRows: N } option to set a custom limit if needed.'
+      );
+    }
 
     return finalHooks.applyAfterConvert(result, {
       operation: 'csvToJson',
@@ -374,8 +397,8 @@ function csvToJson(csv, options = {}) {
   }, 'PARSE_FAILED', { function: 'csvToJson' });
 }
 
-/* istanbul ignore next */
-async function* csvToJsonIterator(csv, options = {}) {
+/* istanbul ignore next */
+async function* csvToJsonIterator(csv, options = {}) {
   validateCsvInput(csv, options);
 
   const opts = options && typeof options === 'object' ? options : {};
@@ -387,15 +410,15 @@ async function* csvToJsonIterator(csv, options = {}) {
     hasHeaders = true,
     renameMap = {},
     trim = true,
-    parseNumbers = false,
-    parseBooleans = false,
-    maxRows,
-    useCache = true,
-    cache: customCache,
-    useFastPath = true,
-    fastPathMode = 'objects',
-    hooks = {}
-  } = opts;
+    parseNumbers = false,
+    parseBooleans = false,
+    maxRows,
+    useCache = true,
+    cache: customCache,
+    useFastPath = true,
+    fastPathMode = 'objects',
+    hooks = {}
+  } = opts;
 
   const cacheToUse = useCache ? (customCache || globalDelimiterCache) : null;
 
@@ -446,20 +469,20 @@ async function* csvToJsonIterator(csv, options = {}) {
       headers = fields.map((_, index) => `column${index + 1}`);
     }
 
-    if (!fields || fields.length === 0) {
-      return null;
-    }
-
-    const fieldCount = Math.min(fields.length, headers.length);
+    if (!fields || fields.length === 0) {
+      return null;
+    }
+
+    const fieldCount = Math.min(fields.length, headers.length);
     let row;
 
-    const resolvedFastPathMode = fastPathMode === 'stream' ? 'objects' : fastPathMode;
-
-    if (resolvedFastPathMode === 'compact') {
-      row = new Array(fieldCount);
-      for (let j = 0; j < fieldCount; j++) {
-        row[j] = parseCsvValue(fields[j], { trim, parseNumbers, parseBooleans });
-      }
+    const resolvedFastPathMode = fastPathMode === 'stream' ? 'objects' : fastPathMode;
+
+    if (resolvedFastPathMode === 'compact') {
+      row = new Array(fieldCount);
+      for (let j = 0; j < fieldCount; j++) {
+        row[j] = parseCsvValue(fields[j], { trim, parseNumbers, parseBooleans });
+      }
     } else {
       row = {};
       for (let j = 0; j < fieldCount; j++) {
@@ -477,37 +500,37 @@ async function* csvToJsonIterator(csv, options = {}) {
     return processedRow;
   };
 
-  try {
-    const parseOptions = { delimiter: finalDelimiter };
-    if (useFastPath === false) {
-      parseOptions.forceEngine = 'STANDARD';
-    }
-
-    for (const fields of globalFastPathEngine.iterateRows(processedCsv, parseOptions)) {
-      totalRows++;
-      if (maxRows && totalRows > maxRows) {
-        throw new LimitError(
-          `CSV size exceeds maximum limit of ${maxRows} rows`,
-          maxRows,
-          totalRows
-        );
-      }
-      const processedRow = handleFields(fields, totalRows);
-      if (processedRow !== undefined && processedRow !== null) {
-        if (fields.length > headers.length && process.env.NODE_ENV === 'development') {
-          console.warn(`[jtcsv] Line ${totalRows}: ${fields.length - headers.length} extra fields ignored`);
-        }
-        yield processedRow;
-      }
-    }
-  } catch (error) {
-    if (error && error.code === 'FAST_PATH_UNCLOSED_QUOTES') {
-      const lineInfo = error.lineNumber ? ` at line ${error.lineNumber}` : '';
-      throw new ParsingError(`Unclosed quotes in CSV${lineInfo}`, error.lineNumber);
-    }
-    throw error;
-  }
-}
+  try {
+    const parseOptions = { delimiter: finalDelimiter };
+    if (useFastPath === false) {
+      parseOptions.forceEngine = 'STANDARD';
+    }
+
+    for (const fields of globalFastPathEngine.iterateRows(processedCsv, parseOptions)) {
+      totalRows++;
+      if (maxRows && totalRows > maxRows) {
+        throw new LimitError(
+          `CSV size exceeds maximum limit of ${maxRows} rows`,
+          maxRows,
+          totalRows
+        );
+      }
+      const processedRow = handleFields(fields, totalRows);
+      if (processedRow !== undefined && processedRow !== null) {
+        if (fields.length > headers.length && process.env.NODE_ENV === 'development') {
+          console.warn(`[jtcsv] Line ${totalRows}: ${fields.length - headers.length} extra fields ignored`);
+        }
+        yield processedRow;
+      }
+    }
+  } catch (error) {
+    if (error && error.code === 'FAST_PATH_UNCLOSED_QUOTES') {
+      const lineInfo = error.lineNumber ? ` at line ${error.lineNumber}` : '';
+      throw new ParsingError(`Unclosed quotes in CSV${lineInfo}`, error.lineNumber);
+    }
+    throw error;
+  }
+}
 
 /**
  * Validates file path for CSV reading
@@ -643,27 +666,27 @@ function createTransformHooks() {
  * @param {number} maxSize - Maximum cache size (default: 100)
  * @returns {DelimiterCache} New DelimiterCache instance
  */
-/* istanbul ignore next */
-function createDelimiterCache(maxSize = 100) {
-  return new DelimiterCache(maxSize);
-}
+/* istanbul ignore next */
+function createDelimiterCache(maxSize = 100) {
+  return new DelimiterCache(maxSize);
+}
 
 /**
  * Gets statistics from the global delimiter cache
  * @returns {Object} Cache statistics
  */
-/* istanbul ignore next */
-function getDelimiterCacheStats() {
-  return globalDelimiterCache.getStats();
-}
+/* istanbul ignore next */
+function getDelimiterCacheStats() {
+  return globalDelimiterCache.getStats();
+}
 
 /**
  * Clears the global delimiter cache
  */
-/* istanbul ignore next */
-function clearDelimiterCache() {
-  globalDelimiterCache.clear();
-}
+/* istanbul ignore next */
+function clearDelimiterCache() {
+  globalDelimiterCache.clear();
+}
 
 // Export the functions
 module.exports = {
@@ -682,7 +705,7 @@ module.exports = {
 };
 
 // For ES6 module compatibility
-/* istanbul ignore next */
-if (typeof module !== 'undefined' && module.exports) {
-  module.exports.default = csvToJson;
-}
+/* istanbul ignore next */
+if (typeof module !== 'undefined' && module.exports) {
+  module.exports.default = csvToJson;
+}