jtcsv 3.0.0 → 3.1.0

package/src/utils/bom-utils.js

@@ -1,259 +0,0 @@
-/**
- * BOM (Byte Order Mark) Utilities for jtcsv
- * 
- * Provides functions to detect and strip BOM characters from UTF-8, UTF-16 LE/BE,
- * and UTF-32 encoded strings/buffers.
- * 
- * @module bom-utils
- */
-
-/**
- * BOM signatures for different encodings
- */
-const BOM_SIGNATURES = {
-  'utf-8': Buffer.from([0xEF, 0xBB, 0xBF]),
-  'utf-16le': Buffer.from([0xFF, 0xFE]),
-  'utf-16be': Buffer.from([0xFE, 0xFF]),
-  'utf-32le': Buffer.from([0xFF, 0xFE, 0x00, 0x00]),
-  'utf-32be': Buffer.from([0x00, 0x00, 0xFE, 0xFF])
-};
-
-/**
- * Detects if a buffer or string starts with a BOM
- * 
- * @param {Buffer|string} input - Input to check for BOM
- * @returns {Object|null} Detection result or null if no BOM found
- * @property {string} encoding - Detected encoding ('utf-8', 'utf-16le', etc.)
- * @property {number} bomLength - Length of BOM in bytes
- */
-function detectBom(input) {
-  if (!input) {
-    return null;
-  }
-  
-  let buffer;
-  if (typeof input === 'string') {
-    buffer = Buffer.from(input, 'utf8');
-  } else if (Buffer.isBuffer(input)) {
-    buffer = input;
-  } else {
-    return null;
-  }
-  
-  // Check each BOM signature
-  for (const [encoding, signature] of Object.entries(BOM_SIGNATURES)) {
-    if (buffer.length >= signature.length) {
-      if (buffer.slice(0, signature.length).equals(signature)) {
-        return {
-          encoding,
-          bomLength: signature.length,
-          hasBom: true
-        };
-      }
-    }
-  }
-  
-  return null;
-}
-
-/**
- * Strips BOM from a buffer or string
- * 
- * @param {Buffer|string} input - Input to strip BOM from
- * @returns {Buffer|string} Input without BOM
- */
-function stripBom(input) {
-  if (!input) {
-    return input;
-  }
-  
-  const bomInfo = detectBom(input);
-  if (!bomInfo) {
-    return input;
-  }
-  
-  if (Buffer.isBuffer(input)) {
-    return input.slice(bomInfo.bomLength);
-  }
-  
-  if (typeof input === 'string') {
-    // Convert to buffer, strip BOM, then convert back to string
-    const buffer = Buffer.from(input, 'utf8');
-    const strippedBuffer = buffer.slice(bomInfo.bomLength);
-    
-    // Determine correct encoding for conversion
-    let encoding = 'utf8';
-    if (bomInfo.encoding === 'utf-16le') {
-      encoding = 'utf16le';
-    } else if (bomInfo.encoding === 'utf-16be') {
-      encoding = 'utf16be';
-    }
-    
-    return strippedBuffer.toString(encoding);
-  }
-  
-  return input;
-}
-
-/**
- * Strips BOM from a string (optimized for strings)
- * 
- * @param {string} str - String to strip BOM from
- * @returns {string} String without BOM
- */
-function stripBomFromString(str) {
-  if (typeof str !== 'string') {
-    return str;
-  }
-  
-  // Check for UTF-8 BOM (most common)
-  if (str.charCodeAt(0) === 0xFEFF) {
-    return str.slice(1);
-  }
-  
-  // Check for UTF-8 BOM bytes as characters
-  if (str.length >= 3 && 
-      str.charCodeAt(0) === 0xEF && 
-      str.charCodeAt(1) === 0xBB && 
-      str.charCodeAt(2) === 0xBF) {
-    return str.slice(3);
-  }
-  
-  return str;
-}
-
-/**
- * Creates a transform stream that strips BOM from incoming data
- * 
- * @returns {Transform} Transform stream
- */
-function createBomStripStream() {
-  const { Transform } = require('stream');
-  let bomStripped = false;
-  
-  return new Transform({
-    transform(chunk, encoding, callback) {
-      if (!bomStripped) {
-        const bomInfo = detectBom(chunk);
-        if (bomInfo) {
-          // Strip BOM from first chunk
-          chunk = chunk.slice(bomInfo.bomLength);
-          bomStripped = true;
-        } else {
-          bomStripped = true; // No BOM found, but we've checked
-        }
-      }
-      
-      this.push(chunk);
-      callback();
-    }
-  });
-}
-
-/**
- * Reads a file and automatically handles BOM
- * 
- * @param {string} filePath - Path to file
- * @param {Object} options - Read options
- * @returns {Promise<{data: Buffer|string, encoding: string, hadBom: boolean}>}
- */
-async function readFileWithBomHandling(filePath, options = {}) {
-  const fs = require('fs').promises;
-  const buffer = await fs.readFile(filePath);
-  
-  const bomInfo = detectBom(buffer);
-  const hadBom = !!bomInfo;
-  
-  let data;
-  let encoding = options.encoding || 'utf8';
-  
-  if (bomInfo) {
-    // Strip BOM
-    data = buffer.slice(bomInfo.bomLength);
-    
-    // Use detected encoding if not specified
-    if (!options.encoding) {
-      encoding = bomInfo.encoding;
-    }
-  } else {
-    data = buffer;
-  }
-  
-  // Convert to string if encoding is specified
-  if (options.encoding || bomInfo) {
-    data = data.toString(encoding);
-  }
-  
-  return {
-    data,
-    encoding,
-    hadBom,
-    bomInfo: bomInfo || null
-  };
-}
-
-/**
- * Checks if a file has BOM (synchronous)
- * 
- * @param {string} filePath - Path to file
- * @returns {Object|null} BOM info or null
- */
-function fileHasBomSync(filePath) {
-  const fs = require('fs');
-  const fd = fs.openSync(filePath, 'r');
-  const buffer = Buffer.alloc(4);
-  const bytesRead = fs.readSync(fd, buffer, 0, 4, 0);
-  fs.closeSync(fd);
-  
-  if (bytesRead < 2) {
-    return null;
-  }
-  
-  return detectBom(buffer.slice(0, bytesRead));
-}
-
-/**
- * Normalizes CSV input by stripping BOM and ensuring proper encoding
- * 
- * @param {string|Buffer} csvInput - CSV input
- * @param {Object} options - Processing options
- * @returns {string} Normalized CSV string
- */
-function normalizeCsvInput(csvInput, options = {}) {
-  if (!csvInput) {
-    return '';
-  }
-  
-  let normalized;
-  
-  if (Buffer.isBuffer(csvInput)) {
-    const bomInfo = detectBom(csvInput);
-    if (bomInfo) {
-      normalized = csvInput.slice(bomInfo.bomLength).toString(bomInfo.encoding);
-    } else {
-      normalized = csvInput.toString(options.encoding || 'utf8');
-    }
-  } else if (typeof csvInput === 'string') {
-    normalized = stripBomFromString(csvInput);
-  } else {
-    throw new Error('CSV input must be a string or Buffer');
-  }
-  
-  // Ensure proper line endings
-  if (options.normalizeLineEndings !== false) {
-    normalized = normalized.replace(/\r\n|\r/g, '\n');
-  }
-  
-  return normalized;
-}
-
-module.exports = {
-  detectBom,
-  stripBom,
-  stripBomFromString,
-  createBomStripStream,
-  readFileWithBomHandling,
-  fileHasBomSync,
-  normalizeCsvInput,
-  BOM_SIGNATURES
-};

package/src/utils/encoding-support.js

@@ -1,124 +0,0 @@
-/**
- * Encoding detection and conversion utilities.
- * 
- * Provides basic auto‑detection of UTF‑8, UTF‑16LE, UTF‑16BE with BOM,
- * and fallback to a default encoding.
- * 
- * @example
- * const { detectEncoding, convertToUtf8 } = require('./encoding-support');
- * 
- * const buffer = fs.readFileSync('data.csv');
- * const encoding = detectEncoding(buffer, { fallback: 'utf8' });
- * const utf8Text = convertToUtf8(buffer, encoding);
- */
-
-const { ValidationError } = require('../errors');
-
-/**
- * Detects encoding from buffer based on BOM.
- * 
- * @param {Buffer} buffer - Input buffer
- * @param {Object} options - Detection options
- * @param {string} options.fallback - Fallback encoding if detection fails (default: 'utf8')
- * @returns {string} Detected encoding: 'utf8', 'utf16le', 'utf16be', or fallback
- */
-function detectEncoding(buffer, options = {}) {
-  const { fallback = 'utf8' } = options;
-  
-  if (!Buffer.isBuffer(buffer)) {
-    throw new ValidationError('Input must be a Buffer');
-  }
-  
-  // Check BOM
-  if (buffer.length >= 3 && buffer[0] === 0xEF && buffer[1] === 0xBB && buffer[2] === 0xBF) {
-    return 'utf8';
-  }
-  if (buffer.length >= 2 && buffer[0] === 0xFE && buffer[1] === 0xFF) {
-    return 'utf16be';
-  }
-  if (buffer.length >= 2 && buffer[0] === 0xFF && buffer[1] === 0xFE) {
-    return 'utf16le';
-  }
-  
-  // No BOM detected, use fallback
-  return fallback;
-}
-
-/**
- * Converts buffer to UTF‑8 string using detected encoding.
- * Strips BOM if present.
- * 
- * @param {Buffer} buffer - Input buffer
- * @param {string} encoding - Source encoding ('utf8', 'utf16le', 'utf16be')
- * @returns {string} UTF‑8 string without BOM
- */
-function convertToUtf8(buffer, encoding = 'utf8') {
-  if (!Buffer.isBuffer(buffer)) {
-    throw new ValidationError('Input must be a Buffer');
-  }
-  
-  let offset = 0;
-  
-  // Skip BOM
-  if (encoding === 'utf8' && buffer.length >= 3 && buffer[0] === 0xEF && buffer[1] === 0xBB && buffer[2] === 0xBF) {
-    offset = 3;
-  } else if (encoding === 'utf16be' && buffer.length >= 2 && buffer[0] === 0xFE && buffer[1] === 0xFF) {
-    offset = 2;
-  } else if (encoding === 'utf16le' && buffer.length >= 2 && buffer[0] === 0xFF && buffer[1] === 0xFE) {
-    offset = 2;
-  }
-  
-  const slice = buffer.slice(offset);
-  return slice.toString(encoding);
-}
-
-/**
- * Auto‑detects encoding and converts buffer to UTF‑8 string.
- * 
- * @param {Buffer} buffer - Input buffer
- * @param {Object} options - Options
- * @param {string} options.fallback - Fallback encoding (default: 'utf8')
- * @returns {{ encoding: string, text: string }} Detected encoding and converted text
- */
-function autoDetectAndConvert(buffer, options = {}) {
-  const encoding = detectEncoding(buffer, options);
-  const text = convertToUtf8(buffer, encoding);
-  return { encoding, text };
-}
-
-/**
- * Creates a wrapper around csvToJson that accepts Buffer or string with encoding detection.
- * 
- * @param {Buffer|string} input - CSV as Buffer or string
- * @param {Object} parseOptions - Options for csvToJson
- * @param {string} parseOptions.encoding - Explicit encoding (default: 'auto')
- * @param {string} parseOptions.fallbackEncoding - Fallback if auto detection fails (default: 'utf8')
- * @returns {Promise<Array>} Parsed JSON data
- */
-async function csvToJsonWithEncoding(input, parseOptions = {}) {
-  const { csvToJson } = require('../index');
-  const { encoding = 'auto', fallbackEncoding = 'utf8', ...restOptions } = parseOptions;
-  
-  let text;
-  if (Buffer.isBuffer(input)) {
-    if (encoding === 'auto') {
-      const detected = autoDetectAndConvert(input, { fallback: fallbackEncoding });
-      text = detected.text;
-    } else {
-      text = convertToUtf8(input, encoding);
-    }
-  } else if (typeof input === 'string') {
-    text = input;
-  } else {
-    throw new ValidationError('Input must be a Buffer or string');
-  }
-  
-  return csvToJson(text, restOptions);
-}
-
-module.exports = {
-  detectEncoding,
-  convertToUtf8,
-  autoDetectAndConvert,
-  csvToJsonWithEncoding
-};