jtcsv 1.1.0 → 2.1.0

package/bin/jtcsv.js

@@ -41,8 +41,8 @@ ${color('USAGE:', 'bright')}
   jtcsv [command] [options] [file...]
 
 ${color('COMMANDS:', 'bright')}
-  ${color('json2csv', 'green')}     Convert JSON to CSV
-  ${color('csv2json', 'green')}     Convert CSV to JSON
+  ${color('json-to-csv', 'green')}     Convert JSON to CSV (alias: json2csv)
+  ${color('csv-to-json', 'green')}     Convert CSV to JSON (alias: csv2json)
   ${color('stream', 'yellow')}      Streaming conversion for large files
   ${color('batch', 'yellow')}       Batch process multiple files
   ${color('tui', 'magenta')}        Launch Terminal User Interface (requires blessed)
@@ -51,25 +51,31 @@ ${color('COMMANDS:', 'bright')}
 
 ${color('EXAMPLES:', 'bright')}
   ${color('Convert JSON file to CSV:', 'dim')}
-  jtcsv json2csv input.json output.csv --delimiter=,
+  jtcsv json-to-csv input.json output.csv --delimiter=,
 
   ${color('Convert CSV file to JSON:', 'dim')}
-  jtcsv csv2json input.csv output.json --parse-numbers
+  jtcsv csv-to-json input.csv output.json --parse-numbers --auto-detect
 
   ${color('Stream large JSON file to CSV:', 'dim')}
-  jtcsv stream json2csv large.json output.csv --max-records=1000000
+  jtcsv stream json-to-csv large.json output.csv --max-records=1000000
 
   ${color('Launch TUI interface:', 'dim')}
   jtcsv tui
 
 ${color('OPTIONS:', 'bright')}
   ${color('--delimiter=', 'cyan')}CHAR    CSV delimiter (default: ;)
+  ${color('--auto-detect', 'cyan')}        Auto-detect delimiter (default: true)
+  ${color('--candidates=', 'cyan')}LIST    Delimiter candidates (default: ;,\t|)
   ${color('--no-headers', 'cyan')}         Exclude headers from CSV output
   ${color('--parse-numbers', 'cyan')}      Parse numeric values in CSV
   ${color('--parse-booleans', 'cyan')}     Parse boolean values in CSV
+  ${color('--no-trim', 'cyan')}            Don't trim whitespace from CSV values
+  ${color('--rename=', 'cyan')}JSON       Rename columns (JSON map)
+  ${color('--template=', 'cyan')}JSON      Column order template (JSON object)
   ${color('--no-injection-protection', 'cyan')}  Disable CSV injection protection
-  ${color('--max-records=', 'cyan')}N      Maximum records to process (default: 1000000)
-  ${color('--max-rows=', 'cyan')}N         Maximum rows to process (default: 1000000)
+  ${color('--no-rfc4180', 'cyan')}         Disable RFC 4180 compliance
+  ${color('--max-records=', 'cyan')}N      Maximum records to process (optional, no limit by default)
+  ${color('--max-rows=', 'cyan')}N         Maximum rows to process (optional, no limit by default)
   ${color('--pretty', 'cyan')}             Pretty print JSON output
   ${color('--silent', 'cyan')}             Suppress all output except errors
   ${color('--verbose', 'cyan')}            Show detailed progress information
@@ -109,8 +115,19 @@ async function convertJsonToCsv(inputFile, outputFile, options) {
     
     console.log(color(`Converting ${jsonData.length} records...`, 'dim'));
     
+    // Prepare options for jtcsv
+    const jtcsvOptions = {
+      delimiter: options.delimiter,
+      includeHeaders: options.includeHeaders,
+      renameMap: options.renameMap,
+      template: options.template,
+      maxRecords: options.maxRecords,
+      preventCsvInjection: options.preventCsvInjection,
+      rfc4180Compliant: options.rfc4180Compliant
+    };
+    
     // Convert to CSV
-    const csvData = jtcsv.jsonToCsv(jsonData, options);
+    const csvData = jtcsv.jsonToCsv(jsonData, jtcsvOptions);
     
     // Write output file
     await fs.promises.writeFile(outputFile, csvData, 'utf8');
@@ -129,10 +146,23 @@ async function convertCsvToJson(inputFile, outputFile, options) {
   const startTime = Date.now();
   
   try {
-    console.log(color(`Reading CSV file...`, 'dim'));
+    console.log(color('Reading CSV file...', 'dim'));
+    
+    // Prepare options for jtcsv
+    const jtcsvOptions = {
+      delimiter: options.delimiter,
+      autoDetect: options.autoDetect,
+      candidates: options.candidates,
+      hasHeaders: options.hasHeaders,
+      renameMap: options.renameMap,
+      trim: options.trim,
+      parseNumbers: options.parseNumbers,
+      parseBooleans: options.parseBooleans,
+      maxRows: options.maxRows
+    };
     
     // Read and convert CSV
-    const jsonData = await jtcsv.readCsvAsJson(inputFile, options);
+    const jsonData = await jtcsv.readCsvAsJson(inputFile, jtcsvOptions);
     
     // Format JSON
     const jsonOutput = options.pretty 
@@ -156,7 +186,7 @@ async function streamJsonToCsv(inputFile, outputFile, options) {
   const startTime = Date.now();
   
   try {
-    console.log(color(`Streaming conversion started...`, 'dim'));
+    console.log(color('Streaming conversion started...', 'dim'));
     
     // Create streams
     const readStream = fs.createReadStream(inputFile, 'utf8');
@@ -235,12 +265,19 @@ async function launchTUI() {
 function parseOptions(args) {
   const options = {
     delimiter: ';',
+    autoDetect: true,
+    candidates: [';', ',', '\t', '|'],
+    hasHeaders: true,
     includeHeaders: true,
+    renameMap: undefined,
+    template: undefined,
+    trim: true,
     parseNumbers: false,
     parseBooleans: false,
     preventCsvInjection: true,
-    maxRecords: 1000000,
-    maxRows: 1000000,
+    rfc4180Compliant: true,
+    maxRecords: undefined,
+    maxRows: undefined,
     pretty: false,
     silent: false,
     verbose: false
@@ -255,36 +292,72 @@ function parseOptions(args) {
       const [key, value] = arg.slice(2).split('=');
       
       switch (key) {
-        case 'delimiter':
-          options.delimiter = value || ',';
-          break;
-        case 'no-headers':
-          options.includeHeaders = false;
-          break;
-        case 'parse-numbers':
-          options.parseNumbers = true;
-          break;
-        case 'parse-booleans':
-          options.parseBooleans = true;
-          break;
-        case 'no-injection-protection':
-          options.preventCsvInjection = false;
-          break;
-        case 'max-records':
-          options.maxRecords = parseInt(value, 10);
-          break;
-        case 'max-rows':
-          options.maxRows = parseInt(value, 10);
-          break;
-        case 'pretty':
-          options.pretty = true;
-          break;
-        case 'silent':
-          options.silent = true;
-          break;
-        case 'verbose':
-          options.verbose = true;
-          break;
+      case 'delimiter':
+        options.delimiter = value || ',';
+        options.autoDetect = false; // Disable auto-detect if delimiter is specified
+        break;
+      case 'auto-detect':
+        options.autoDetect = value !== 'false';
+        break;
+      case 'candidates':
+        options.candidates = value ? value.split(',') : [';', ',', '\t', '|'];
+        break;
+      case 'no-headers':
+        options.includeHeaders = false;
+        options.hasHeaders = false;
+        break;
+      case 'parse-numbers':
+        options.parseNumbers = true;
+        break;
+      case 'parse-booleans':
+        options.parseBooleans = true;
+        break;
+      case 'no-trim':
+        options.trim = false;
+        break;
+      case 'rename':
+        try {
+          // Handle both quoted and unquoted JSON
+          const jsonStr = value || '{}';
+          // Remove surrounding single quotes if present
+          const cleanStr = jsonStr.replace(/^'|'$/g, '').replace(/^"|"$/g, '');
+          options.renameMap = JSON.parse(cleanStr);
+        } catch (e) {
+          throw new Error(`Invalid JSON in --rename option: ${e.message}. Value: ${value}`);
+        }
+        break;
+      case 'template':
+        try {
+          // Handle both quoted and unquoted JSON
+          const jsonStr = value || '{}';
+          // Remove surrounding single quotes if present
+          const cleanStr = jsonStr.replace(/^'|'$/g, '').replace(/^"|"$/g, '');
+          options.template = JSON.parse(cleanStr);
+        } catch (e) {
+          throw new Error(`Invalid JSON in --template option: ${e.message}. Value: ${value}`);
+        }
+        break;
+      case 'no-injection-protection':
+        options.preventCsvInjection = false;
+        break;
+      case 'no-rfc4180':
+        options.rfc4180Compliant = false;
+        break;
+      case 'max-records':
+        options.maxRecords = parseInt(value, 10);
+        break;
+      case 'max-rows':
+        options.maxRows = parseInt(value, 10);
+        break;
+      case 'pretty':
+        options.pretty = true;
+        break;
+      case 'silent':
+        options.silent = true;
+        break;
+      case 'verbose':
+        options.verbose = true;
+        break;
       }
     } else if (!arg.startsWith('-')) {
       files.push(arg);
@@ -312,57 +385,59 @@ async function main() {
   }
   
   switch (command) {
-    case 'json2csv':
-      if (files.length < 2) {
-        console.error(color('Error: Input and output files required', 'red'));
-        console.log(color('Usage: jtcsv json2csv input.json output.csv', 'cyan'));
-        process.exit(1);
-      }
-      await convertJsonToCsv(files[0], files[1], options);
-      break;
+  case 'json-to-csv':
+  case 'json2csv': // Backward compatibility
+    if (files.length < 2) {
+      console.error(color('Error: Input and output files required', 'red'));
+      console.log(color('Usage: jtcsv json-to-csv input.json output.csv', 'cyan'));
+      process.exit(1);
+    }
+    await convertJsonToCsv(files[0], files[1], options);
+    break;
       
-    case 'csv2json':
-      if (files.length < 2) {
-        console.error(color('Error: Input and output files required', 'red'));
-        console.log(color('Usage: jtcsv csv2json input.csv output.json', 'cyan'));
-        process.exit(1);
-      }
-      await convertCsvToJson(files[0], files[1], options);
-      break;
+  case 'csv-to-json':
+  case 'csv2json': // Backward compatibility
+    if (files.length < 2) {
+      console.error(color('Error: Input and output files required', 'red'));
+      console.log(color('Usage: jtcsv csv-to-json input.csv output.json', 'cyan'));
+      process.exit(1);
+    }
+    await convertCsvToJson(files[0], files[1], options);
+    break;
       
-    case 'stream':
-      if (args.length < 2) {
-        console.error(color('Error: Streaming mode requires subcommand', 'red'));
-        console.log(color('Usage: jtcsv stream [json2csv|csv2json] input output', 'cyan'));
-        process.exit(1);
-      }
-      const streamCommand = args[1].toLowerCase();
-      if (streamCommand === 'json2csv' && files.length >= 2) {
-        await streamJsonToCsv(files[0], files[1], options);
-      } else {
-        console.error(color('Error: Invalid streaming command', 'red'));
-        process.exit(1);
-      }
-      break;
+  case 'stream':
+    if (args.length < 2) {
+      console.error(color('Error: Streaming mode requires subcommand', 'red'));
+      console.log(color('Usage: jtcsv stream [json2csv|csv2json] input output', 'cyan'));
+      process.exit(1);
+    }
+    const streamCommand = args[1].toLowerCase();
+    if (streamCommand === 'json2csv' && files.length >= 2) {
+      await streamJsonToCsv(files[0], files[1], options);
+    } else {
+      console.error(color('Error: Invalid streaming command', 'red'));
+      process.exit(1);
+    }
+    break;
       
-    case 'tui':
-      await launchTUI();
-      break;
+  case 'tui':
+    await launchTUI();
+    break;
       
-    case 'help':
-      showHelp();
-      break;
+  case 'help':
+    showHelp();
+    break;
       
-    case 'version':
-    case '-v':
-    case '--version':
-      showVersion();
-      break;
+  case 'version':
+  case '-v':
+  case '--version':
+    showVersion();
+    break;
       
-    default:
-      console.error(color(`Error: Unknown command '${command}'`, 'red'));
-      console.log(color('Use jtcsv help for available commands', 'cyan'));
-      process.exit(1);
+  default:
+    console.error(color(`Error: Unknown command '${command}'`, 'red'));
+    console.log(color('Use jtcsv help for available commands', 'cyan'));
+    process.exit(1);
   }
 }
 
@@ -391,4 +466,11 @@ if (require.main === module) {
   });
 }
 
-module.exports = { main };
+module.exports = { main };
+
+
+
+
+
+
+

package/csv-to-json.js

@@ -41,8 +41,18 @@ function validateCsvInput(csv, options) {
     throw new ConfigurationError('Delimiter must be a single character');
   }
   
+  // Validate autoDetect
+  if (options?.autoDetect !== undefined && typeof options.autoDetect !== 'boolean') {
+    throw new ConfigurationError('autoDetect must be a boolean');
+  }
+  
+  // Validate candidates
+  if (options?.candidates && !Array.isArray(options.candidates)) {
+    throw new ConfigurationError('candidates must be an array');
+  }
+  
   // Validate maxRows
-  if (options?.maxRows && (typeof options.maxRows !== 'number' || options.maxRows <= 0)) {
+  if (options?.maxRows !== undefined && (typeof options.maxRows !== 'number' || options.maxRows <= 0)) {
     throw new ConfigurationError('maxRows must be a positive number');
   }
   
@@ -50,7 +60,7 @@ function validateCsvInput(csv, options) {
 }
 
 /**
- * Parses a single CSV line with proper escaping
+но * Parses a single CSV line with proper escaping
  * @private
  */
 function parseCsvLine(line, lineNumber, delimiter) {
@@ -69,7 +79,17 @@ function parseCsvLine(line, lineNumber, delimiter) {
     }
 
     if (char === '\\') {
-      escapeNext = true;
+      if (i + 1 === line.length) {
+        // Backslash at end of line - treat as literal
+        currentField += char;
+      } else if (line[i + 1] === '\\') {
+        // Double backslash - add one backslash to field and skip next
+        currentField += char;
+        i++; // Skip next backslash
+      } else {
+        // Escape next character
+        escapeNext = true;
+      }
       continue;
     }
 
@@ -87,6 +107,22 @@ function parseCsvLine(line, lineNumber, delimiter) {
             // Escaped quote inside quotes ("" -> ")
             currentField += '"';
             i++; // Skip next quote
+            // Check if this is the end of the quoted field
+            // Look ahead to see if next char is delimiter or end of line
+            let isEndOfField = false;
+            let j = i + 1;
+            // Skip whitespace
+            while (j < line.length && (line[j] === ' ' || line[j] === '\t')) {
+              j++;
+            }
+            if (j === line.length || line[j] === delimiter) {
+              isEndOfField = true;
+            }
+            
+            if (isEndOfField) {
+              // This is the closing quote
+              insideQuotes = false;
+            }
           }
         } else {
           // Check if this is really the end of the quoted field
@@ -126,6 +162,13 @@ function parseCsvLine(line, lineNumber, delimiter) {
     currentField += char;
   }
 
+  // Handle case where escapeNext is still true at end of line
+  if (escapeNext) {
+    // This happens when line ends with backslash
+    // Add the backslash as literal character
+    currentField += '\\';
+  }
+
   // Add last field
   fields.push(currentField);
 
@@ -171,11 +214,11 @@ function parseCsvValue(value, options) {
   // Parse booleans
   if (parseBooleans) {
     const lowerValue = result.toLowerCase();
-    if (lowerValue === 'true') {
-      return true; 
+    if (lowerValue === 'true') {
+      return true; 
     }
-    if (lowerValue === 'false') {
-      return false; 
+    if (lowerValue === 'false') {
+      return false; 
     }
   }
   
@@ -187,18 +230,66 @@ function parseCsvValue(value, options) {
   return result;
 }
 
+/**
+ * Auto-detect CSV delimiter from content
+ * @private
+ */
+function autoDetectDelimiter(csv, candidates = [';', ',', '\t', '|']) {
+  if (!csv || typeof csv !== 'string') {
+    return ';'; // default
+  }
+
+  const lines = csv.split('\n').filter(line => line.trim().length > 0);
+  
+  if (lines.length === 0) {
+    return ';'; // default
+  }
+
+  // Use first non-empty line for detection
+  const firstLine = lines[0];
+  
+  const counts = {};
+  candidates.forEach(delim => {
+    // Escape special regex characters
+    const escapedDelim = delim.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+    const regex = new RegExp(escapedDelim, 'g');
+    const matches = firstLine.match(regex);
+    counts[delim] = matches ? matches.length : 0;
+  });
+
+  // Find delimiter with maximum count
+  let maxCount = -1;
+  let detectedDelimiter = ';'; // default
+  
+  for (const [delim, count] of Object.entries(counts)) {
+    if (count > maxCount) {
+      maxCount = count;
+      detectedDelimiter = delim;
+    }
+  }
+
+  // If no delimiter found or tie, return default
+  if (maxCount === 0) {
+    return ';'; // default
+  }
+
+  return detectedDelimiter;
+}
+
 /**
  * Converts CSV string to JSON array
  * 
- * @param {string} csv - CSV string to convert
+  * @param {string} csv - CSV string to convert
  * @param {Object} [options] - Configuration options
- * @param {string} [options.delimiter=';'] - CSV delimiter character
+ * @param {string} [options.delimiter] - CSV delimiter character (default: auto-detected)
+ * @param {boolean} [options.autoDetect=true] - Auto-detect delimiter if not specified
+ * @param {Array} [options.candidates=[';', ',', '\t', '|']] - Candidate delimiters for auto-detection
  * @param {boolean} [options.hasHeaders=true] - Whether CSV has headers row
  * @param {Object} [options.renameMap={}] - Map for renaming column headers (newKey: oldKey)
  * @param {boolean} [options.trim=true] - Trim whitespace from values
  * @param {boolean} [options.parseNumbers=false] - Parse numeric values
  * @param {boolean} [options.parseBooleans=false] - Parse boolean values
- * @param {number} [options.maxRows=1000000] - Maximum number of rows to process
+ * @param {number} [options.maxRows] - Maximum number of rows to process (optional, no limit by default)
  * @returns {Array<Object>} JSON array
  * 
  * @example
@@ -218,15 +309,24 @@ function csvToJson(csv, options = {}) {
     const opts = options && typeof options === 'object' ? options : {};
     
     const {
-      delimiter = ';',
+      delimiter,
+      autoDetect = true,
+      candidates = [';', ',', '\t', '|'],
       hasHeaders = true,
       renameMap = {},
       trim = true,
       parseNumbers = false,
       parseBooleans = false,
-      maxRows = 1000000
+      maxRows
     } = opts;
 
+    // Determine delimiter
+    let finalDelimiter = delimiter;
+    if (!finalDelimiter && autoDetect) {
+      finalDelimiter = autoDetectDelimiter(csv, candidates);
+    }
+    finalDelimiter = finalDelimiter || ';'; // fallback
+
     // Handle empty CSV
     if (csv.trim() === '') {
       return [];
@@ -274,16 +374,28 @@ function csvToJson(csv, options = {}) {
     }
     
     // Check for unclosed quotes
-    if (insideQuotes) {
-      throw new ParsingError('Unclosed quotes in CSV', lines.length);
-    }
+    // Note: This check is moved to parseCsvLine which has better context
+    // for handling escaped quotes like ""
+    // if (insideQuotes) {
+    //   throw new ParsingError('Unclosed quotes in CSV', lines.length);
+    // }
     
     if (lines.length === 0) {
       return [];
     }
 
-    // Limit rows to prevent OOM
-    if (lines.length > maxRows) {
+    // Show warning for large datasets (optional limit)
+    if (lines.length > 1000000 && !maxRows && process.env.NODE_ENV !== 'test') {
+      console.warn(
+        '⚠️ Warning: Processing >1M records in memory may be slow.\n' +
+        '💡 Consider using createCsvToJsonStream() for better performance with large files.\n' +
+        '📊 Current size: ' + lines.length.toLocaleString() + ' rows\n' +
+        '🔧 Tip: Use { maxRows: N } option to set a custom limit if needed.'
+      );
+    }
+
+    // Apply optional row limit if specified
+    if (maxRows && lines.length > maxRows) {
       throw new LimitError(
         `CSV size exceeds maximum limit of ${maxRows} rows`,
         maxRows,
@@ -297,7 +409,7 @@ function csvToJson(csv, options = {}) {
     // Parse headers if present
     if (hasHeaders && lines.length > 0) {
       try {
-        headers = parseCsvLine(lines[0], 1, delimiter).map(header => {
+        headers = parseCsvLine(lines[0], 1, finalDelimiter).map(header => {
           const trimmed = trim ? header.trim() : header;
           // Apply rename map
           return renameMap[trimmed] || trimmed;
@@ -312,7 +424,7 @@ function csvToJson(csv, options = {}) {
     } else {
       // Generate numeric headers from first line
       try {
-        const firstLineFields = parseCsvLine(lines[0], 1, delimiter);
+        const firstLineFields = parseCsvLine(lines[0], 1, finalDelimiter);
         headers = firstLineFields.map((_, index) => `column${index + 1}`);
       } catch (error) {
         if (error instanceof ParsingError) {
@@ -334,7 +446,7 @@ function csvToJson(csv, options = {}) {
       }
       
       try {
-        const fields = parseCsvLine(line, i + 1, delimiter);
+        const fields = parseCsvLine(line, i + 1, finalDelimiter);
         
         // Handle mismatched field count
         const row = {};
@@ -399,7 +511,7 @@ function validateCsvFilePath(filePath) {
  * @returns {Promise<Array<Object>>} Promise that resolves to JSON array
  * 
  * @example
- * const { readCsvAsJson } = require('./csv-to-json');
+ *     const { readCsvAsJson } = require('./csv-to-json');
  * 
  * const json = await readCsvAsJson('./data.csv', {
  *   delimiter: ',',
@@ -483,7 +595,8 @@ function readCsvAsJsonSync(filePath, options = {}) {
 module.exports = {
   csvToJson,
   readCsvAsJson,
-  readCsvAsJsonSync
+  readCsvAsJsonSync,
+  autoDetectDelimiter
 };
 
 // For ES6 module compatibility