jtcsv 3.0.0 → 3.1.0

package/bin/jtcsv.ts

@@ -11,7 +11,7 @@ import * as fs from 'fs';
 import * as path from 'path';
 import * as readline from 'readline';
 const { pipeline } = require('stream/promises');
-import * as jtcsv from '../index.js';
+import * as jtcsv from '../index';
 const transformLoader = require('../src/utils/transform-loader');
 const schemaValidator = require('../src/utils/schema-validator');
 
@@ -88,7 +88,7 @@ function color(text, colorName: any): any {
 function showHelp(): void {
   console.log(`
 ${color('jtcsv CLI v' + VERSION, 'cyan')}
-${color('The Complete JSON↔CSV Converter for Node.js', 'dim')}
+${color('The Complete JSON<->CSV Converter for Node.js', 'dim')}
 
 ${color('USAGE:', 'bright')}
   jtcsv [command] [options] [file...]
@@ -106,7 +106,7 @@ ${color('MAIN COMMANDS:', 'bright')}
   ${color('batch', 'yellow')}          Batch process multiple files
   ${color('preprocess', 'magenta')}    Preprocess JSON with deep unwrapping
   ${color('unwrap', 'magenta')}        Flatten nested JSON structures (alias: flatten)
-    ${color('tui', 'magenta')}           Launch Terminal User Interface (@jtcsv/tui)
+  ${color('tui', 'magenta')}           Launch Terminal User Interface (@jtcsv/tui)
   ${color('web', 'magenta')}           Launch Web Interface (http://localhost:3000)
   ${color('help', 'blue')}             Show this help message
   ${color('version', 'blue')}          Show version information
@@ -147,7 +147,7 @@ ${color('EXAMPLES:', 'bright')}
   ${color('Batch convert JSON files:', 'dim')}
   jtcsv batch json-to-csv "data/*.json" "output/" --delimiter=;
 
-    ${color('Launch TUI interface:', 'dim')}
+  ${color('Launch TUI interface:', 'dim')}
   jtcsv tui
 
   ${color('Launch Web interface:', 'dim')}
@@ -160,10 +160,12 @@ ${color('CONVERSION OPTIONS:', 'bright')}
   ${color('--no-headers', 'cyan')}         Exclude headers from CSV output
   ${color('--parse-numbers', 'cyan')}      Parse numeric values in CSV
   ${color('--parse-booleans', 'cyan')}     Parse boolean values in CSV
-  ${color('--no-trim', 'cyan')}            Don't trim whitespace from CSV values
-  ${color('--no-fast-path', 'cyan')}       Disable fast-path parser (force quote-aware)
-  ${color('--fast-path-mode=', 'cyan')}MODE  Fast path output mode (objects|compact)
-  ${color('--rename=', 'cyan')}JSON       Rename columns (JSON map)
+  ${color('--no-trim', 'cyan')}            Don't trim whitespace from CSV values
+  ${color('--no-fast-path', 'cyan')}       Disable fast-path parser (force quote-aware)
+  ${color('--fast-path-mode=', 'cyan')}MODE  Fast path output mode (objects|compact)
+  ${color('--repair-row-shifts', 'cyan')}  Repair shifted rows with trailing empty fields
+  ${color('--normalize-quotes', 'cyan')}  Normalize excessive quotes in parsed fields
+  ${color('--rename=', 'cyan')}JSON       Rename columns (JSON map)
   ${color('--template=', 'cyan')}JSON      Column order template (JSON object)
   ${color('--no-injection-protection', 'cyan')}  Disable CSV injection protection
   ${color('--no-rfc4180', 'cyan')}         Disable RFC 4180 compliance
@@ -196,17 +198,17 @@ ${color('CONVERSION OPTIONS:', 'bright')}
   ${color('--debug', 'cyan')}              Show debug information
   ${color('--dry-run', 'cyan')}            Show what would be done without actually doing it
   ${color('SECURITY FEATURES:', 'bright')}
-  • CSV injection protection (enabled by default)
-  • Path traversal protection
-  • Input validation and sanitization
-  • Size limits to prevent DoS attacks
-  • Schema validation support
+  - CSV injection protection (enabled by default)
+  - Path traversal protection
+  - Input validation and sanitization
+  - Size limits to prevent DoS attacks
+  - Schema validation support
 
   ${color('PERFORMANCE FEATURES:', 'bright')}
-  • Streaming for files >100MB
-  • Batch processing with parallel execution
-  • Memory-efficient preprocessing
-  • Configurable buffer sizes
+  - Streaming for files >100MB
+  - Batch processing with parallel execution
+  - Memory-efficient preprocessing
+  - Configurable buffer sizes
 
   ${color('LEARN MORE:', 'dim')}
   GitHub: https://github.com/Linol-Hamelton/jtcsv
@@ -215,57 +217,74 @@ ${color('CONVERSION OPTIONS:', 'bright')}
   `);
 }
 
-function showVersion(): void {
-  console.log(`jtcsv v${VERSION}`);
-  console.log(`Node.js ${process.version}`);
-  console.log(`Platform: ${process.platform} ${process.arch}`);
-}
-
-// ============================================================================
-// CONVERSION FUNCTIONS
-// ============================================================================
+function showVersion(): void {
+  console.log(`jtcsv v${VERSION}`);
+  console.log(`Node.js ${process.version}`);
+  console.log(`Platform: ${process.platform} ${process.arch}`);
+}
+
+async function readStdin(): Promise<string> {
+  return new Promise((resolve, reject) => {
+    let data = '';
+    process.stdin.setEncoding('utf8');
+    process.stdin.on('data', (chunk) => {
+      data += chunk;
+    });
+    process.stdin.on('end', () => resolve(data));
+    process.stdin.on('error', reject);
+  });
+}
+
+// ============================================================================
+// CONVERSION FUNCTIONS
+// ============================================================================
 
 async function convertJsonToCsv(inputFile, outputFile, options: any): Promise<ConversionResult> {
-  const startTime = Date.now();
-
-  try {
-    // Read input file
-    const inputData = await fs.promises.readFile(inputFile, 'utf8');
-    const jsonData = JSON.parse(inputData);
-
-    if (!Array.isArray(jsonData)) {
-      throw new Error('JSON data must be an array of objects');
-    }
-
-    if (!options.silent) {
-      console.log(
-        color(
-          `Converting ${jsonData.length.toLocaleString()} records...`,
-          'dim'
-        )
-      );
-    }
-
-    if (options.transform) {
-      if (!options.silent) {
-        console.log(
-          color(`Applying transform from: ${options.transform}`, 'dim')
-        );
-      }
+  const startTime = Date.now();
+  const useStdin = inputFile === '-';
+  const useStdout = outputFile === '-';
+  const shouldLog = !options.silent && !useStdout;
+
+  try {
+    // Read input
+    const inputData = useStdin
+      ? await readStdin()
+      : await fs.promises.readFile(inputFile, 'utf8');
+    const jsonData = JSON.parse(inputData);
+
+    if (!Array.isArray(jsonData)) {
+      throw new Error('JSON data must be an array of objects');
+    }
+
+    if (shouldLog) {
+      console.log(
+        color(
+          `Converting ${jsonData.length.toLocaleString()} records...`,
+          'dim'
+        )
+      );
+    }
+
+    if (options.transform) {
+      if (shouldLog) {
+        console.log(
+          color(`Applying transform from: ${options.transform}`, 'dim')
+        );
+      }
       let transformedData;
 try {
         transformedData = transformLoader.applyTransform(
           jsonData,
           options.transform
         );
-        if (!options.silent) {
-          console.log(
-            color(
-              `✓ Transform applied to ${transformedData.length} records`,
-              'green'
-            )
-          );
-        }
+        if (shouldLog) {
+          console.log(
+            color(
+              `✓ Transform applied to ${transformedData.length} records`,
+              'green'
+            )
+          );
+        }
       } catch (transformError: any) {
         console.error(
           color(`✗ Transform error: ${transformError.message}`, 'red')
@@ -278,19 +297,20 @@ try {
     }
 
     // Prepare options for jtcsv
-    const jtcsvOptions = {
-      delimiter: options.delimiter,
-      includeHeaders: options.includeHeaders,
-      renameMap: options.renameMap,
-      template: options.template,
-      maxRecords: options.maxRecords,
-      preventCsvInjection: options.preventCsvInjection,
-      rfc4180Compliant: options.rfc4180Compliant,
-      schema: options.schema, // Add schema option
-      flatten: options.flatten,
-      flattenSeparator: options.flattenSeparator,
-      flattenMaxDepth: options.flattenMaxDepth,
-      arrayHandling: options.arrayHandling
+    const jtcsvOptions = {
+      delimiter: options.delimiter,
+      includeHeaders: options.includeHeaders,
+      renameMap: options.renameMap,
+      template: options.template,
+      maxRecords: options.maxRecords,
+      preventCsvInjection: options.preventCsvInjection,
+      rfc4180Compliant: options.rfc4180Compliant,
+      normalizeQuotes: options.normalizeQuotes,
+      schema: options.schema, // Add schema option
+      flatten: options.flatten,
+      flattenSeparator: options.flattenSeparator,
+      flattenMaxDepth: options.flattenMaxDepth,
+      arrayHandling: options.arrayHandling
     };
 
     // Apply transform function if provided
@@ -302,24 +322,30 @@ try {
     // Convert to CSV
     const csvData = jtcsv.jsonToCsv(transformedData, jtcsvOptions);
 
-    // Write output file
-    await fs.promises.writeFile(outputFile, csvData, 'utf8');
-
-    const elapsed = Date.now() - startTime;
-    if (!options.silent) {
-      console.log(
-        color(
-          `✓ Converted ${transformedData.length.toLocaleString()} records in ${elapsed}ms`,
-          'green'
-        )
-      );
-      console.log(
-        color(
-          `  Output: ${outputFile} (${csvData.length.toLocaleString()} bytes)`,
-          'dim'
-        )
-      );
-    }
+    // Write output
+    if (useStdout) {
+      process.stdout.write(csvData);
+    } else {
+      await fs.promises.writeFile(outputFile, csvData, 'utf8');
+    }
+
+    const elapsed = Date.now() - startTime;
+    if (shouldLog) {
+      console.log(
+        color(
+          `✓ Converted ${transformedData.length.toLocaleString()} records in ${elapsed}ms`,
+          'green'
+        )
+      );
+      if (!useStdout) {
+        console.log(
+          color(
+            `  Output: ${outputFile} (${csvData.length.toLocaleString()} bytes)`,
+            'dim'
+          )
+        );
+      }
+    }
 
     return {
       records: transformedData.length,
@@ -336,12 +362,15 @@ try {
 }
 
 async function convertCsvToJson(inputFile, outputFile, options: any): Promise<ConversionResult> {
-  const startTime = Date.now();
-
-  try {
-    if (!options.silent) {
-      console.log(color('Reading CSV file...', 'dim'));
-    }
+  const startTime = Date.now();
+  const useStdin = inputFile === '-';
+  const useStdout = outputFile === '-';
+  const shouldLog = !options.silent && !useStdout;
+
+  try {
+    if (shouldLog) {
+      console.log(color('Reading CSV file...', 'dim'));
+    }
 
     // Prepare options for jtcsv
     const jtcsvOptions = {
@@ -351,38 +380,46 @@ async function convertCsvToJson(inputFile, outputFile, options: any): Promise<Co
       hasHeaders: options.hasHeaders,
       renameMap: options.renameMap,
       trim: options.trim,
-      parseNumbers: options.parseNumbers,
-      parseBooleans: options.parseBooleans,
-      maxRows: options.maxRows,
-      useFastPath: options.useFastPath,
-      fastPathMode: options.fastPathMode,
-      schema: options.schema // Add schema option if supported
-    };
-
-    // Read and convert CSV
-    const jsonData = await jtcsv.readCsvAsJson(inputFile, jtcsvOptions);
+      parseNumbers: options.parseNumbers,
+      parseBooleans: options.parseBooleans,
+      maxRows: options.maxRows,
+      useFastPath: options.useFastPath,
+      fastPathMode: options.fastPathMode,
+      repairRowShifts: options.repairRowShifts,
+      normalizeQuotes: options.normalizeQuotes,
+      schema: options.schema // Add schema option if supported
+    };
+
+    // Read and convert CSV
+    let jsonData: any;
+    if (useStdin) {
+      const csvContent = await readStdin();
+      jsonData = jtcsv.csvToJson(csvContent, jtcsvOptions);
+    } else {
+      jsonData = await jtcsv.readCsvAsJson(inputFile, jtcsvOptions);
+    }
 
     // Apply transform if specified
     let transformedData = jsonData;
-    if (options.transform) {
-      if (!options.silent) {
-        console.log(
-          color(`Applying transform from: ${options.transform}`, 'dim')
-        );
-      }
+    if (options.transform) {
+      if (shouldLog) {
+        console.log(
+          color(`Applying transform from: ${options.transform}`, 'dim')
+        );
+      }
       try {
         transformedData = transformLoader.applyTransform(
           jsonData,
           options.transform
         );
-        if (!options.silent) {
-          console.log(
-            color(
-              `✓ Transform applied to ${transformedData.length} rows`,
-              'green'
-            )
-          );
-        }
+        if (shouldLog) {
+          console.log(
+            color(
+              `✓ Transform applied to ${transformedData.length} rows`,
+              'green'
+            )
+          );
+        }
       } catch (transformError: any) {
         console.error(
           color(`✗ Transform error: ${transformError.message}`, 'red')
@@ -399,24 +436,30 @@ async function convertCsvToJson(inputFile, outputFile, options: any): Promise<Co
       ? JSON.stringify(transformedData, null, 2)
       : JSON.stringify(transformedData);
 
-    // Write output file
-    await fs.promises.writeFile(outputFile, jsonOutput, 'utf8');
-
-    const elapsed = Date.now() - startTime;
-    if (!options.silent) {
-      console.log(
-        color(
-          `✓ Converted ${transformedData.length.toLocaleString()} rows in ${elapsed}ms`,
-          'green'
-        )
-      );
-      console.log(
-        color(
-          `  Output: ${outputFile} (${jsonOutput.length.toLocaleString()} bytes)`,
-          'dim'
-        )
-      );
-    }
+    // Write output
+    if (useStdout) {
+      process.stdout.write(jsonOutput);
+    } else {
+      await fs.promises.writeFile(outputFile, jsonOutput, 'utf8');
+    }
+
+    const elapsed = Date.now() - startTime;
+    if (shouldLog) {
+      console.log(
+        color(
+          `✓ Converted ${transformedData.length.toLocaleString()} rows in ${elapsed}ms`,
+          'green'
+        )
+      );
+      if (!useStdout) {
+        console.log(
+          color(
+            `  Output: ${outputFile} (${jsonOutput.length.toLocaleString()} bytes)`,
+            'dim'
+          )
+        );
+      }
+    }
 
     return {
       rows: transformedData.length,
@@ -453,14 +496,16 @@ async function saveAsCsv(inputFile, outputFile, options: any): Promise<Conversio
       }
       try {
         // Для CSV нужно сначала распарсить, применить трансформацию, затем снова сериализовать
-        const parsedData = jtcsv.csvToJson(inputData, {
-          delimiter: options.delimiter,
-          autoDetect: options.autoDetect,
-          hasHeaders: options.hasHeaders,
-          trim: options.trim,
-          parseNumbers: options.parseNumbers,
-          parseBooleans: options.parseBooleans
-        });
+        const parsedData = jtcsv.csvToJson(inputData, {
+          delimiter: options.delimiter,
+          autoDetect: options.autoDetect,
+          hasHeaders: options.hasHeaders,
+          trim: options.trim,
+          parseNumbers: options.parseNumbers,
+          parseBooleans: options.parseBooleans,
+          repairRowShifts: options.repairRowShifts,
+          normalizeQuotes: options.normalizeQuotes
+        });
 
         const transformedJson = transformLoader.applyTransform(
           parsedData,
@@ -468,10 +513,11 @@ async function saveAsCsv(inputFile, outputFile, options: any): Promise<Conversio
         );
 
         // Конвертировать обратно в CSV
-        transformedData = jtcsv.jsonToCsv(transformedJson, {
-          delimiter: options.delimiter,
-          includeHeaders: options.includeHeaders
-        });
+        transformedData = jtcsv.jsonToCsv(transformedJson, {
+          delimiter: options.delimiter,
+          includeHeaders: options.includeHeaders,
+          normalizeQuotes: options.normalizeQuotes
+        });
 
         if (!options.silent) {
           console.log(color('✓ Transform applied', 'green'));
@@ -597,14 +643,15 @@ async function convertNdjsonToCsv(inputFile, outputFile, options: any): Promise<
     }
 
     // Prepare options for jtcsv
-    const jtcsvOptions = {
-      delimiter: options.delimiter,
-      includeHeaders: options.includeHeaders,
-      renameMap: options.renameMap,
-      template: options.template,
-      preventCsvInjection: options.preventCsvInjection,
-      rfc4180Compliant: options.rfc4180Compliant
-    };
+    const jtcsvOptions = {
+      delimiter: options.delimiter,
+      includeHeaders: options.includeHeaders,
+      renameMap: options.renameMap,
+      template: options.template,
+      preventCsvInjection: options.preventCsvInjection,
+      rfc4180Compliant: options.rfc4180Compliant,
+      normalizeQuotes: options.normalizeQuotes
+    };
 
     // Convert to CSV
     const csvData = jtcsv.jsonToCsv(jsonData, jtcsvOptions);
@@ -1866,10 +1913,12 @@ function parseOptions(args: any): any {
     template: undefined,
     trim: true,
     parseNumbers: false,
-    parseBooleans: false,
-    useFastPath: true,
-    fastPathMode: 'objects',
-    preventCsvInjection: true,
+    parseBooleans: false,
+    useFastPath: true,
+    fastPathMode: 'objects',
+    repairRowShifts: true,
+    normalizeQuotes: true,
+    preventCsvInjection: true,
     rfc4180Compliant: true,
     maxRecords: undefined,
     maxRows: undefined,
@@ -1882,11 +1931,13 @@ function parseOptions(args: any): any {
     addBOM: false,
     unwrapArrays: false,
     stringifyObjects: false,
-    recursive: false,
-    pattern: '**/*',
-    outputDir: './output',
-    overwrite: false,
-    parallel: 4,
+    recursive: false,
+    pattern: '**/*',
+    outputDir: './output',
+    patternProvided: false,
+    outputDirProvided: false,
+    overwrite: false,
+    parallel: 4,
     chunkSize: 65536,
     bufferSize: 1000,
     schema: undefined,
@@ -1938,16 +1989,22 @@ function parseOptions(args: any): any {
       case 'fast-path':
         options.useFastPath = value !== 'false';
         break;
-      case 'fast-path-mode':
-        options.fastPathMode = value || 'objects';
-        if (
-          options.fastPathMode !== 'objects' &&
-            options.fastPathMode !== 'compact'
-        ) {
-          throw new Error('Invalid --fast-path-mode value (objects|compact)');
-        }
-        break;
-      case 'rename':
+      case 'fast-path-mode':
+        options.fastPathMode = value || 'objects';
+        if (
+          options.fastPathMode !== 'objects' &&
+            options.fastPathMode !== 'compact'
+        ) {
+          throw new Error('Invalid --fast-path-mode value (objects|compact)');
+        }
+        break;
+      case 'repair-row-shifts':
+        options.repairRowShifts = true;
+        break;
+      case 'normalize-quotes':
+        options.normalizeQuotes = true;
+        break;
+      case 'rename':
         try {
           const jsonStr = value || '{}';
           const cleanStr = jsonStr
@@ -2026,12 +2083,14 @@ function parseOptions(args: any): any {
       case 'recursive':
         options.recursive = true;
         break;
-      case 'pattern':
-        options.pattern = value || '**/*';
-        break;
-      case 'output-dir':
-        options.outputDir = value || './output';
-        break;
+      case 'pattern':
+        options.pattern = value || '**/*';
+        options.patternProvided = true;
+        break;
+      case 'output-dir':
+        options.outputDir = value || './output';
+        options.outputDirProvided = true;
+        break;
       case 'overwrite':
         options.overwrite = true;
         break;
@@ -2065,10 +2124,10 @@ function parseOptions(args: any): any {
         options.host = value || 'localhost';
         break;
       }
-    } else if (!arg.startsWith('-')) {
-      files.push(arg);
-    }
-  }
+    } else if (arg === '-' || !arg.startsWith('-')) {
+      files.push(arg);
+    }
+  }
 
   return { options, files };
 }
@@ -2453,20 +2512,37 @@ async function main(): Promise<void> {
 
     const batchCommand = args[1].toLowerCase();
     // Для batch команд нужно парсить опции начиная с 3-го аргумента (после batch и подкоманды)
-    const batchArgs = args.slice(2);
-    const { options: batchOptions, files: batchFiles } =
-        parseOptions(batchArgs);
-
-    if (batchCommand === 'json-to-csv' && batchFiles.length >= 2) {
-      await batchJsonToCsv(batchFiles[0], batchFiles[1], batchOptions);
-    } else if (batchCommand === 'csv-to-json' && batchFiles.length >= 2) {
-      await batchCsvToJson(batchFiles[0], batchFiles[1], batchOptions);
-    } else if (batchCommand === 'process' && files.length >= 2) {
-      await batchProcessMixed(files[0], files[1], options);
-    } else {
-      console.error(
-        color('Error: Invalid batch command or missing files', 'red')
-      );
+    const batchArgs = args.slice(2);
+    const { options: batchOptions, files: batchFiles } =
+        parseOptions(batchArgs);
+    const inputPattern = batchFiles[0]
+      || (batchOptions.patternProvided ? batchOptions.pattern : undefined);
+    const outputDir = batchFiles[1]
+      || (batchOptions.outputDirProvided ? batchOptions.outputDir : undefined);
+
+    if (!inputPattern || !outputDir) {
+      console.error(
+        color('Error: Batch mode requires input pattern and output directory', 'red')
+      );
+      console.log(
+        color('Usage: jtcsv batch [json-to-csv|csv-to-json|process] "<pattern>" "<outputDir>"', 'cyan')
+      );
+      console.log(
+        color('Or:    jtcsv batch <subcommand> --pattern="<pattern>" --output-dir="<outputDir>"', 'cyan')
+      );
+      process.exit(1);
+    }
+
+    if (batchCommand === 'json-to-csv') {
+      await batchJsonToCsv(inputPattern, outputDir, batchOptions);
+    } else if (batchCommand === 'csv-to-json') {
+      await batchCsvToJson(inputPattern, outputDir, batchOptions);
+    } else if (batchCommand === 'process') {
+      await batchProcessMixed(inputPattern, outputDir, batchOptions);
+    } else {
+      console.error(
+        color('Error: Invalid batch command or missing files', 'red')
+      );
       process.exit(1);
     }
     break;
@@ -2532,3 +2608,5 @@ if (require.main === module) {
 }
 
 module.exports = { main };
+
+