jtcsv 1.1.0

package/stream-csv-to-json.js

@@ -0,0 +1,613 @@
+/**
+ * Stream CSV to JSON Converter - Node.js Module
+ * 
+ * A streaming implementation for converting CSV data to JSON format
+ * with memory-efficient processing for large files.
+ * 
+ * @module stream-csv-to-json
+ */
+
+const {
+  ValidationError,
+  SecurityError,
+  FileSystemError,
+  ParsingError,
+  LimitError,
+  ConfigurationError,
+  safeExecute
+} = require('./errors');
+
+const { Transform, Writable } = require('stream');
+const { pipeline } = require('stream/promises');
+
+/**
+ * Creates a transform stream that converts CSV chunks to JSON objects
+ * 
+ * @param {Object} options - Configuration options
+ * @param {string} [options.delimiter=';'] - CSV delimiter character
+ * @param {boolean} [options.hasHeaders=true] - Whether CSV has headers row
+ * @param {Object} [options.renameMap={}] - Map for renaming column headers { newKey: oldKey }
+ * @param {boolean} [options.trim=true] - Trim whitespace from values
+ * @param {boolean} [options.parseNumbers=false] - Parse numeric values
+ * @param {boolean} [options.parseBooleans=false] - Parse boolean values
+ * @param {number} [options.maxRows=Infinity] - Maximum number of rows to process
+ * @param {Function} [options.transform] - Custom transform function for each row
+ * @param {Object} [options.schema] - JSON schema for validation and formatting
+ * @returns {Transform} Transform stream
+ * 
+ * @example
+ * const { createCsvToJsonStream } = require('./stream-csv-to-json');
+ * 
+ * const transformStream = createCsvToJsonStream({
+ *   delimiter: ',',
+ *   parseNumbers: true,
+ *   parseBooleans: true
+ * });
+ * 
+ * // Pipe CSV text to JSON objects
+ * csvReadableStream.pipe(transformStream).pipe(jsonWritableStream);
+ */
+function createCsvToJsonStream(options = {}) {
+  return safeExecute(() => {
+    const opts = options && typeof options === 'object' ? options : {};
+    
+    const {
+      delimiter = ';',
+      hasHeaders = true,
+      renameMap = {},
+      trim = true,
+      parseNumbers = false,
+      parseBooleans = false,
+      maxRows = Infinity,
+      transform = null,
+      schema = null
+    } = opts;
+
+    // Validate options
+    if (delimiter && typeof delimiter !== 'string') {
+      throw new ConfigurationError('Delimiter must be a string');
+    }
+    
+    if (delimiter && delimiter.length !== 1) {
+      throw new ConfigurationError('Delimiter must be a single character');
+    }
+    
+    if (renameMap && typeof renameMap !== 'object') {
+      throw new ConfigurationError('renameMap must be an object');
+    }
+    
+    if (maxRows !== Infinity && (typeof maxRows !== 'number' || maxRows <= 0)) {
+      throw new ConfigurationError('maxRows must be a positive number or Infinity');
+    }
+    
+    if (transform && typeof transform !== 'function') {
+      throw new ConfigurationError('transform must be a function');
+    }
+    
+    if (schema && typeof schema !== 'object') {
+      throw new ConfigurationError('schema must be an object');
+    }
+
+    let buffer = '';
+    let headers = null;
+    let headersProcessed = false;
+    let rowCount = 0;
+    let lineNumber = 0;
+    let insideQuotes = false;
+    let schemaValidators = null;
+
+    // Initialize schema validators if schema is provided
+    if (schema) {
+      schemaValidators = createSchemaValidators(schema);
+    }
+
+    /**
+     * Parses a CSV line with proper quote handling
+     * 
+     * @private
+     * @param {string} line - CSV line to parse
+     * @returns {string[]} Array of field values
+     */
+    const parseCsvLine = (line) => {
+      const fields = [];
+      let currentField = '';
+      let insideQuotesLocal = false;
+      let i = 0;
+
+      while (i < line.length) {
+        const char = line[i];
+
+        if (char === '"') {
+          if (insideQuotesLocal) {
+            // Check for escaped quote ("")
+            if (i + 1 < line.length && line[i + 1] === '"') {
+              currentField += '"';
+              i++; // Skip next quote
+            } else {
+              insideQuotesLocal = false;
+            }
+          } else {
+            insideQuotesLocal = true;
+          }
+          i++;
+          continue;
+        }
+
+        if (!insideQuotesLocal && char === delimiter) {
+          fields.push(currentField);
+          currentField = '';
+          i++;
+          continue;
+        }
+
+        currentField += char;
+        i++;
+      }
+
+      // Add last field
+      fields.push(currentField);
+
+      // Check for unclosed quotes
+      if (insideQuotesLocal) {
+        throw new ParsingError('Unclosed quotes in CSV', lineNumber);
+      }
+
+      return fields;
+    };
+
+    /**
+     * Parses a CSV value based on options
+     * 
+     * @private
+     * @param {string} value - Raw CSV value
+     * @returns {*} Parsed value
+     */
+    const parseCsvValue = (value) => {
+      let result = value;
+      
+      if (trim) {
+        result = result.trim();
+      }
+      
+      // Remove Excel formula protection
+      if (result.startsWith("'")) {
+        result = result.substring(1);
+      }
+      
+      // Parse numbers
+      if (parseNumbers && /^-?\d+(\.\d+)?$/.test(result)) {
+        const num = parseFloat(result);
+        if (!isNaN(num)) {
+          return num;
+        }
+      }
+      
+      // Parse booleans
+      if (parseBooleans) {
+        const lowerValue = result.toLowerCase();
+        if (lowerValue === 'true') {
+          return true;
+        }
+        if (lowerValue === 'false') {
+          return false;
+        }
+      }
+      
+      // Parse empty strings as null
+      if (result === '') {
+        return null;
+      }
+      
+      return result;
+    };
+
+    /**
+     * Formats value based on schema
+     * 
+     * @private
+     * @param {*} value - The value to format
+     * @param {string} key - The key/field name
+     * @returns {*} Formatted value
+     */
+    const formatValue = (value, key) => {
+      if (!schemaValidators || !schemaValidators[key]) {
+        return value;
+      }
+      
+      const validator = schemaValidators[key];
+      
+      // Apply formatting if available
+      if (validator.format) {
+        return validator.format(value);
+      }
+      
+      return value;
+    };
+
+    /**
+     * Validates value against schema
+     * 
+     * @private
+     * @param {*} value - The value to validate
+     * @param {string} key - The key/field name
+     * @returns {boolean} True if valid
+     */
+    const validateValue = (value, key) => {
+      if (!schemaValidators || !schemaValidators[key]) {
+        return true;
+      }
+      
+      const validator = schemaValidators[key];
+      
+      // Apply validation if available
+      if (validator.validate) {
+        return validator.validate(value);
+      }
+      
+      return true;
+    };
+
+    /**
+     * Processes a complete line of CSV
+     * 
+     * @private
+     * @param {string} line - Complete CSV line
+     * @returns {Object|null} JSON object or null for headers
+     */
+    const processLine = (line) => {
+      lineNumber++;
+      
+      // Skip empty lines
+      if (line.trim() === '') {
+        return null;
+      }
+
+      try {
+        const fields = parseCsvLine(line);
+        
+        // Process headers
+        if (hasHeaders && !headersProcessed) {
+          headers = fields.map(field => {
+            const trimmed = trim ? field.trim() : field;
+            return renameMap[trimmed] || trimmed;
+          });
+          headersProcessed = true;
+          return null;
+        }
+        
+        // Generate headers if not provided
+        if (!headers) {
+          headers = fields.map((_, index) => `column${index + 1}`);
+        }
+        
+        // Check row limit
+        if (rowCount >= maxRows) {
+          throw new LimitError(
+            `CSV size exceeds maximum limit of ${maxRows} rows`,
+            maxRows,
+            rowCount
+          );
+        }
+        
+        // Build JSON object
+        const row = {};
+        const fieldCount = Math.min(fields.length, headers.length);
+        
+        for (let j = 0; j < fieldCount; j++) {
+          let value = parseCsvValue(fields[j]);
+          const key = headers[j];
+          
+          // Format value based on schema
+          value = formatValue(value, key);
+          
+          // Validate value against schema
+          if (!validateValue(value, key)) {
+            throw new ValidationError(`Invalid value for field '${key}': ${value}`);
+          }
+          
+          row[key] = value;
+        }
+        
+        // Apply custom transform if provided
+        let result = row;
+        if (transform) {
+          try {
+            result = transform(row);
+            if (!result || typeof result !== 'object') {
+              throw new ValidationError('Transform function must return an object');
+            }
+          } catch (error) {
+            throw new ValidationError(`Transform function error: ${error.message}`);
+          }
+        }
+        
+        rowCount++;
+        return result;
+      } catch (error) {
+        if (error instanceof ParsingError) {
+          error.lineNumber = lineNumber;
+        }
+        throw error;
+      }
+    };
+
+    return new Transform({
+      objectMode: true,
+      writableObjectMode: false,
+      readableObjectMode: true,
+      
+      transform(chunk, encoding, callback) {
+        try {
+          const chunkStr = chunk.toString();
+          buffer += chunkStr;
+          
+          // Process complete lines
+          const lines = [];
+          let start = 0;
+          
+          for (let i = 0; i < buffer.length; i++) {
+            const char = buffer[i];
+            
+            if (char === '"') {
+              insideQuotes = !insideQuotes;
+            }
+            
+            if (!insideQuotes && char === '\n') {
+              const line = buffer.substring(start, i).replace(/\r$/, '');
+              lines.push(line);
+              start = i + 1;
+            }
+          }
+          
+          // Keep incomplete line in buffer
+          buffer = buffer.substring(start);
+          
+          // Process complete lines
+          for (const line of lines) {
+            const result = processLine(line);
+            if (result !== null) {
+              this.push(result);
+            }
+          }
+          
+          callback();
+        } catch (error) {
+          callback(error);
+        }
+      },
+      
+      flush(callback) {
+        try {
+          // Process remaining buffer
+          if (buffer.trim() !== '') {
+            const result = processLine(buffer.replace(/\r$/, ''));
+            if (result !== null) {
+              this.push(result);
+            }
+          }
+          
+          // If no headers were found but were expected, generate them
+          if (hasHeaders && !headersProcessed && headers === null) {
+            // This means the CSV was empty or had no data rows
+            // Nothing to do here
+          }
+          
+          callback();
+        } catch (error) {
+          callback(error);
+        }
+      }
+    });
+  }, 'STREAM_CREATION_ERROR', { function: 'createCsvToJsonStream' });
+}
+
+/**
+ * Converts a readable stream of CSV text to JSON objects
+ * 
+ * @param {Readable} inputStream - Readable stream of CSV text
+ * @param {Writable} outputStream - Writable stream for JSON objects
+ * @param {Object} options - Configuration options (same as createCsvToJsonStream)
+ * @returns {Promise<void>}
+ * 
+ * @example
+ * const { streamCsvToJson } = require('./stream-csv-to-json');
+ * 
+ * await streamCsvToJson(csvStream, jsonStream, {
+ *   delimiter: ',',
+ *   parseNumbers: true,
+ *   schema: {
+ *     properties: {
+ *       id: { type: 'integer' },
+ *       name: { type: 'string', minLength: 1 }
+ *     }
+ *   }
+ * });
+ */
+async function streamCsvToJson(inputStream, outputStream, options = {}) {
+  return safeExecute(async () => {
+    const transformStream = createCsvToJsonStream(options);
+    
+    await pipeline(
+      inputStream,
+      transformStream,
+      outputStream
+    );
+  }, 'STREAM_PROCESSING_ERROR', { function: 'streamCsvToJson' });
+}
+
+/**
+ * Reads CSV file and converts it to JSON using streaming
+ * 
+ * @param {string} filePath - Path to CSV file
+ * @param {Object} options - Configuration options (same as createCsvToJsonStream)
+ * @returns {Promise<Readable>} Readable stream of JSON objects
+ * 
+ * @example
+ * const { createCsvFileToJsonStream } = require('./stream-csv-to-json');
+ * 
+ * const jsonStream = await createCsvFileToJsonStream('./large-data.csv', {
+ *   delimiter: ',',
+ *   parseNumbers: true
+ * });
+ * 
+ * jsonStream.pipe(process.stdout);
+ */
+async function createCsvFileToJsonStream(filePath, options = {}) {
+  return safeExecute(async () => {
+    const fs = require('fs');
+    const path = require('path');
+    
+    // Validate file path
+    if (typeof filePath !== 'string' || filePath.trim() === '') {
+      throw new ValidationError('File path must be a non-empty string');
+    }
+    
+    if (!filePath.toLowerCase().endsWith('.csv')) {
+      throw new ValidationError('File must have .csv extension');
+    }
+    
+    // Prevent directory traversal attacks
+    const normalizedPath = path.normalize(filePath);
+    if (normalizedPath.includes('..') || 
+        /\\\.\.\\|\/\.\.\//.test(filePath) ||
+        filePath.startsWith('..') ||
+        filePath.includes('/..')) {
+      throw new SecurityError('Directory traversal detected in file path');
+    }
+    
+    const safePath = path.resolve(filePath);
+    
+    // Check if file exists
+    try {
+      await fs.promises.access(safePath, fs.constants.R_OK);
+    } catch (error) {
+      throw new FileSystemError(`File not found or not readable: ${safePath}`, error);
+    }
+    
+    // Create read stream
+    const readStream = fs.createReadStream(safePath, 'utf8');
+    const transformStream = createCsvToJsonStream(options);
+    
+    // Pipe through transform
+    return readStream.pipe(transformStream);
+  }, 'FILE_STREAM_ERROR', { function: 'createCsvFileToJsonStream' });
+}
+
+/**
+ * Creates schema validators from JSON schema
+ * 
+ * @private
+ * @param {Object} schema - JSON schema
+ * @returns {Object} Validators object
+ */
+function createSchemaValidators(schema) {
+  const validators = {};
+  
+  if (!schema.properties) {
+    return validators;
+  }
+  
+  for (const [key, definition] of Object.entries(schema.properties)) {
+    const validator = {
+      type: definition.type,
+      required: schema.required && schema.required.includes(key)
+    };
+    
+    // Add format function for dates
+    if (definition.type === 'string' && definition.format === 'date-time') {
+      validator.format = (value) => {
+        if (value instanceof Date) {
+          return value.toISOString();
+        }
+        if (typeof value === 'string') {
+          // Try to parse as date
+          const date = new Date(value);
+          if (!isNaN(date.getTime())) {
+            return date.toISOString();
+          }
+        }
+        return value;
+      };
+    }
+    
+    // Add validation function
+    validator.validate = (value) => {
+      if (value === null || value === undefined) {
+        return !validator.required;
+      }
+      
+      // Type validation
+      if (definition.type === 'string' && typeof value !== 'string') {
+        return false;
+      }
+      if (definition.type === 'number' && typeof value !== 'number') {
+        return false;
+      }
+      if (definition.type === 'integer' && (!Number.isInteger(value) || typeof value !== 'number')) {
+        return false;
+      }
+      if (definition.type === 'boolean' && typeof value !== 'boolean') {
+        return false;
+      }
+      
+      // Additional constraints
+      if (definition.minimum !== undefined && value < definition.minimum) {
+        return false;
+      }
+      if (definition.maximum !== undefined && value > definition.maximum) {
+        return false;
+      }
+      if (definition.minLength !== undefined && value.length < definition.minLength) {
+        return false;
+      }
+      if (definition.maxLength !== undefined && value.length > definition.maxLength) {
+        return false;
+      }
+      if (definition.pattern && !new RegExp(definition.pattern).test(value)) {
+        return false;
+      }
+      
+      return true;
+    };
+    
+    validators[key] = validator;
+  }
+  
+  return validators;
+}
+
+/**
+ * Creates a writable stream that collects JSON objects into an array
+ * 
+ * @returns {Writable} Writable stream that collects data
+ */
+function createJsonCollectorStream() {
+  const collectedData = [];
+  
+  return new Writable({
+    objectMode: true,
+    
+    write(chunk, encoding, callback) {
+      collectedData.push(chunk);
+      callback();
+    },
+    
+    final(callback) {
+      this._collectedData = collectedData;
+      callback();
+    }
+  });
+}
+
+module.exports = {
+  createCsvToJsonStream,
+  streamCsvToJson,
+  createCsvFileToJsonStream,
+  createJsonCollectorStream,
+  createSchemaValidators
+};
+
+// For ES6 module compatibility
+if (typeof module !== 'undefined' && module.exports) {
+  module.exports.default = createCsvToJsonStream;
+}