smarter_csv 1.1.5 → 1.12.1

data/lib/smarter_csv/options.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+module SmarterCSV
+  #
+  # NOTE: this is not called when "parse" methods are tested by themselves
+  #
+  # ONLY FOR BACKWARDS-COMPATIBILITY
+  def self.default_options
+    Options::DEFAULT_OPTIONS
+  end
+
+  module Options
+    DEFAULT_OPTIONS = {
+      acceleration: true, # if user wants to use accelleration or not
+      auto_row_sep_chars: 500,
+      chunk_size: nil,
+      col_sep: :auto, # was: ',',
+      comment_regexp: nil, # was: /\A#/,
+      convert_values_to_numeric: true,
+      downcase_header: true,
+      duplicate_header_suffix: '', # was: nil,
+      file_encoding: 'utf-8',
+      force_simple_split: false,
+      force_utf8: false,
+      headers_in_file: true,
+      invalid_byte_sequence: '',
+      keep_original_headers: false,
+      key_mapping: nil,
+      quote_char: '"',
+      remove_empty_hashes: true,
+      remove_empty_values: true,
+      remove_unmapped_keys: false,
+      remove_values_matching: nil,
+      remove_zero_values: false,
+      required_headers: nil,
+      required_keys: nil,
+      row_sep: :auto, # was: $/,
+      silence_missing_keys: false,
+      skip_lines: nil,
+      strings_as_keys: false,
+      strip_chars_from_headers: nil,
+      strip_whitespace: true,
+      user_provided_headers: nil,
+      value_converters: nil,
+      verbose: false,
+      with_line_numbers: false,
+    }.freeze
+
+    # NOTE: this is not called when "parse" methods are tested by themselves
+    def process_options(given_options = {})
+      puts "User provided options:\n#{pp(given_options)}\n" if given_options[:verbose]
+
+      @options = DEFAULT_OPTIONS.dup.merge!(given_options)
+
+      # fix invalid input
+      @options[:invalid_byte_sequence] ||= ''
+
+      puts "Computed options:\n#{pp(@options)}\n" if @options[:verbose]
+
+      validate_options!(@options)
+      @options
+    end
+
+    private
+
+    def validate_options!(options)
+      # deprecate required_headers
+      unless options[:required_headers].nil?
+        puts "DEPRECATION WARNING: please use 'required_keys' instead of 'required_headers'"
+        if options[:required_keys].nil?
+          options[:required_keys] = options[:required_headers]
+          options[:required_headers] = nil
+        end
+      end
+
+      keys = options.keys
+      errors = []
+      errors << "invalid row_sep" if keys.include?(:row_sep) && !option_valid?(options[:row_sep])
+      errors << "invalid col_sep" if keys.include?(:col_sep) && !option_valid?(options[:col_sep])
+      errors << "invalid quote_char" if keys.include?(:quote_char) && !option_valid?(options[:quote_char])
+      raise SmarterCSV::ValidationError, errors.inspect if errors.any?
+    end
+
+    def option_valid?(str)
+      return true if str.is_a?(Symbol) && str == :auto
+      return true if str.is_a?(String) && !str.empty?
+
+      false
+    end
+
+    def pp(value)
+      defined?(AwesomePrint) ? value.awesome_inspect(index: nil) : value.inspect
+    end
+  end
+end

data/lib/smarter_csv/parser.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module SmarterCSV
+  module Parser
+    protected
+
+    ###
+    ### Thin wrapper around C-extension
+    ###
+    def parse(line, options, header_size = nil)
+      # puts "SmarterCSV.parse OPTIONS: #{options[:acceleration]}" if options[:verbose]
+
+      if options[:acceleration] && has_acceleration
+        # :nocov:
+        has_quotes = line =~ /#{options[:quote_char]}/
+        elements = parse_csv_line_c(line, options[:col_sep], options[:quote_char], header_size)
+        elements.map!{|x| cleanup_quotes(x, options[:quote_char])} if has_quotes
+        [elements, elements.size]
+        # :nocov:
+      else
+        # puts "WARNING: SmarterCSV is using un-accelerated parsing of lines. Check options[:acceleration]"
+        parse_csv_line_ruby(line, options, header_size)
+      end
+    end
+
+    # ------------------------------------------------------------------
+    # Ruby equivalent of the C-extension for parse_line
+    #
+    # parses a single line: either a CSV header and body line
+    # - quoting rules compared to RFC-4180 are somewhat relaxed
+    # - we are not assuming that quotes inside a fields need to be doubled
+    # - we are not assuming that all fields need to be quoted (0 is even)
+    # - works with multi-char col_sep
+    # - if header_size is given, only up to header_size fields are parsed
+    #
+    # We use header_size for parsing the body lines to make sure we always match the number of headers
+    # in case there are trailing col_sep characters in line
+    #
+    # Our convention is that empty fields are returned as empty strings, not as nil.
+    #
+    #
+    # the purpose of the max_size parameter is to handle a corner case where
+    # CSV lines contain more fields than the header.
+    # In which case the remaining fields in the line are ignored
+    #
+    def parse_csv_line_ruby(line, options, header_size = nil)
+      return [] if line.nil?
+
+      line_size = line.size
+      col_sep = options[:col_sep]
+      col_sep_size = col_sep.size
+      quote = options[:quote_char]
+      quote_count = 0
+      elements = []
+      start = 0
+      i = 0
+
+      previous_char = ''
+      while i < line_size
+        if line[i...i+col_sep_size] == col_sep && quote_count.even?
+          break if !header_size.nil? && elements.size >= header_size
+
+          elements << cleanup_quotes(line[start...i], quote)
+          previous_char = line[i]
+          i += col_sep.size
+          start = i
+        else
+          quote_count += 1 if line[i] == quote && previous_char != '\\'
+          previous_char = line[i]
+          i += 1
+        end
+      end
+      elements << cleanup_quotes(line[start..-1], quote) if header_size.nil? || elements.size < header_size
+      [elements, elements.size]
+    end
+
+    def cleanup_quotes(field, quote)
+      return field if field.nil?
+
+      # return if field !~ /#{quote}/ # this check can probably eliminated
+
+      if field.start_with?(quote) && field.end_with?(quote)
+        field.delete_prefix!(quote)
+        field.delete_suffix!(quote)
+      end
+      field.gsub!("#{quote}#{quote}", quote)
+      field
+    end
+  end
+end

data/lib/smarter_csv/reader.rb

@@ -0,0 +1,243 @@
+# frozen_string_literal: true
+
+module SmarterCSV
+  class Reader
+    include ::SmarterCSV::Options
+    include ::SmarterCSV::FileIO
+    include ::SmarterCSV::AutoDetection
+    include ::SmarterCSV::Headers
+    include ::SmarterCSV::HeaderTransformations
+    include ::SmarterCSV::HeaderValidations
+    include ::SmarterCSV::HashTransformations
+    include ::SmarterCSV::Parser
+
+    attr_reader :input, :options
+    attr_reader :csv_line_count, :chunk_count, :file_line_count
+    attr_reader :enforce_utf8, :has_rails, :has_acceleration
+    attr_reader :errors, :warnings, :headers, :raw_header, :result
+
+    # :nocov:
+    # rubocop:disable Naming/MethodName
+    def headerA
+      warn "Deprecarion Warning: 'headerA' will be removed in future versions. Use 'headders'"
+      @headerA
+    end
+    # rubocop:enable Naming/MethodName
+    # :nocov:
+
+    # first parameter: filename or input object which responds to readline method
+    def initialize(input, given_options = {})
+      @input = input
+      @has_rails = !!defined?(Rails)
+      @csv_line_count = 0
+      @chunk_count = 0
+      @errors = {}
+      @file_line_count = 0
+      @headerA = []
+      @headers = nil
+      @raw_header = nil # header as it appears in the file
+      @result = []
+      @warnings = {}
+      @enforce_utf8 = false # only set to true if needed (after options parsing)
+      @options = process_options(given_options)
+      # true if it is compiled with accelleration
+      @has_acceleration = !!SmarterCSV::Parser.respond_to?(:parse_csv_line_c)
+    end
+
+    def process(&block) # rubocop:disable Lint/UnusedMethodArgument
+      @enforce_utf8 = options[:force_utf8] || options[:file_encoding] !~ /utf-8/i
+      @verbose = options[:verbose]
+
+      begin
+        fh = input.respond_to?(:readline) ? input : File.open(input, "r:#{options[:file_encoding]}")
+
+        if (options[:force_utf8] || options[:file_encoding] =~ /utf-8/i) && (fh.respond_to?(:external_encoding) && fh.external_encoding != Encoding.find('UTF-8') || fh.respond_to?(:encoding) && fh.encoding != Encoding.find('UTF-8'))
+          puts 'WARNING: you are trying to process UTF-8 input, but did not open the input with "b:utf-8" option. See README file "NOTES about File Encodings".'
+        end
+
+        # auto-detect the row separator
+        options[:row_sep] = guess_line_ending(fh, options) if options[:row_sep]&.to_sym == :auto
+        # attempt to auto-detect column separator
+        options[:col_sep] = guess_column_separator(fh, options) if options[:col_sep]&.to_sym == :auto
+
+        skip_lines(fh, options)
+
+        @headers, header_size = process_headers(fh, options)
+        @headerA = @headers # @headerA is deprecated, use @headers
+
+        puts "Effective headers:\n#{pp(@headers)}\n" if @verbose
+
+        header_validations(@headers, options)
+
+        # in case we use chunking.. we'll need to set it up..
+        if options[:chunk_size].to_i > 0
+          use_chunks = true
+          chunk_size = options[:chunk_size].to_i
+          @chunk_count = 0
+          chunk = []
+        else
+          use_chunks = false
+        end
+
+        # now on to processing all the rest of the lines in the CSV file:
+        # fh.each_line |line|
+        until fh.eof? # we can't use fh.readlines() here, because this would read the whole file into memory at once, and eof => true
+          line = readline_with_counts(fh, options)
+
+          # replace invalid byte sequence in UTF-8 with question mark to avoid errors
+          line = enforce_utf8_encoding(line, options) if @enforce_utf8
+
+          print "processing file line %10d, csv line %10d\r" % [@file_line_count, @csv_line_count] if @verbose
+
+          next if options[:comment_regexp] && line =~ options[:comment_regexp] # ignore all comment lines if there are any
+
+          # cater for the quoted csv data containing the row separator carriage return character
+          # in which case the row data will be split across multiple lines (see the sample content in spec/fixtures/carriage_returns_rn.csv)
+          # by detecting the existence of an uneven number of quote characters
+          multiline = count_quote_chars(line, options[:quote_char]).odd?
+
+          while multiline
+            next_line = fh.readline(options[:row_sep])
+            next_line = enforce_utf8_encoding(next_line, options) if @enforce_utf8
+            line += next_line
+            @file_line_count += 1
+
+            break if fh.eof? # Exit loop if end of file is reached
+
+            multiline = count_quote_chars(line, options[:quote_char]).odd?
+          end
+
+          # :nocov:
+          if multiline && @verbose
+            print "\nline contains uneven number of quote chars so including content through file line %d\n" % @file_line_count
+          end
+          # :nocov:
+
+          line.chomp!(options[:row_sep])
+
+          # --- SPLIT LINE & DATA TRANSFORMATIONS ------------------------------------------------------------
+          dataA, _data_size = parse(line, options, header_size)
+
+          dataA.map!{|x| x.strip} if options[:strip_whitespace]
+
+          # if all values are blank, then ignore this line
+          next if options[:remove_empty_hashes] && (dataA.empty? || blank?(dataA))
+
+          # --- HASH TRANSFORMATIONS ------------------------------------------------------------
+          hash = @headers.zip(dataA).to_h
+
+          hash = hash_transformations(hash, options)
+
+          # --- HASH VALIDATIONS ----------------------------------------------------------------
+          # will go here, and be able to:
+          #  - validate correct format of the values for fields
+          #  - required fields to be non-empty
+          #  - ...
+          # -------------------------------------------------------------------------------------
+
+          next if options[:remove_empty_hashes] && hash.empty?
+
+          puts "CSV Line #{@file_line_count}: #{pp(hash)}" if @verbose == '2' # very verbose setting
+          # optional adding of csv_line_number to the hash to help debugging
+          hash[:csv_line_number] = @csv_line_count if options[:with_line_numbers]
+
+          # process the chunks or the resulting hash
+          if use_chunks
+            chunk << hash # append temp result to chunk
+
+            if chunk.size >= chunk_size || fh.eof? # if chunk if full, or EOF reached
+              # do something with the chunk
+              if block_given?
+                yield chunk # do something with the hashes in the chunk in the block
+              else
+                @result << chunk.dup # Append chunk to result (use .dup to keep a copy after we do chunk.clear)
+              end
+              @chunk_count += 1
+              chunk.clear # re-initialize for next chunk of data
+            else
+              # the last chunk may contain partial data, which is handled below
+            end
+            # while a chunk is being filled up we don't need to do anything else here
+
+          else # no chunk handling
+            if block_given?
+              yield [hash] # do something with the hash in the block (better to use chunking here)
+            else
+              @result << hash
+            end
+          end
+        end
+
+        # print new line to retain last processing line message
+        print "\n" if @verbose
+
+        # handling of last chunk:
+        if !chunk.nil? && chunk.size > 0
+          # do something with the chunk
+          if block_given?
+            yield chunk # do something with the hashes in the chunk in the block
+          else
+            @result << chunk.dup # Append chunk to result (use .dup to keep a copy after we do chunk.clear)
+          end
+          @chunk_count += 1
+          # chunk = [] # initialize for next chunk of data
+        end
+      ensure
+        fh.close if fh.respond_to?(:close)
+      end
+
+      if block_given?
+        @chunk_count # when we do processing through a block we only care how many chunks we processed
+      else
+        @result # returns either an Array of Hashes, or an Array of Arrays of Hashes (if in chunked mode)
+      end
+    end
+
+    def count_quote_chars(line, quote_char)
+      return 0 if line.nil? || quote_char.nil? || quote_char.empty?
+
+      count = 0
+      escaped = false
+
+      line.each_char do |char|
+        if char == '\\' && !escaped
+          escaped = true
+        else
+          count += 1 if char == quote_char && !escaped
+          escaped = false
+        end
+      end
+
+      count
+    end
+
+    protected
+
+    # SEE: https://github.com/rails/rails/blob/32015b6f369adc839c4f0955f2d9dce50c0b6123/activesupport/lib/active_support/core_ext/object/blank.rb#L121
+    # and in the future we might also include UTF-8 space characters: https://www.compart.com/en/unicode/category/Zs
+    BLANK_RE = /\A\s*\z/.freeze
+
+    def blank?(value)
+      case value
+      when String
+        BLANK_RE.match?(value)
+      when NilClass
+        true
+      when Array
+        value.all? { |elem| blank?(elem) }
+      when Hash
+        value.values.all? { |elem| blank?(elem) } # Focus on values only
+      else
+        false
+      end
+    end
+
+    private
+
+    def enforce_utf8_encoding(line, options)
+      # return line unless options[:force_utf8] || options[:file_encoding] !~ /utf-8/i
+
+      line.force_encoding('utf-8').encode('utf-8', invalid: :replace, undef: :replace, replace: options[:invalid_byte_sequence])
+    end
+  end
+end

data/lib/smarter_csv/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module SmarterCSV
-  VERSION = "1.1.5"
+  VERSION = "1.12.1"
 end

data/lib/smarter_csv/writer.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+module SmarterCSV
+  #
+  # Generate CSV files
+  #
+  # Create an instance of the Writer class with the filename and options.
+  # call `<<` one or mulltiple times to append data to the file.
+  # call `finalize` to save the file.
+  #
+  # The `<<` method can take different arguments:
+  #  * a signle Hash
+  #  * an array of Hashes
+  #  * nested arrays of arrays of Hashes
+  #
+  # By default SmarterCSV::Writer automatically discovers all headers that are present
+  # in the data on-the-fly. This can be disabled, then only given headers are used.
+  # Disabling can be useful when you want to select attributes from hashes, or ActiveRecord instances.
+  #
+  # If `discover_headers` is enabled, and headers are given, any new headers that are found in the data will still be appended.
+  #
+  # The Writer automatically quotes fields containing the col_sep, row_sep, or the quote_char.
+  #
+  # Options:
+  #   col_sep : defaults to , but can be set to any other character
+  #   row_sep : defaults to LF \n , but can be set to \r\n or \r or anything else
+  #   quote_char : defaults to "
+  #   discover_headers : defaults to true
+  #   headers : defaults to []
+  #   force_quotes: defaults to false
+  #   map_headers: defaults to {}, can be a hash of key -> value mappings
+
+  # IMPORTANT NOTES:
+  #  * Data hashes could contain strings or symbols as keys.
+  #    Make sure to use the correct form when specifying headers manually,
+  #    in combination with the :discover_headers option
+
+  attr_reader :options, :row_sep, :col_sep, :quote_char, :force_quotes, :discover_headers, :headers, :map_headers, :output_file
+
+  class Writer
+    def initialize(file_path, options = {})
+      @options = options
+
+      @row_sep = options[:row_sep] || $/ # Defaults to system's row separator. RFC4180 "\r\n"
+      @col_sep = options[:col_sep] || ','
+      @quote_char = options[:quote_char] || '"'
+      @force_quotes = options[:force_quotes] == true
+      @discover_headers = true # defaults to true
+      if options.has_key?(:discover_headers)
+        # passing in the option overrides the default behavior
+        @discover_headers = options[:discover_headers] == true
+      else
+        # disable discover_headers when headers are given explicitly
+        @discover_headers = !(options.has_key?(:map_headers) || options.has_key?(:headers))
+      end
+      @headers = [] # start with empty headers
+      @headers = options[:headers] if options.has_key?(:headers) # unless explicitly given
+      @headers = options[:map_headers].keys if options.has_key?(:map_headers) && !options.has_key?(:headers)
+      @map_headers = options[:map_headers] || {}
+
+      @output_file = File.open(file_path, 'w+')
+      # hidden state:
+      @temp_file = Tempfile.new('tempfile', '/tmp')
+      @quote_regex = Regexp.union(@col_sep, @row_sep, @quote_char)
+    end
+
+    # this can be called many times in order to append lines to the csv file
+    def <<(data)
+      case data
+      when Hash
+        process_hash(data)
+      when Array
+        data.each { |item| self << item }
+      when NilClass
+        # ignore
+      else
+        raise InvalidInputData, "Invalid data type: #{data.class}. Must be a Hash or an Array."
+      end
+    end
+
+    def finalize
+      # Map headers if :map_headers option is provided
+      mapped_headers = @headers.map { |header| @map_headers[header] || header }
+
+      @temp_file.rewind
+      @output_file.write(mapped_headers.join(@col_sep) + @row_sep)
+      @output_file.write(@temp_file.read)
+      @output_file.flush
+      @output_file.close
+      @temp_file.delete
+    end
+
+    private
+
+    def process_hash(hash)
+      if @discover_headers
+        hash_keys = hash.keys
+        new_keys = hash_keys - @headers
+        @headers.concat(new_keys)
+      end
+
+      # Reorder the hash to match the current headers order and fill missing fields
+      ordered_row = @headers.map { |header| hash[header] || '' }
+
+      @temp_file.write ordered_row.map { |value| escape_csv_field(value) }.join(@col_sep) + @row_sep
+    end
+
+    def escape_csv_field(field)
+      if @force_quotes || field.to_s.match(@quote_regex)
+        "\"#{field}\""
+      else
+        field.to_s
+      end
+    end
+  end
+end

data/lib/smarter_csv.rb

@@ -1,4 +1,92 @@
-require 'csv'
+# frozen_string_literal: true
+
 require "smarter_csv/version"
-require "extensions/hash.rb"
-require "smarter_csv/smarter_csv.rb"
+require "smarter_csv/errors"
+
+require "smarter_csv/file_io"
+require "smarter_csv/options"
+require "smarter_csv/auto_detection"
+require 'smarter_csv/header_transformations'
+require 'smarter_csv/header_validations'
+require "smarter_csv/headers"
+require "smarter_csv/hash_transformations"
+
+require "smarter_csv/parser"
+require "smarter_csv/writer"
+require "smarter_csv/reader"
+
+# load the C-extension:
+case RUBY_ENGINE
+when 'ruby'
+  begin
+    if `uname -s`.chomp == 'Darwin'
+      #
+      # Please report if you see cases where the rake-compiler is building x86_64 code on arm64 cpus:
+      # https://github.com/rake-compiler/rake-compiler/issues/231
+      #
+      require 'smarter_csv/smarter_csv.bundle'
+    else
+      # :nocov:
+      require_relative "smarter_csv/smarter_csv"
+      # :nocov:
+    end
+  rescue Exception # rubocop:disable Lint/RescueException
+    #  require_relative 'smarter_csv/smarter_csv'
+  end
+# :nocov:
+# when 'truffleruby'
+#   puts "\n\n truffleruby case in the load path | RUBY_ENGINE: #{RUBY_ENGINE} , #{RUBY_VERSION}\n\n"
+#   # this might not work - if you encounter problems, please contribute and create a PR
+#   # require 'truffleruby/smarter_csv'
+else
+  puts <<-BLOCK_COMMENT
+
+    -------------------------------------------------------------------------
+      RUBY_ENGINE: #{RUBY_ENGINE} , #{RUBY_VERSION}
+
+      Acceleration via C-Extension is currently not supported for #{RUBY_ENGINE}
+
+      Please contribute and create a pull request if you need this
+    -------------------------------------------------------------------------
+
+  BLOCK_COMMENT
+end
+# :nocov:
+
+module SmarterCSV
+  # For backwards compatibility:
+  #
+  # while `SmarterCSV.process` works for simple cases, you can't get access to the internal state any longer.
+  # e.g. you need the instance of the Reader to access the original headers
+  #
+  # Please use this instead:
+  #
+  #   reader = SmarterCSV::Reader.new(input, options)
+  #   reader.process # with or without block
+  #
+  def self.process(input, given_options = {}, &block)
+    reader = Reader.new(input, given_options)
+    reader.process(&block)
+  end
+
+  # Convenience method for generating CSV files:
+  #
+  # SmarterCSV.generate(filename, options) do |csv_writer|
+  #   MyModel.find_in_batches(batch_size: 100) do |batch|
+  #    batch.pluck(:name, :description, :instructor).each do |record|
+  #       csv_writer << record
+  #     end
+  #   end
+  # end
+  #
+  # rubocop:disable Lint/UnusedMethodArgument
+  def self.generate(filename, options = {}, &block)
+    raise unless block_given?
+
+    writer = Writer.new(filename, options)
+    yield writer
+  ensure
+    writer.finalize
+  end
+  # rubocop:enable Lint/UnusedMethodArgument
+end