smarter_csv 1.11.2 → 1.12.0.pre1

data/ext/smarter_csv/smarter_csv.c

@@ -87,9 +87,11 @@ static VALUE rb_parse_csv_line(VALUE self, VALUE line, VALUE col_sep, VALUE quot
 }
 
 VALUE SmarterCSV = Qnil;
+VALUE Parser = Qnil;
 
 void Init_smarter_csv(void) {
-  VALUE SmarterCSV = rb_define_module("SmarterCSV");
+  SmarterCSV = rb_define_module("SmarterCSV");
+  Parser = rb_define_module_under(SmarterCSV, "Parser");
 
-  rb_define_module_function(SmarterCSV, "parse_csv_line_c", rb_parse_csv_line, 4);
+  rb_define_module_function(Parser, "parse_csv_line_c", rb_parse_csv_line, 4);
 }

data/lib/smarter_csv/auto_detection.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module SmarterCSV
-  class << self
+  module AutoDetection
     protected
 
     # If file has headers, then guesses column separator from headers.

data/lib/smarter_csv/file_io.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module SmarterCSV
-  class << self
+  module FileIO
     protected
 
     def readline_with_counts(filehandle, options)

data/lib/smarter_csv/hash_transformations.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module SmarterCSV
-  class << self
+  module HashTransformations
     def hash_transformations(hash, options)
       # there may be unmapped keys, or keys purposedly mapped to nil or an empty key..
       # make sure we delete any key/value pairs from the hash, which the user wanted to delete:

data/lib/smarter_csv/header_transformations.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module SmarterCSV
-  class << self
+  module HeaderTransformations
     # transform the headers that were in the file:
     def header_transformations(header_array, options)
       header_array.map!{|x| x.gsub(%r/#{options[:quote_char]}/, '')}

data/lib/smarter_csv/header_validations.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module SmarterCSV
-  class << self
+  module HeaderValidations
     def header_validations(headers, options)
       check_duplicate_headers(headers, options)
       check_required_headers(headers, options)
@@ -26,7 +26,7 @@ module SmarterCSV
         missing_keys = options[:required_keys].select { |k| !headers_set.include?(k) }
 
         unless missing_keys.empty?
-          raise SmarterCSV::MissingKeys, "ERROR: missing attributes: #{missing_keys.join(',')}. Check `SmarterCSV.headers` for original headers."
+          raise SmarterCSV::MissingKeys, "ERROR: missing attributes: #{missing_keys.join(',')}. Check `reader.headers` for original headers."
         end
       end
     end

data/lib/smarter_csv/headers.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module SmarterCSV
-  class << self
+  module Headers
     def process_headers(filehandle, options)
       @raw_header = nil # header as it appears in the file
       @headers = nil # the processed headers

data/lib/smarter_csv/{options_processing.rb → options.rb}

@@ -1,43 +1,51 @@
 # frozen_string_literal: true
 
 module SmarterCSV
-  DEFAULT_OPTIONS = {
-    acceleration: true,
-    auto_row_sep_chars: 500,
-    chunk_size: nil,
-    col_sep: :auto, # was: ',',
-    comment_regexp: nil, # was: /\A#/,
-    convert_values_to_numeric: true,
-    downcase_header: true,
-    duplicate_header_suffix: '', # was: nil,
-    file_encoding: 'utf-8',
-    force_simple_split: false,
-    force_utf8: false,
-    headers_in_file: true,
-    invalid_byte_sequence: '',
-    keep_original_headers: false,
-    key_mapping: nil,
-    quote_char: '"',
-    remove_empty_hashes: true,
-    remove_empty_values: true,
-    remove_unmapped_keys: false,
-    remove_values_matching: nil,
-    remove_zero_values: false,
-    required_headers: nil,
-    required_keys: nil,
-    row_sep: :auto, # was: $/,
-    silence_missing_keys: false,
-    skip_lines: nil,
-    strings_as_keys: false,
-    strip_chars_from_headers: nil,
-    strip_whitespace: true,
-    user_provided_headers: nil,
-    value_converters: nil,
-    verbose: false,
-    with_line_numbers: false,
-  }.freeze
+  #
+  # NOTE: this is not called when "parse" methods are tested by themselves
+  #
+  # ONLY FOR BACKWARDS-COMPATIBILITY
+  def self.default_options
+    Options::DEFAULT_OPTIONS
+  end
+
+  module Options
+    DEFAULT_OPTIONS = {
+      acceleration: true, # if user wants to use accelleration or not
+      auto_row_sep_chars: 500,
+      chunk_size: nil,
+      col_sep: :auto, # was: ',',
+      comment_regexp: nil, # was: /\A#/,
+      convert_values_to_numeric: true,
+      downcase_header: true,
+      duplicate_header_suffix: '', # was: nil,
+      file_encoding: 'utf-8',
+      force_simple_split: false,
+      force_utf8: false,
+      headers_in_file: true,
+      invalid_byte_sequence: '',
+      keep_original_headers: false,
+      key_mapping: nil,
+      quote_char: '"',
+      remove_empty_hashes: true,
+      remove_empty_values: true,
+      remove_unmapped_keys: false,
+      remove_values_matching: nil,
+      remove_zero_values: false,
+      required_headers: nil,
+      required_keys: nil,
+      row_sep: :auto, # was: $/,
+      silence_missing_keys: false,
+      skip_lines: nil,
+      strings_as_keys: false,
+      strip_chars_from_headers: nil,
+      strip_whitespace: true,
+      user_provided_headers: nil,
+      value_converters: nil,
+      verbose: false,
+      with_line_numbers: false,
+    }.freeze
 
-  class << self
     # NOTE: this is not called when "parse" methods are tested by themselves
     def process_options(given_options = {})
       puts "User provided options:\n#{pp(given_options)}\n" if given_options[:verbose]
@@ -53,13 +61,6 @@ module SmarterCSV
       @options
     end
 
-    # NOTE: this is not called when "parse" methods are tested by themselves
-    #
-    # ONLY FOR BACKWARDS-COMPATIBILITY
-    def default_options
-      DEFAULT_OPTIONS
-    end
-
     private
 
     def validate_options!(options)

data/lib/smarter_csv/{parse.rb → parser.rb}

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module SmarterCSV
-  class << self
+  module Parser
     protected
 
     ###
@@ -10,7 +10,7 @@ module SmarterCSV
     def parse(line, options, header_size = nil)
       # puts "SmarterCSV.parse OPTIONS: #{options[:acceleration]}" if options[:verbose]
 
-      if options[:acceleration] && has_acceleration?
+      if options[:acceleration] && has_acceleration
         # :nocov:
         has_quotes = line =~ /#{options[:quote_char]}/
         elements = parse_csv_line_c(line, options[:col_sep], options[:quote_char], header_size)

data/lib/smarter_csv/reader.rb

@@ -0,0 +1,243 @@
+# frozen_string_literal: true
+
+module SmarterCSV
+  class Reader
+    include ::SmarterCSV::Options
+    include ::SmarterCSV::FileIO
+    include ::SmarterCSV::AutoDetection
+    include ::SmarterCSV::Headers
+    include ::SmarterCSV::HeaderTransformations
+    include ::SmarterCSV::HeaderValidations
+    include ::SmarterCSV::HashTransformations
+    include ::SmarterCSV::Parser
+
+    attr_reader :input, :options
+    attr_reader :csv_line_count, :chunk_count, :file_line_count
+    attr_reader :enforce_utf8, :has_rails, :has_acceleration
+    attr_reader :errors, :warnings, :headers, :raw_header, :result
+
+    # :nocov:
+    # rubocop:disable Naming/MethodName
+    def headerA
+      warn "Deprecarion Warning: 'headerA' will be removed in future versions. Use 'headders'"
+      @headerA
+    end
+    # rubocop:enable Naming/MethodName
+    # :nocov:
+
+    # first parameter: filename or input object which responds to readline method
+    def initialize(input, given_options = {})
+      @input = input
+      @has_rails = !!defined?(Rails)
+      @csv_line_count = 0
+      @chunk_count = 0
+      @errors = {}
+      @file_line_count = 0
+      @headerA = []
+      @headers = nil
+      @raw_header = nil # header as it appears in the file
+      @result = []
+      @warnings = {}
+      @enforce_utf8 = false # only set to true if needed (after options parsing)
+      @options = process_options(given_options)
+      # true if it is compiled with accelleration
+      @has_acceleration = !!SmarterCSV::Parser.respond_to?(:parse_csv_line_c)
+    end
+
+    def process(&block) # rubocop:disable Lint/UnusedMethodArgument
+      @enforce_utf8 = options[:force_utf8] || options[:file_encoding] !~ /utf-8/i
+      @verbose = options[:verbose]
+
+      begin
+        fh = input.respond_to?(:readline) ? input : File.open(input, "r:#{options[:file_encoding]}")
+
+        if (options[:force_utf8] || options[:file_encoding] =~ /utf-8/i) && (fh.respond_to?(:external_encoding) && fh.external_encoding != Encoding.find('UTF-8') || fh.respond_to?(:encoding) && fh.encoding != Encoding.find('UTF-8'))
+          puts 'WARNING: you are trying to process UTF-8 input, but did not open the input with "b:utf-8" option. See README file "NOTES about File Encodings".'
+        end
+
+        # auto-detect the row separator
+        options[:row_sep] = guess_line_ending(fh, options) if options[:row_sep]&.to_sym == :auto
+        # attempt to auto-detect column separator
+        options[:col_sep] = guess_column_separator(fh, options) if options[:col_sep]&.to_sym == :auto
+
+        skip_lines(fh, options)
+
+        @headers, header_size = process_headers(fh, options)
+        @headerA = @headers # @headerA is deprecated, use @headers
+
+        puts "Effective headers:\n#{pp(@headers)}\n" if @verbose
+
+        header_validations(@headers, options)
+
+        # in case we use chunking.. we'll need to set it up..
+        if options[:chunk_size].to_i > 0
+          use_chunks = true
+          chunk_size = options[:chunk_size].to_i
+          @chunk_count = 0
+          chunk = []
+        else
+          use_chunks = false
+        end
+
+        # now on to processing all the rest of the lines in the CSV file:
+        # fh.each_line |line|
+        until fh.eof? # we can't use fh.readlines() here, because this would read the whole file into memory at once, and eof => true
+          line = readline_with_counts(fh, options)
+
+          # replace invalid byte sequence in UTF-8 with question mark to avoid errors
+          line = enforce_utf8_encoding(line, options) if @enforce_utf8
+
+          print "processing file line %10d, csv line %10d\r" % [@file_line_count, @csv_line_count] if @verbose
+
+          next if options[:comment_regexp] && line =~ options[:comment_regexp] # ignore all comment lines if there are any
+
+          # cater for the quoted csv data containing the row separator carriage return character
+          # in which case the row data will be split across multiple lines (see the sample content in spec/fixtures/carriage_returns_rn.csv)
+          # by detecting the existence of an uneven number of quote characters
+          multiline = count_quote_chars(line, options[:quote_char]).odd?
+
+          while multiline
+            next_line = fh.readline(options[:row_sep])
+            next_line = enforce_utf8_encoding(next_line, options) if @enforce_utf8
+            line += next_line
+            @file_line_count += 1
+
+            break if fh.eof? # Exit loop if end of file is reached
+
+            multiline = count_quote_chars(line, options[:quote_char]).odd?
+          end
+
+          # :nocov:
+          if multiline && @verbose
+            print "\nline contains uneven number of quote chars so including content through file line %d\n" % @file_line_count
+          end
+          # :nocov:
+
+          line.chomp!(options[:row_sep])
+
+          # --- SPLIT LINE & DATA TRANSFORMATIONS ------------------------------------------------------------
+          dataA, _data_size = parse(line, options, header_size)
+
+          dataA.map!{|x| x.strip} if options[:strip_whitespace]
+
+          # if all values are blank, then ignore this line
+          next if options[:remove_empty_hashes] && (dataA.empty? || blank?(dataA))
+
+          # --- HASH TRANSFORMATIONS ------------------------------------------------------------
+          hash = @headers.zip(dataA).to_h
+
+          hash = hash_transformations(hash, options)
+
+          # --- HASH VALIDATIONS ----------------------------------------------------------------
+          # will go here, and be able to:
+          #  - validate correct format of the values for fields
+          #  - required fields to be non-empty
+          #  - ...
+          # -------------------------------------------------------------------------------------
+
+          next if options[:remove_empty_hashes] && hash.empty?
+
+          puts "CSV Line #{@file_line_count}: #{pp(hash)}" if @verbose == '2' # very verbose setting
+          # optional adding of csv_line_number to the hash to help debugging
+          hash[:csv_line_number] = @csv_line_count if options[:with_line_numbers]
+
+          # process the chunks or the resulting hash
+          if use_chunks
+            chunk << hash # append temp result to chunk
+
+            if chunk.size >= chunk_size || fh.eof? # if chunk if full, or EOF reached
+              # do something with the chunk
+              if block_given?
+                yield chunk # do something with the hashes in the chunk in the block
+              else
+                @result << chunk.dup # Append chunk to result (use .dup to keep a copy after we do chunk.clear)
+              end
+              @chunk_count += 1
+              chunk.clear # re-initialize for next chunk of data
+            else
+              # the last chunk may contain partial data, which is handled below
+            end
+            # while a chunk is being filled up we don't need to do anything else here
+
+          else # no chunk handling
+            if block_given?
+              yield [hash] # do something with the hash in the block (better to use chunking here)
+            else
+              @result << hash
+            end
+          end
+        end
+
+        # print new line to retain last processing line message
+        print "\n" if @verbose
+
+        # handling of last chunk:
+        if !chunk.nil? && chunk.size > 0
+          # do something with the chunk
+          if block_given?
+            yield chunk # do something with the hashes in the chunk in the block
+          else
+            @result << chunk.dup # Append chunk to result (use .dup to keep a copy after we do chunk.clear)
+          end
+          @chunk_count += 1
+          # chunk = [] # initialize for next chunk of data
+        end
+      ensure
+        fh.close if fh.respond_to?(:close)
+      end
+
+      if block_given?
+        @chunk_count # when we do processing through a block we only care how many chunks we processed
+      else
+        @result # returns either an Array of Hashes, or an Array of Arrays of Hashes (if in chunked mode)
+      end
+    end
+
+    def count_quote_chars(line, quote_char)
+      return 0 if line.nil? || quote_char.nil? || quote_char.empty?
+
+      count = 0
+      escaped = false
+
+      line.each_char do |char|
+        if char == '\\' && !escaped
+          escaped = true
+        else
+          count += 1 if char == quote_char && !escaped
+          escaped = false
+        end
+      end
+
+      count
+    end
+
+    protected
+
+    # SEE: https://github.com/rails/rails/blob/32015b6f369adc839c4f0955f2d9dce50c0b6123/activesupport/lib/active_support/core_ext/object/blank.rb#L121
+    # and in the future we might also include UTF-8 space characters: https://www.compart.com/en/unicode/category/Zs
+    BLANK_RE = /\A\s*\z/.freeze
+
+    def blank?(value)
+      case value
+      when String
+        BLANK_RE.match?(value)
+      when NilClass
+        true
+      when Array
+        value.all? { |elem| blank?(elem) }
+      when Hash
+        value.values.all? { |elem| blank?(elem) } # Focus on values only
+      else
+        false
+      end
+    end
+
+    private
+
+    def enforce_utf8_encoding(line, options)
+      # return line unless options[:force_utf8] || options[:file_encoding] !~ /utf-8/i
+
+      line.force_encoding('utf-8').encode('utf-8', invalid: :replace, undef: :replace, replace: options[:invalid_byte_sequence])
+    end
+  end
+end

data/lib/smarter_csv/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SmarterCSV
-  VERSION = "1.11.2"
+  VERSION = "1.12.0.pre1"
 end

data/lib/smarter_csv/writer.rb

@@ -40,7 +40,8 @@ module SmarterCSV
   class Writer
     def initialize(file_path, options = {})
       @options = options
-      @row_sep = options[:row_sep] || "\n" # RFC4180 "\r\n"
+
+      @row_sep = options[:row_sep] || $/ # Defaults to system's row separator. RFC4180 "\r\n"
       @col_sep = options[:col_sep] || ','
       @quote_char = options[:quote_char] || '"'
       @force_quotes = options[:force_quotes] == true

data/lib/smarter_csv.rb

@@ -4,17 +4,16 @@ require "smarter_csv/version"
 require "smarter_csv/errors"
 
 require "smarter_csv/file_io"
-require "smarter_csv/options_processing"
+require "smarter_csv/options"
 require "smarter_csv/auto_detection"
-require "smarter_csv/variables"
 require 'smarter_csv/header_transformations'
 require 'smarter_csv/header_validations'
 require "smarter_csv/headers"
 require "smarter_csv/hash_transformations"
 
-require "smarter_csv/parse"
+require "smarter_csv/parser"
 require "smarter_csv/writer"
-require "smarter_csv/smarter_csv"
+require "smarter_csv/reader"
 
 # load the C-extension:
 case RUBY_ENGINE
@@ -55,6 +54,23 @@ end
 # :nocov:
 
 module SmarterCSV
+  # For backwards compatibility:
+  #
+  # while `SmarterCSV.process` works for simple cases, you can't get access to the internal state any longer.
+  # e.g. you need the instance of the Reader to access the original headers
+  #
+  # Please use this instead:
+  #
+  #   reader = SmarterCSV::Reader.new(input, options)
+  #   reader.process # with or without block
+  #
+  def self.process(input, given_options = {}, &block)
+    reader = Reader.new(input, given_options)
+    reader.process(&block)
+  end
+
+  # Convenience method for generating CSV files:
+  #
   # SmarterCSV.generate(filename, options) do |csv_writer|
   #   MyModel.find_in_batches(batch_size: 100) do |batch|
   #    batch.pluck(:name, :description, :instructor).each do |record|

data/smarter_csv.gemspec

@@ -9,7 +9,7 @@ Gem::Specification.new do |spec|
   spec.authors       = ["Tilo Sloboda"]
   spec.email         = ["tilo.sloboda@gmail.com"]
 
-  spec.summary       = "CSV Reading and Writing"
+  spec.summary       = "Convenient CSV Reading and Writing"
   spec.description   = "Ruby Gem for convenient reading and writing: importing of CSV Files as Array(s) of Hashes, with lots of features for processing large files in parallel, embedded comments, unusual field- and record-separators, flexible mapping of CSV-headers to Hash-keys"
   spec.homepage      = "https://github.com/tilo/smarter_csv"
   spec.license       = 'MIT'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: smarter_csv
 version: !ruby/object:Gem::Version
-  version: 1.11.2
+  version: 1.12.0.pre1
 platform: ruby
 authors:
 - Tilo Sloboda
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-07-06 00:00:00.000000000 Z
+date: 2024-07-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: awesome_print
@@ -115,6 +115,17 @@ files:
 - README.md
 - Rakefile
 - TO_DO_v2.md
+- docs/_introduction.md
+- docs/basic_api.md
+- docs/batch_processing.md
+- docs/data_transformations.md
+- docs/examples.md
+- docs/header_transformations.md
+- docs/header_validations.md
+- docs/notes.md
+- docs/options.md
+- docs/row_col_sep.md
+- docs/value_converters.md
 - ext/smarter_csv/extconf.rb
 - ext/smarter_csv/smarter_csv.c
 - lib/smarter_csv.rb
@@ -125,10 +136,9 @@ files:
 - lib/smarter_csv/header_transformations.rb
 - lib/smarter_csv/header_validations.rb
 - lib/smarter_csv/headers.rb
-- lib/smarter_csv/options_processing.rb
-- lib/smarter_csv/parse.rb
-- lib/smarter_csv/smarter_csv.rb
-- lib/smarter_csv/variables.rb
+- lib/smarter_csv/options.rb
+- lib/smarter_csv/parser.rb
+- lib/smarter_csv/reader.rb
 - lib/smarter_csv/version.rb
 - lib/smarter_csv/writer.rb
 - smarter_csv.gemspec
@@ -151,12 +161,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 2.5.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubygems_version: 3.2.3
 signing_key:
 specification_version: 4
-summary: CSV Reading and Writing
+summary: Convenient CSV Reading and Writing
 test_files: []