smarter_csv 1.11.2 → 1.12.0

data/lib/smarter_csv.rb

@@ -4,17 +4,16 @@ require "smarter_csv/version"
 require "smarter_csv/errors"
 
 require "smarter_csv/file_io"
-require "smarter_csv/options_processing"
+require "smarter_csv/options"
 require "smarter_csv/auto_detection"
-require "smarter_csv/variables"
 require 'smarter_csv/header_transformations'
 require 'smarter_csv/header_validations'
 require "smarter_csv/headers"
 require "smarter_csv/hash_transformations"
 
-require "smarter_csv/parse"
+require "smarter_csv/parser"
 require "smarter_csv/writer"
-require "smarter_csv/smarter_csv"
+require "smarter_csv/reader"
 
 # load the C-extension:
 case RUBY_ENGINE
@@ -55,6 +54,23 @@ end
 # :nocov:
 
 module SmarterCSV
+  # For backwards compatibility:
+  #
+  # while `SmarterCSV.process` works for simple cases, you can't get access to the internal state any longer.
+  # e.g. you need the instance of the Reader to access the original headers
+  #
+  # Please use this instead:
+  #
+  #   reader = SmarterCSV::Reader.new(input, options)
+  #   reader.process # with or without block
+  #
+  def self.process(input, given_options = {}, &block)
+    reader = Reader.new(input, given_options)
+    reader.process(&block)
+  end
+
+  # Convenience method for generating CSV files:
+  #
   # SmarterCSV.generate(filename, options) do |csv_writer|
   #   MyModel.find_in_batches(batch_size: 100) do |batch|
   #    batch.pluck(:name, :description, :instructor).each do |record|

data/smarter_csv.gemspec

@@ -9,8 +9,8 @@ Gem::Specification.new do |spec|
   spec.authors       = ["Tilo Sloboda"]
   spec.email         = ["tilo.sloboda@gmail.com"]
 
-  spec.summary       = "CSV Reading and Writing"
-  spec.description   = "Ruby Gem for convenient reading and writing: importing of CSV Files as Array(s) of Hashes, with lots of features for processing large files in parallel, embedded comments, unusual field- and record-separators, flexible mapping of CSV-headers to Hash-keys"
+  spec.summary       = "Convenient CSV Reading and Writing"
+  spec.description   = "Ruby Gem for convenient reading and writing of CSV files. It has intelligent defaults, and auto-discovery of column and row separators. It imports CSV Files as Array(s) of Hashes, suitable for direct processing with ActiveRecord, kicking-off batch jobs with Sidekiq, parallel processing, or oploading data to S3. Similarly, writing CSV files takes Hashes, or Arrays of Hashes to create a CSV file."
   spec.homepage      = "https://github.com/tilo/smarter_csv"
   spec.license       = 'MIT'
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: smarter_csv
 version: !ruby/object:Gem::Version
-  version: 1.11.2
+  version: 1.12.0
 platform: ruby
 authors:
 - Tilo Sloboda
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-07-06 00:00:00.000000000 Z
+date: 2024-07-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: awesome_print
@@ -94,10 +94,11 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: 'Ruby Gem for convenient reading and writing: importing of CSV Files
-  as Array(s) of Hashes, with lots of features for processing large files in parallel,
-  embedded comments, unusual field- and record-separators, flexible mapping of CSV-headers
-  to Hash-keys'
+description: Ruby Gem for convenient reading and writing of CSV files. It has intelligent
+  defaults, and auto-discovery of column and row separators. It imports CSV Files
+  as Array(s) of Hashes, suitable for direct processing with ActiveRecord, kicking-off
+  batch jobs with Sidekiq, parallel processing, or oploading data to S3. Similarly,
+  writing CSV files takes Hashes, or Arrays of Hashes to create a CSV file.
 email:
 - tilo.sloboda@gmail.com
 executables: []
@@ -115,6 +116,16 @@ files:
 - README.md
 - Rakefile
 - TO_DO_v2.md
+- docs/_introduction.md
+- docs/basic_api.md
+- docs/batch_processing.md
+- docs/data_transformations.md
+- docs/examples.md
+- docs/header_transformations.md
+- docs/header_validations.md
+- docs/options.md
+- docs/row_col_sep.md
+- docs/value_converters.md
 - ext/smarter_csv/extconf.rb
 - ext/smarter_csv/smarter_csv.c
 - lib/smarter_csv.rb
@@ -125,10 +136,9 @@ files:
 - lib/smarter_csv/header_transformations.rb
 - lib/smarter_csv/header_validations.rb
 - lib/smarter_csv/headers.rb
-- lib/smarter_csv/options_processing.rb
-- lib/smarter_csv/parse.rb
-- lib/smarter_csv/smarter_csv.rb
-- lib/smarter_csv/variables.rb
+- lib/smarter_csv/options.rb
+- lib/smarter_csv/parser.rb
+- lib/smarter_csv/reader.rb
 - lib/smarter_csv/version.rb
 - lib/smarter_csv/writer.rb
 - smarter_csv.gemspec
@@ -158,5 +168,5 @@ requirements: []
 rubygems_version: 3.2.3
 signing_key:
 specification_version: 4
-summary: CSV Reading and Writing
+summary: Convenient CSV Reading and Writing
 test_files: []

data/lib/smarter_csv/smarter_csv.rb

@@ -1,210 +0,0 @@
-# frozen_string_literal: true
-
-module SmarterCSV
-  # first parameter: filename or input object which responds to readline method
-  def SmarterCSV.process(input, given_options = {}, &block) # rubocop:disable Lint/UnusedMethodArgument
-    initialize_variables
-
-    options = process_options(given_options)
-
-    @enforce_utf8 = options[:force_utf8] || options[:file_encoding] !~ /utf-8/i
-    @verbose = options[:verbose]
-
-    begin
-      fh = input.respond_to?(:readline) ? input : File.open(input, "r:#{options[:file_encoding]}")
-
-      if (options[:force_utf8] || options[:file_encoding] =~ /utf-8/i) && (fh.respond_to?(:external_encoding) && fh.external_encoding != Encoding.find('UTF-8') || fh.respond_to?(:encoding) && fh.encoding != Encoding.find('UTF-8'))
-        puts 'WARNING: you are trying to process UTF-8 input, but did not open the input with "b:utf-8" option. See README file "NOTES about File Encodings".'
-      end
-
-      # auto-detect the row separator
-      options[:row_sep] = guess_line_ending(fh, options) if options[:row_sep]&.to_sym == :auto
-      # attempt to auto-detect column separator
-      options[:col_sep] = guess_column_separator(fh, options) if options[:col_sep]&.to_sym == :auto
-
-      skip_lines(fh, options)
-
-      @headers, header_size = process_headers(fh, options)
-      @headerA = @headers # @headerA is deprecated, use @headers
-
-      puts "Effective headers:\n#{pp(@headers)}\n" if @verbose
-
-      header_validations(@headers, options)
-
-      # in case we use chunking.. we'll need to set it up..
-      if options[:chunk_size].to_i > 0
-        use_chunks = true
-        chunk_size = options[:chunk_size].to_i
-        @chunk_count = 0
-        chunk = []
-      else
-        use_chunks = false
-      end
-
-      # now on to processing all the rest of the lines in the CSV file:
-      # fh.each_line |line|
-      until fh.eof? # we can't use fh.readlines() here, because this would read the whole file into memory at once, and eof => true
-        line = readline_with_counts(fh, options)
-
-        # replace invalid byte sequence in UTF-8 with question mark to avoid errors
-        line = enforce_utf8_encoding(line, options) if @enforce_utf8
-
-        print "processing file line %10d, csv line %10d\r" % [@file_line_count, @csv_line_count] if @verbose
-
-        next if options[:comment_regexp] && line =~ options[:comment_regexp] # ignore all comment lines if there are any
-
-        # cater for the quoted csv data containing the row separator carriage return character
-        # in which case the row data will be split across multiple lines (see the sample content in spec/fixtures/carriage_returns_rn.csv)
-        # by detecting the existence of an uneven number of quote characters
-        multiline = count_quote_chars(line, options[:quote_char]).odd?
-
-        while multiline
-          next_line = fh.readline(options[:row_sep])
-          next_line = enforce_utf8_encoding(next_line, options) if @enforce_utf8
-          line += next_line
-          @file_line_count += 1
-
-          break if fh.eof? # Exit loop if end of file is reached
-
-          multiline = count_quote_chars(line, options[:quote_char]).odd?
-        end
-
-        # :nocov:
-        if multiline && @verbose
-          print "\nline contains uneven number of quote chars so including content through file line %d\n" % @file_line_count
-        end
-        # :nocov:
-
-        line.chomp!(options[:row_sep])
-
-        # --- SPLIT LINE & DATA TRANSFORMATIONS ------------------------------------------------------------
-        dataA, _data_size = parse(line, options, header_size)
-
-        dataA.map!{|x| x.strip} if options[:strip_whitespace]
-
-        # if all values are blank, then ignore this line
-        next if options[:remove_empty_hashes] && (dataA.empty? || blank?(dataA))
-
-        # --- HASH TRANSFORMATIONS ------------------------------------------------------------
-        hash = @headers.zip(dataA).to_h
-
-        hash = hash_transformations(hash, options)
-
-        # --- HASH VALIDATIONS ----------------------------------------------------------------
-        # will go here, and be able to:
-        #  - validate correct format of the values for fields
-        #  - required fields to be non-empty
-        #  - ...
-        # -------------------------------------------------------------------------------------
-
-        next if options[:remove_empty_hashes] && hash.empty?
-
-        puts "CSV Line #{@file_line_count}: #{pp(hash)}" if @verbose == '2' # very verbose setting
-        # optional adding of csv_line_number to the hash to help debugging
-        hash[:csv_line_number] = @csv_line_count if options[:with_line_numbers]
-
-        # process the chunks or the resulting hash
-        if use_chunks
-          chunk << hash # append temp result to chunk
-
-          if chunk.size >= chunk_size || fh.eof? # if chunk if full, or EOF reached
-            # do something with the chunk
-            if block_given?
-              yield chunk # do something with the hashes in the chunk in the block
-            else
-              @result << chunk.dup # Append chunk to result (use .dup to keep a copy after we do chunk.clear)
-            end
-            @chunk_count += 1
-            chunk.clear # re-initialize for next chunk of data
-          else
-            # the last chunk may contain partial data, which is handled below
-          end
-          # while a chunk is being filled up we don't need to do anything else here
-
-        else # no chunk handling
-          if block_given?
-            yield [hash] # do something with the hash in the block (better to use chunking here)
-          else
-            @result << hash
-          end
-        end
-      end
-
-      # print new line to retain last processing line message
-      print "\n" if @verbose
-
-      # handling of last chunk:
-      if !chunk.nil? && chunk.size > 0
-        # do something with the chunk
-        if block_given?
-          yield chunk # do something with the hashes in the chunk in the block
-        else
-          @result << chunk.dup # Append chunk to result (use .dup to keep a copy after we do chunk.clear)
-        end
-        @chunk_count += 1
-        # chunk = [] # initialize for next chunk of data
-      end
-    ensure
-      fh.close if fh.respond_to?(:close)
-    end
-
-    if block_given?
-      @chunk_count # when we do processing through a block we only care how many chunks we processed
-    else
-      @result # returns either an Array of Hashes, or an Array of Arrays of Hashes (if in chunked mode)
-    end
-  end
-
-  class << self
-    def count_quote_chars(line, quote_char)
-      return 0 if line.nil? || quote_char.nil? || quote_char.empty?
-
-      count = 0
-      escaped = false
-
-      line.each_char do |char|
-        if char == '\\' && !escaped
-          escaped = true
-        else
-          count += 1 if char == quote_char && !escaped
-          escaped = false
-        end
-      end
-
-      count
-    end
-
-    def has_acceleration?
-      @has_acceleration ||= !!defined?(parse_csv_line_c)
-    end
-
-    protected
-
-    # SEE: https://github.com/rails/rails/blob/32015b6f369adc839c4f0955f2d9dce50c0b6123/activesupport/lib/active_support/core_ext/object/blank.rb#L121
-    # and in the future we might also include UTF-8 space characters: https://www.compart.com/en/unicode/category/Zs
-    BLANK_RE = /\A\s*\z/.freeze
-
-    def blank?(value)
-      case value
-      when String
-        BLANK_RE.match?(value)
-      when NilClass
-        true
-      when Array
-        value.all? { |elem| blank?(elem) }
-      when Hash
-        value.values.all? { |elem| blank?(elem) } # Focus on values only
-      else
-        false
-      end
-    end
-
-    private
-
-    def enforce_utf8_encoding(line, options)
-      # return line unless options[:force_utf8] || options[:file_encoding] !~ /utf-8/i
-
-      line.force_encoding('utf-8').encode('utf-8', invalid: :replace, undef: :replace, replace: options[:invalid_byte_sequence])
-    end
-  end
-end

data/lib/smarter_csv/variables.rb

@@ -1,30 +0,0 @@
-# frozen_string_literal: true
-
-module SmarterCSV
-  class << self
-    attr_reader :has_rails, :csv_line_count, :chunk_count, :errors, :file_line_count, :headers, :raw_header, :result, :warnings
-
-    def initialize_variables
-      @has_rails = !!defined?(Rails)
-      @csv_line_count = 0
-      @chunk_count = 0
-      @errors = {}
-      @file_line_count = 0
-      @headerA = []
-      @headers = nil
-      @raw_header = nil # header as it appears in the file
-      @result = []
-      @warnings = {}
-      @enforce_utf8 = false # only set to true if needed (after options parsing)
-    end
-
-    # :nocov:
-    # rubocop:disable Naming/MethodName
-    def headerA
-      warn "Deprecarion Warning: 'headerA' will be removed in future versions. Use 'headders'"
-      @headerA
-    end
-    # rubocop:enable Naming/MethodName
-    # :nocov:
-  end
-end