smarter_csv 1.11.0.pre2 → 1.11.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9027a37c4b29e68fcbc559a6a8285e5076684883612e98b13f116526dadc6e4b
-  data.tar.gz: 43cfa0254ac2caa8ca02a8863fc790f1c56beb0b16e175a16fd947f92eda8c08
+  metadata.gz: 4a650fbccc5bf703199c6771da75ed95c87c9e3763e176b688a2daaa1b0c4669
+  data.tar.gz: 89cfff5bb92ee8faaeea7a644b61e63a7e47c61f30faefc515b602808eec0989
 SHA512:
-  metadata.gz: d5b2eac35e33bdeb9ec632578207c47f34a08a4595c1ebf04929a7fe302efc3fc08565c0d6fc6454fd538b99cc1c4a5662599a88693e3aa1b80c5c0c7fc1b05e
-  data.tar.gz: 36f622f12d5412ef8919c30a267def315985aef4b3f33c209341a046442795bb66df6c8ebb97399f35e24fd565df658a0e4b7a94d3563831d7ce32facbeab33f
+  metadata.gz: 70ea1888be23a467d15001086935117252a30860ee5a418e8b61b61bfb19a03377391ffb28589db412b0aac4293ac029a047b268d0fa6836ec60f2319cabf102
+  data.tar.gz: 49d7d1d1c4e258611168056a0b8b7433dfedcf37eeac25c22c0aade552a878361d019b86156cbadcacc5996a43cae11d9d55990a0ef9c4a599cf447019cf8d7f

data/.rspec

@@ -0,0 +1 @@
+--require spec_helper

data/.rubocop.yml

@@ -88,18 +88,12 @@ Style/IfInsideElse:
 Style/IfUnlessModifier:
   Enabled: false
 
-Style/InverseMethods:
-  Enabled: false
-
 Style/NestedTernaryOperator:
   Enabled: false
 
 Style/PreferredHashMethods:
   Enabled: false
 
-Style/Proc:
-  Enabled: false
-
 Style/NumericPredicate:
   Enabled: false
 
@@ -118,6 +112,9 @@ Style/SlicingWithRange:
 Style/SpecialGlobalVars: # DANGER: unsafe rule!!
   Enabled: false
 
+Style/StringConcatenation:
+  Enabled: false
+
 Style/StringLiterals:
   Enabled: false
   EnforcedStyle: double_quotes
@@ -135,9 +132,6 @@ Style/SymbolProc: # old Ruby versions can't do this
 Style/TrailingCommaInHashLiteral:
   Enabled: false
 
-Style/TrailingCommaInArrayLiteral:
-  Enabled: false
-
 Style/TrailingUnderscoreVariable:
   Enabled: false
 
@@ -147,9 +141,6 @@ Style/TrivialAccessors:
 # Style/UnlessModifier:
 #   Enabled: false
 
-Style/WordArray:
-  Enabled: false
-
 Style/ZeroLengthPredicate:
   Enabled: false
 

data/CHANGELOG.md

@@ -1,42 +1,23 @@
 
 # SmarterCSV 1.x Change Log
 
-## T.B.D.
-
-  * code refactor
+## 1.11.2 (2024-07-05)
+  * fixing missing errors definition
+    
+## 1.11.1 (2024-07-05) (YANKED)
+  * improved behavior of Writer class
+  * added SmarterCSV.generate shortcut for CSV writing
+    
+## 1.11.0 (2024-07-02)
+  * added SmarterCSV::Writer to output CSV files ([issue #44](https://github.com/tilo/smarter_csv/issues/44))
   
-  * NEW BEHAVIOR:
-    - hidden `:v2_mode` options (incomplete!)
-    - pre-processing for v2 options
-    - implemented v2 `:header_transformations` (DO NOT USE YET!)
-      + -> check if all v1 transformations are correctly done
-        How are we going to 
-        * disambiguate headers?
-
-
-        * do key_mapping? -> seems to work
-          - remove_unmapped_keys  ? 
-          - silence missing keys ... a missing mapped key should raise an exception, except when silenced
-          - required_keys needs to be a header-validation
+## 1.10.3 (2024-03-10)
+  * fixed issue when frozen options are handed in (thanks to Daniel Pepper)
+  * cleaned-up rspec tests (thanks to Daniel Pepper)
+  * fixed link in README (issue #251)
 
-
-        * keep original headers? -> :none
-        * do strings_as_* ? -> either :keys_as_symbols, :keys_as_strings
-        * remove quote_chars? -> included in keys_as_*
-        * strip whitespace? -> included in keys_as_*
-
-        TODO:
-        
-        - add tests for header_validations
-        
-        - modify options to handle v1 and v2 options
-        - add v1 defaults in v2 processing
-        - add tests for all options processing
-        - 100% backwards compatibility when working in v1 mode
-
-
-## 1.10.1 (2024-01-07)
-  * fix incorrect warning about UTF-8 (issue #268, thanks hirowatari)
+## 1.10.2 (2024-02-11)
+  * improve error message for missing keys
 
 ## 1.10.1 (2024-01-07)
   * fix incorrect warning about UTF-8 (issue #268, thanks hirowatari)

data/CONTRIBUTORS.md

@@ -51,4 +51,5 @@ A Big Thank you to everyone who filed issues, sent comments, and who contributed
  * [Rahul Chaudhary](https://github.com/rahulch95)
  * [Alessandro Fazzi](https://github.com/pioneerskies)
  * [JP Camara](https://github.com/jpcamara)
- * [Hiro Watari](https://github.com/hirowatari)
+ * [Kenton Hirowatari](https://github.com/hirowatari)
+ * [Daniel Pepper](https://github.com/dpep)

data/README.md

@@ -3,10 +3,17 @@
 
  [![codecov](https://codecov.io/gh/tilo/smarter_csv/branch/main/graph/badge.svg?token=1L7OD80182)](https://codecov.io/gh/tilo/smarter_csv) [![Gem Version](https://badge.fury.io/rb/smarter_csv.svg)](http://badge.fury.io/rb/smarter_csv)
 
+SmarterCSV provides a complete interface to CSV files and data. It offers tools to enable you to read and write to and from Strings or IO objects, as needed.
 
-#### LATEST CHANGES
+SmarterCSV focuses on representing the data for each row as a hash. 
 
-* Version 1.10.0 has BREAKING CHANGES:
+When reading CSV files, Using an array-of-hashes format makes it much easier to further process the data, or creating database records with it.
+
+When writing CSV data to file, it similarly takes arrays of hashes, and converts them to a CSV file.
+
+#### BREAKING CHANGES
+
+* Version 1.10.0 had BREAKING CHANGES:
 
     Changed behavior:
      + when `user_provided_headers` are provided:
@@ -23,13 +30,8 @@
 
 * default branch is `main` for 1.x development
   
-* 2.x development is on `2.0-development` (check this branch for 2.0 documentation) 
-  - This is an EXPERIMENTAL branch - DO NOT USE in production
-
-#### Work towards Future Version 2.x
-
-* Work towards SmarterCSV 2.x is still ongoing, with improved features, and more streamlined options, but consider it as experimental at this time.
-  Please check the [2.0-develop branch](https://github.com/tilo/smarter_csv/tree/2.0-develop), open any issues and pull requests with mention of tag v2.0.
+* 2.x development is [MOVED TO THIS PR](https://github.com/tilo/smarter_csv/pull/267)
+  - 2.x behavior is still EXPERIMENTAL - DO NOT USE in production
 
 ---------------
 
@@ -394,10 +396,9 @@ And header and data validations will also be supported in 2.x
  * some CSV files use un-escaped quotation characters inside fields. This can cause the import to break. To get around this, use the `:force_simple_split => true` option in combination with `:strip_chars_from_headers => /[\-"]/` . This will also significantly speed up the import.
    If you would force a different :quote_char instead (setting it to a non-used character), then the import would be up to 5-times slower than using `:force_simple_split`.
 
-## See also:
-
-  http://www.unixgods.org/~tilo/Ruby/process_csv_as_hashes.html
+## The original post that started SmarterCSV:
 
+  http://www.unixgods.org/Ruby/process_csv_as_hashes.html
 
 
 ## Installation

data/lib/smarter_csv/errors.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module SmarterCSV
+  class Error < StandardError; end # new code should rescue this instead
+  # Reader:
+  class SmarterCSVException < Error; end # for backwards compatibility
+  class HeaderSizeMismatch < SmarterCSVException; end
+  class IncorrectOption < SmarterCSVException; end
+  class ValidationError < SmarterCSVException; end
+  class DuplicateHeaders < SmarterCSVException; end
+  class MissingKeys < SmarterCSVException; end # previously known as MissingHeaders
+  class NoColSepDetected < SmarterCSVException; end
+  class KeyMappingError < SmarterCSVException; end
+  # Writer:
+  class InvalidInputData < SmarterCSVException; end
+end

data/lib/smarter_csv/hash_transformations.rb

@@ -2,16 +2,7 @@
 
 module SmarterCSV
   class << self
-    # this is processing the headers from the input file
     def hash_transformations(hash, options)
-      if options[:v2_mode]
-        hash_transformations_v2(hash, options)
-      else
-        hash_transformations_v1(hash, options)
-      end
-    end
-
-    def hash_transformations_v1(hash, options)
       # there may be unmapped keys, or keys purposedly mapped to nil or an empty key..
       # make sure we delete any key/value pairs from the hash, which the user wanted to delete:
       remove_empty_values = options[:remove_empty_values] == true
@@ -42,117 +33,46 @@ module SmarterCSV
       end
     end
 
-    def hash_transformations_v2(hash, options)
-      return hash if options[:hash_transformations].nil? || options[:hash_transformations].empty?
-
-      # do the header transformations the user requested:
-      if options[:hash_transformations]
-        options[:hash_transformations].each do |transformation|
-          if transformation.respond_to?(:call) # this is used when a user-provided Proc is passed in
-            hash = transformation.call(hash, options)
-          else
-            case transformation
-            when Symbol # this is used for pre-defined transformations that are defined in the SmarterCSV module
-              hash = public_send(transformation, hash, options)
-            when Hash # this is called for hash arguments, e.g. hash_transformations
-              trans, args = transformation.first # .first treats the hash first element as an array
-              hash = apply_transformation(trans, hash, args, options)
-            when Array # this can be used for passing additional arguments in array form (e.g. into a Proc)
-              trans, *args = transformation
-              hash = apply_transformation(trans, hash, args, options)
-            else
-              raise SmarterCSV::IncorrectOption, "Invalid transformation type: #{transformation.class}"
-            end
-          end
-        end
-      end
-
-      hash
-    end
-
-    #
-    # To handle v1-backward-compatible behavior, it is faster to roll all behavior into one method
-    #
-    def v1_backwards_compatibility(hash, options)
-      hash.each_with_object({}) do |(k, v), new_hash|
-        next if k.nil? || k == '' || k == :"" # remove_empty_keys
-        next if has_rails ? v.blank? : blank?(v) # remove_empty_values
-
-        # convert_values_to_numeric:
-        # deal with the :only / :except options to :convert_values_to_numeric
-        unless limit_execution_for_only_or_except(options, :convert_values_to_numeric, k)
-          if v =~ /^[+-]?\d+\.\d+$/
-            v = v.to_f
-          elsif v =~ /^[+-]?\d+$/
-            v = v.to_i
-          end
-        end
-
-        new_hash[k] = v
-      end
-    end
-
-    #
-    # Building Blocks in case you want to build your own flow:
-    #
-
-    def value_converters(hash, _options)
-      #
-      # TO BE IMPLEMENTED
-      #
-    end
-
-    def strip_spaces(hash, _options)
-      hash.each_key {|key| hash[key].strip! unless hash[key].nil? } # &. syntax was introduced in Ruby 2.3 - need to stay backwards compatible
-    end
-
-    def remove_blank_values(hash, _options)
-      hash.each_key {|key| hash.delete(key) if hash[key].nil? || hash[key].is_a?(String) && hash[key] !~ /[^[:space:]]/ }
-    end
-
-    def remove_zero_values(hash, _options)
-      hash.each_key {|key| hash.delete(key) if hash[key].is_a?(Numeric) && hash[key].zero? }
-    end
-
-    def remove_empty_keys(hash, _options)
-      hash.reject!{|key, _v| key.nil? || key.empty?}
-    end
-
-    def convert_values_to_numeric(hash, _options)
-      hash.each_key do |k|
-        case hash[k]
-        when /^[+-]?\d+\.\d+$/
-          hash[k] = hash[k].to_f
-        when /^[+-]?\d+$/
-          hash[k] = hash[k].to_i
-        end
-      end
-    end
-
-    def convert_values_to_numeric_unless_leading_zeroes(hash, _options)
-      hash.each_key do |k|
-        case hash[k]
-        when /^[+-]?[1-9]\d*\.\d+$/
-          hash[k] = hash[k].to_f
-        when /^[+-]?[1-9]\d*$/
-          hash[k] = hash[k].to_i
-        end
-      end
-    end
-
-    # IMPORTANT NOTE:
-    # this can lead to cases where a nil or empty value gets converted into 0 or 0.0,
-    # and can then not be properly removed!
-    #
-    # you should first try to use convert_values_to_numeric or convert_values_to_numeric_unless_leading_zeroes
-    #
-    def convert_to_integer(hash, _options)
-      hash.each_key {|key| hash[key] = hash[key].to_i }
-    end
-
-    def convert_to_float(hash, _options)
-      hash.each_key {|key| hash[key] = hash[key].to_f }
-    end
+    # def hash_transformations(hash, options)
+    #   # there may be unmapped keys, or keys purposedly mapped to nil or an empty key..
+    #   # make sure we delete any key/value pairs from the hash, which the user wanted to delete:
+    #   hash.delete(nil)
+    #   hash.delete('')
+    #   hash.delete(:"")
+
+    #   if options[:remove_empty_values] == true
+    #     hash.delete_if{|_k, v| has_rails ? v.blank? : blank?(v)}
+    #   end
+
+    #   hash.delete_if{|_k, v| !v.nil? && v =~ /^(0+|0+\.0+)$/} if options[:remove_zero_values] # values are Strings
+    #   hash.delete_if{|_k, v| v =~ options[:remove_values_matching]} if options[:remove_values_matching]
+
+    #   if options[:convert_values_to_numeric]
+    #     hash.each do |k, v|
+    #       # deal with the :only / :except options to :convert_values_to_numeric
+    #       next if limit_execution_for_only_or_except(options, :convert_values_to_numeric, k)
+
+    #       # convert if it's a numeric value:
+    #       case v
+    #       when /^[+-]?\d+\.\d+$/
+    #         hash[k] = v.to_f
+    #       when /^[+-]?\d+$/
+    #         hash[k] = v.to_i
+    #       end
+    #     end
+    #   end
+
+    #   if options[:value_converters]
+    #     hash.each do |k, v|
+    #       converter = options[:value_converters][k]
+    #       next unless converter
+
+    #       hash[k] = converter.convert(v)
+    #     end
+    #   end
+
+    #   hash
+    # end
 
     protected
 

data/lib/smarter_csv/header_transformations.rb

@@ -2,18 +2,8 @@
 
 module SmarterCSV
   class << self
-    # this is processing the headers from the input file
+    # transform the headers that were in the file:
     def header_transformations(header_array, options)
-      if options[:v2_mode]
-        header_transformations_v2(header_array, options)
-      else
-        header_transformations_v1(header_array, options)
-      end
-    end
-
-    # ---- V1.x Version: transform the headers that were in the file: ------------------------------------------
-    #
-    def header_transformations_v1(header_array, options)
       header_array.map!{|x| x.gsub(%r/#{options[:quote_char]}/, '')}
       header_array.map!{|x| x.strip} if options[:strip_whitespace]
 
@@ -67,99 +57,7 @@ module SmarterCSV
           header
         end
       end
-
       headers
     end
-
-    # ---- V2.x Version: transform the headers that were in the file: ------------------------------------------
-    #
-    def header_transformations_v2(header_array, options)
-      return header_array if options[:header_transformations].nil? || options[:header_transformations].empty?
-
-      # do the header transformations the user requested:
-      if options[:header_transformations]
-        options[:header_transformations].each do |transformation|
-          if transformation.respond_to?(:call) # this is used when a user-provided Proc is passed in
-            header_array = transformation.call(header_array, options)
-          else
-            case transformation
-            when Symbol # this is used for pre-defined transformations that are defined in the SmarterCSV module
-              header_array = public_send(transformation, header_array, options)
-            when Hash # this is called for hash arguments, e.g. header_transformations
-              trans, args = transformation.first # .first treats the hash first element as an array
-              header_array = apply_transformation(trans, header_array, args, options)
-            when Array # this can be used for passing additional arguments in array form (e.g. into a Proc)
-              trans, *args = transformation
-              header_array = apply_transformation(trans, header_array, args, options)
-            else
-              raise SmarterCSV::IncorrectOption, "Invalid transformation type: #{transformation.class}"
-            end
-          end
-        end
-      end
-
-      header_array
-    end
-
-    def apply_transformation(transformation, header_array, args, options)
-      if transformation.respond_to?(:call)
-        # If transformation is a callable object (like a Proc)
-        transformation.call(header_array, args, options)
-      else
-        # If transformation is a symbol (method name)
-        public_send(transformation, header_array, args, options)
-      end
-    end
-
-    # pre-defined v2 header transformations:
-
-    # these are some pre-defined header transformations which can be used
-    # all these take the headers array as the input
-    #
-    # the computed options can be accessed via @options
-
-    def keys_as_symbols(headers, options)
-      headers.map do |header|
-        header.strip.downcase.gsub(%r{#{options[:quote_char]}}, '').gsub(/(\s|-)+/, '_').to_sym
-      end
-    end
-
-    def keys_as_strings(headers, options)
-      headers.map do |header|
-        header.strip.gsub(%r{#{options[:quote_char]}}, '').downcase.gsub(/(\s|-)+/, '_')
-      end
-    end
-
-    def downcase_headers(headers, _options)
-      headers.map do |header|
-        header.strip.downcase!
-      end
-    end
-
-    def key_mapping(headers, mapping = {}, options)
-      raise(SmarterCSV::IncorrectOption, "ERROR: incorrect format for key_mapping! Expecting hash with from -> to mappings") if mapping.empty? || !mapping.is_a?(Hash)
-
-      headers_set = headers.to_set
-      mapping_keys_set = mapping.keys.to_set
-      silence_keys_set = (options[:silence_missing_keys] || []).to_set
-
-      # Check for missing keys
-      missing_keys = mapping_keys_set - headers_set - silence_keys_set
-      raise SmarterCSV::KeyMappingError, "ERROR: cannot map headers: #{missing_keys.to_a.join(', ')}" if missing_keys.any? && !options[:silence_missing_keys]
-
-      # Apply key mapping, retaining nils for explicitly mapped headers
-      headers.map do |header|
-        if mapping.key?(header)
-          # Maps the key according to the mapping, including nil mapping
-          mapping[header]
-        elsif options[:remove_unmapped_keys]
-          # Remove headers not specified in the mapping
-          nil
-        else
-          # Keep the original header if not specified in the mapping
-          header
-        end
-      end
-    end
   end
 end

data/lib/smarter_csv/header_validations.rb

@@ -3,21 +3,11 @@
 module SmarterCSV
   class << self
     def header_validations(headers, options)
-      if options[:v2_mode]
-        header_validations_v2(headers, options)
-      else
-        header_validations_v1(headers, options)
-      end
-    end
-
-    # ---- V1.x Version: validate the headers -----------------------------------------------------------------
-
-    def header_validations_v1(headers, options)
-      check_duplicate_headers_v1(headers, options)
-      check_required_headers_v1(headers, options)
+      check_duplicate_headers(headers, options)
+      check_required_headers(headers, options)
     end
 
-    def check_duplicate_headers_v1(headers, _options)
+    def check_duplicate_headers(headers, _options)
       header_counts = Hash.new(0)
       headers.each { |header| header_counts[header] += 1 unless header.nil? }
 
@@ -28,109 +18,17 @@ module SmarterCSV
       end
     end
 
-    def check_required_headers_v1(headers, options)
+    require 'set'
+
+    def check_required_headers(headers, options)
       if options[:required_keys] && options[:required_keys].is_a?(Array)
         headers_set = headers.to_set
         missing_keys = options[:required_keys].select { |k| !headers_set.include?(k) }
 
         unless missing_keys.empty?
-          raise SmarterCSV::MissingKeys, "ERROR: missing attributes: #{missing_keys.join(',')}"
+          raise SmarterCSV::MissingKeys, "ERROR: missing attributes: #{missing_keys.join(',')}. Check `SmarterCSV.headers` for original headers."
         end
       end
     end
-
-    # ---- V2.x Version: validate the headers -----------------------------------------------------------------
-
-    # def header_validations_v2(headers, options)
-    #   return unless options[:header_validations]
-
-    #   options[:header_validations].each do |validation|
-    #     if validation.respond_to?(:call)
-    #       # Directly call if it's a Proc or lambda
-    #       validation.call(headers)
-    #     else
-    #       binding.pry
-    #       # Handle Symbol, Hash, or Array
-    #       method_name, args = validation.is_a?(Symbol) ? [validation, []] : validation
-    #       public_send(method_name, headers, *Array(args))
-    #     end
-    #   end
-    # end
-
-    def header_validations_v2(headers, options)
-      return unless options[:header_validations]
-
-      # do the header validations the user requested:
-      # Header validations typically raise errors directly
-      #
-      options[:header_validations].each do |validation|
-        if validation.respond_to?(:call)
-          # Directly call if it's a Proc or lambda
-          validation.call(headers)
-        else
-          case validation
-          when Symbol
-            public_send(validation, headers)
-          when Hash
-            val, args = validation.first
-            public_send(val, headers, args)
-          when Array
-            val, *args = validation
-            public_send(val, headers, args)
-          else
-            raise SmarterCSV::IncorrectOption, "Invalid validation type: #{validation.class}"
-          end
-        end
-      end
-    end
-
-    # def header_validations_v2_orig(headers, options)
-    #   # do the header validations the user requested:
-    #   # Header validations typically raise errors directly
-    #   #
-    #   if options[:header_validations]
-    #     options[:header_validations].each do |validation|
-    #       case validation
-    #       when Symbol
-    #         public_send(validation, headers)
-    #       when Hash
-    #         val, args = validation.first
-    #         public_send(val, headers, args)
-    #       when Array
-    #         val, args = validation
-    #         public_send(val, headers, args)
-    #       else
-    #         validation.call(headers) unless validation.nil?
-    #       end
-    #     end
-    #   end
-    # end
-
-    # these are some pre-defined header validations which can be used
-    # all these take the headers array as the input
-    #
-    # the computed options can be accessed via @options
-
-    def unique_headers(headers)
-      header_counts = Hash.new(0)
-      headers.each { |header| header_counts[header] += 1 unless header.nil? }
-
-      duplicates = header_counts.select { |_, count| count > 1 }
-
-      unless duplicates.empty?
-        raise(SmarterCSV::DuplicateHeaders, "Duplicate Headers in CSV: #{duplicates.inspect}")
-      end
-    end
-
-    def required_headers(headers, required = [])
-      raise(SmarterCSV::IncorrectOption, "ERROR: required_headers validation needs an array argument") unless required.is_a?(Array)
-
-      headers_set = headers.to_set
-      missing = required.select { |r| !headers_set.include?(r) }
-
-      unless missing.empty?
-        raise(SmarterCSV::MissingKeys, "Missing Headers in CSV: #{missing.inspect}")
-      end
-    end
   end
 end

data/lib/smarter_csv/options_processing.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module SmarterCSV
-  COMMON_OPTIONS = {
+  DEFAULT_OPTIONS = {
     acceleration: true,
     auto_row_sep_chars: 500,
     chunk_size: nil,
@@ -15,66 +15,39 @@ module SmarterCSV
     force_utf8: false,
     headers_in_file: true,
     invalid_byte_sequence: '',
-    quote_char: '"',
-    remove_unmapped_keys: false,
-    row_sep: :auto, # was: $/,
-    silence_deprecations: false, # new in 1.11
-    silence_missing_keys: false,
-    skip_lines: nil,
-    user_provided_headers: nil,
-    verbose: false,
-    with_line_numbers: false,
-    v2_mode: false,
-  }.freeze
-
-  V1_DEFAULT_OPTIONS = {
     keep_original_headers: false,
     key_mapping: nil,
+    quote_char: '"',
     remove_empty_hashes: true,
     remove_empty_values: true,
+    remove_unmapped_keys: false,
     remove_values_matching: nil,
     remove_zero_values: false,
     required_headers: nil,
     required_keys: nil,
+    row_sep: :auto, # was: $/,
+    silence_missing_keys: false,
+    skip_lines: nil,
     strings_as_keys: false,
     strip_chars_from_headers: nil,
     strip_whitespace: true,
+    user_provided_headers: nil,
     value_converters: nil,
-    v2_mode: false,
+    verbose: false,
+    with_line_numbers: false,
   }.freeze
 
-  DEPRECATED_OPTIONS = [
-    :convert_values_to_numeric,
-    :downcase_headers,
-    :keep_original_headers,
-    :key_mapping,
-    :remove_empty_hashes,
-    :remove_empty_values,
-    :remove_values_matching,
-    :remove_zero_values,
-    :required_headers,
-    :required_keys,
-    :stirngs_as_keys,
-    :strip_cars_from_headers,
-    :strip_whitespace,
-    :value_converters,
-  ].freeze
-
   class << self
     # NOTE: this is not called when "parse" methods are tested by themselves
     def process_options(given_options = {})
       puts "User provided options:\n#{pp(given_options)}\n" if given_options[:verbose]
 
-      # fix invalid input
-      given_options[:invalid_byte_sequence] = '' if given_options[:invalid_byte_sequence].nil?
-
-      # warn about deprecated options / raises error for v2_mode
-      handle_deprecations(given_options)
+      @options = DEFAULT_OPTIONS.dup.merge!(given_options)
 
-      given_options = preprocess_v2_options(given_options) if given_options[:v2_mode]
+      # fix invalid input
+      @options[:invalid_byte_sequence] ||= ''
 
-      @options = compute_default_options(given_options).merge!(given_options)
-      puts "Computed options:\n#{pp(@options)}\n" if given_options[:verbose]
+      puts "Computed options:\n#{pp(@options)}\n" if @options[:verbose]
 
       validate_options!(@options)
       @options
@@ -84,35 +57,11 @@ module SmarterCSV
     #
     # ONLY FOR BACKWARDS-COMPATIBILITY
     def default_options
-      COMMON_OPTIONS.merge(V1_DEFAULT_OPTIONS)
+      DEFAULT_OPTIONS
     end
 
     private
 
-    def compute_default_options(options = {})
-      return COMMON_OPTIONS.merge(V1_DEFAULT_OPTIONS) unless options[:v2_mode]
-
-      default_options = {}
-      if options[:defaults].to_s != 'none'
-        default_options = COMMON_OPTIONS.dup.merge(V2_DEFAULT_OPTIONS)
-        if options[:defaults].to_s == 'v1'
-          default_options.merge(V1_TRANSFORMATIONS)
-        else
-          default_options.merge(V2_TRANSFORMATIONS)
-        end
-      end
-    end
-
-    def handle_deprecations(options)
-      used_deprecated_options = DEPRECATED_OPTIONS & options.keys
-      message = "SmarterCSV #{VERSION} DEPRECATED OPTIONS: #{pp(used_deprecated_options)}"
-      if options[:v2_mode]
-        raise(SmarterCSV::DeprecatedOptions, "ERROR: #{message}") unless used_deprecated_options.empty? || options[:silence_deprecations]
-      else
-        puts "DEPRECATION WARNING: #{message}" unless used_deprecated_options.empty? || options[:silence_deprecations]
-      end
-    end
-
     def validate_options!(options)
       # deprecate required_headers
       unless options[:required_headers].nil?
@@ -141,57 +90,5 @@ module SmarterCSV
     def pp(value)
       defined?(AwesomePrint) ? value.awesome_inspect(index: nil) : value.inspect
     end
-
-    # ---- V2 code ----------------------------------------------------------------------------------------
-
-    V2_DEFAULT_OPTIONS = {
-      # These need to go to the COMMON_OPTIONS:
-      remove_empty_hashes: true, # this might need a transformation or move to common options
-      # ------------
-      header_transformations: [:keys_as_symbols],
-      header_validations: [:unique_headers],
-      # data_transformations: [:replace_blank_with_nil],
-      # data_validations: [],
-      hash_transformations: [:strip_spaces, :remove_blank_values],
-      hash_validations: [],
-      v2_mode: true,
-    }.freeze
-
-    V2_TRANSFORMATIONS = {
-      header_transformations: [:keys_as_symbols],
-      header_validations: [:unique_headers],
-      # data_transformations: [:replace_blank_with_nil],
-      # data_validations: [],
-      hash_transformations: [:v1_backwards_compatibility],
-      # hash_transformations: [:remove_empty_keys, :strip_spaces, :remove_blank_values, :convert_values_to_numeric], # ??? :convert_values_to_numeric]
-      hash_validations: [],
-    }.freeze
-
-    V1_TRANSFORMATIONS = {
-      header_transformations: [:keys_as_symbols],
-      header_validations: [:unique_headers],
-      # data_transformations: [:replace_blank_with_nil],
-      # data_validations: [],
-      hash_transformations: [:strip_spaces, :remove_blank_values, :convert_values_to_numeric],
-      hash_validations: [],
-    }.freeze
-
-    def preprocess_v2_options(options)
-      return options unless options[:v2_mode] || options[:header_transformations]
-
-      # We want to provide safe defaults for easy processing, that is why we have a special keyword :none
-      # to not do any header transformations..
-      #
-      # this is why we need to remove the 'none' here:
-      #
-      requested_header_transformations = options[:header_transformations]
-      if requested_header_transformations.to_s == 'none'
-        requested_header_transformations = []
-      else
-        requested_header_transformations = requested_header_transformations.reject {|x| x.to_s == 'none'} unless requested_header_transformations.nil?
-      end
-      options[:header_transformations] = requested_header_transformations || []
-      options
-    end
   end
 end

data/lib/smarter_csv/smarter_csv.rb

@@ -1,16 +1,6 @@
 # frozen_string_literal: true
 
 module SmarterCSV
-  class SmarterCSVException < StandardError; end
-  class DeprecatedOptions < SmarterCSVException; end
-  class HeaderSizeMismatch < SmarterCSVException; end
-  class IncorrectOption < SmarterCSVException; end
-  class ValidationError < SmarterCSVException; end
-  class DuplicateHeaders < SmarterCSVException; end
-  class MissingKeys < SmarterCSVException; end # previously known as MissingHeaders
-  class NoColSepDetected < SmarterCSVException; end
-  class KeyMappingError < SmarterCSVException; end
-
   # first parameter: filename or input object which responds to readline method
   def SmarterCSV.process(input, given_options = {}, &block) # rubocop:disable Lint/UnusedMethodArgument
     initialize_variables
@@ -109,10 +99,6 @@ module SmarterCSV
 
         next if options[:remove_empty_hashes] && hash.empty?
 
-        #
-        # should HASH VALIDATIONS go here instead?
-        #
-
         puts "CSV Line #{@file_line_count}: #{pp(hash)}" if @verbose == '2' # very verbose setting
         # optional adding of csv_line_number to the hash to help debugging
         hash[:csv_line_number] = @csv_line_count if options[:with_line_numbers]
@@ -170,19 +156,22 @@ module SmarterCSV
   end
 
   class << self
-    # Counts the number of quote characters in a line, excluding escaped quotes.
-    # FYI: using Ruby built-in regex processing to determine the number of quotes
     def count_quote_chars(line, quote_char)
       return 0 if line.nil? || quote_char.nil? || quote_char.empty?
 
-      # Escaped quote character (e.g., if quote_char is ", then escaped is \")
-      escaped_quote = Regexp.escape(quote_char)
+      count = 0
+      escaped = false
 
-      # Pattern to match a quote character not preceded by a backslash
-      pattern = /(?<!\\)(?:\\\\)*#{escaped_quote}/
+      line.each_char do |char|
+        if char == '\\' && !escaped
+          escaped = true
+        else
+          count += 1 if char == quote_char && !escaped
+          escaped = false
+        end
+      end
 
-      # Count occurrences
-      line.scan(pattern).count
+      count
     end
 
     def has_acceleration?

data/lib/smarter_csv/variables.rb

@@ -15,7 +15,6 @@ module SmarterCSV
       @raw_header = nil # header as it appears in the file
       @result = []
       @warnings = {}
-      @v2_mode = false
       @enforce_utf8 = false # only set to true if needed (after options parsing)
     end
 

data/lib/smarter_csv/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SmarterCSV
-  VERSION = "1.11.0.pre2"
+  VERSION = "1.11.2"
 end

data/lib/smarter_csv/writer.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+module SmarterCSV
+  #
+  # Generate CSV files
+  #
+  # Create an instance of the Writer class with the filename and options.
+  # call `<<` one or mulltiple times to append data to the file.
+  # call `finalize` to save the file.
+  #
+  # The `<<` method can take different arguments:
+  #  * a signle Hash
+  #  * an array of Hashes
+  #  * nested arrays of arrays of Hashes
+  #
+  # By default SmarterCSV::Writer automatically discovers all headers that are present
+  # in the data on-the-fly. This can be disabled, then only given headers are used.
+  # Disabling can be useful when you want to select attributes from hashes, or ActiveRecord instances.
+  #
+  # If `discover_headers` is enabled, and headers are given, any new headers that are found in the data will still be appended.
+  #
+  # The Writer automatically quotes fields containing the col_sep, row_sep, or the quote_char.
+  #
+  # Options:
+  #   col_sep : defaults to , but can be set to any other character
+  #   row_sep : defaults to LF \n , but can be set to \r\n or \r or anything else
+  #   quote_char : defaults to "
+  #   discover_headers : defaults to true
+  #   headers : defaults to []
+  #   force_quotes: defaults to false
+  #   map_headers: defaults to {}, can be a hash of key -> value mappings
+
+  # IMPORTANT NOTES:
+  #  * Data hashes could contain strings or symbols as keys.
+  #    Make sure to use the correct form when specifying headers manually,
+  #    in combination with the :discover_headers option
+
+  attr_reader :options, :row_sep, :col_sep, :quote_char, :force_quotes, :discover_headers, :headers, :map_headers, :output_file
+
+  class Writer
+    def initialize(file_path, options = {})
+      @options = options
+      @row_sep = options[:row_sep] || "\n" # RFC4180 "\r\n"
+      @col_sep = options[:col_sep] || ','
+      @quote_char = options[:quote_char] || '"'
+      @force_quotes = options[:force_quotes] == true
+      @discover_headers = true # defaults to true
+      if options.has_key?(:discover_headers)
+        # passing in the option overrides the default behavior
+        @discover_headers = options[:discover_headers] == true
+      else
+        # disable discover_headers when headers are given explicitly
+        @discover_headers = !(options.has_key?(:map_headers) || options.has_key?(:headers))
+      end
+      @headers = [] # start with empty headers
+      @headers = options[:headers] if options.has_key?(:headers) # unless explicitly given
+      @headers = options[:map_headers].keys if options.has_key?(:map_headers) && !options.has_key?(:headers)
+      @map_headers = options[:map_headers] || {}
+
+      @output_file = File.open(file_path, 'w+')
+      # hidden state:
+      @temp_file = Tempfile.new('tempfile', '/tmp')
+      @quote_regex = Regexp.union(@col_sep, @row_sep, @quote_char)
+    end
+
+    # this can be called many times in order to append lines to the csv file
+    def <<(data)
+      case data
+      when Hash
+        process_hash(data)
+      when Array
+        data.each { |item| self << item }
+      when NilClass
+        # ignore
+      else
+        raise InvalidInputData, "Invalid data type: #{data.class}. Must be a Hash or an Array."
+      end
+    end
+
+    def finalize
+      # Map headers if :map_headers option is provided
+      mapped_headers = @headers.map { |header| @map_headers[header] || header }
+
+      @temp_file.rewind
+      @output_file.write(mapped_headers.join(@col_sep) + @row_sep)
+      @output_file.write(@temp_file.read)
+      @output_file.flush
+      @output_file.close
+      @temp_file.delete
+    end
+
+    private
+
+    def process_hash(hash)
+      if @discover_headers
+        hash_keys = hash.keys
+        new_keys = hash_keys - @headers
+        @headers.concat(new_keys)
+      end
+
+      # Reorder the hash to match the current headers order and fill missing fields
+      ordered_row = @headers.map { |header| hash[header] || '' }
+
+      @temp_file.write ordered_row.map { |value| escape_csv_field(value) }.join(@col_sep) + @row_sep
+    end
+
+    def escape_csv_field(field)
+      if @force_quotes || field.to_s.match(@quote_regex)
+        "\"#{field}\""
+      else
+        field.to_s
+      end
+    end
+  end
+end

data/lib/smarter_csv.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-require 'set'
-
 require "smarter_csv/version"
+require "smarter_csv/errors"
+
 require "smarter_csv/file_io"
 require "smarter_csv/options_processing"
 require "smarter_csv/auto_detection"
@@ -11,7 +11,10 @@ require 'smarter_csv/header_transformations'
 require 'smarter_csv/header_validations'
 require "smarter_csv/headers"
 require "smarter_csv/hash_transformations"
+
 require "smarter_csv/parse"
+require "smarter_csv/writer"
+require "smarter_csv/smarter_csv"
 
 # load the C-extension:
 case RUBY_ENGINE
@@ -50,4 +53,24 @@ else
   BLOCK_COMMENT
 end
 # :nocov:
-require "smarter_csv/smarter_csv"
+
+module SmarterCSV
+  # SmarterCSV.generate(filename, options) do |csv_writer|
+  #   MyModel.find_in_batches(batch_size: 100) do |batch|
+  #    batch.pluck(:name, :description, :instructor).each do |record|
+  #       csv_writer << record
+  #     end
+  #   end
+  # end
+  #
+  # rubocop:disable Lint/UnusedMethodArgument
+  def self.generate(filename, options = {}, &block)
+    raise unless block_given?
+
+    writer = Writer.new(filename, options)
+    yield writer
+  ensure
+    writer.finalize
+  end
+  # rubocop:enable Lint/UnusedMethodArgument
+end

data/smarter_csv.gemspec

@@ -9,8 +9,8 @@ Gem::Specification.new do |spec|
   spec.authors       = ["Tilo Sloboda"]
   spec.email         = ["tilo.sloboda@gmail.com"]
 
-  spec.summary       = "Ruby Gem for smarter importing of CSV Files (and CSV-like files), with lots of optional features, e.g. chunked processing for huge CSV files"
-  spec.description   = "Ruby Gem for smarter importing of CSV Files as Array(s) of Hashes, with optional features for processing large files in parallel, embedded comments, unusual field- and record-separators, flexible mapping of CSV-headers to Hash-keys"
+  spec.summary       = "CSV Reading and Writing"
+  spec.description   = "Ruby Gem for convenient reading and writing: importing of CSV Files as Array(s) of Hashes, with lots of features for processing large files in parallel, embedded comments, unusual field- and record-separators, flexible mapping of CSV-headers to Hash-keys"
   spec.homepage      = "https://github.com/tilo/smarter_csv"
   spec.license       = 'MIT'
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: smarter_csv
 version: !ruby/object:Gem::Version
-  version: 1.11.0.pre2
+  version: 1.11.2
 platform: ruby
 authors:
 - Tilo Sloboda
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-01-14 00:00:00.000000000 Z
+date: 2024-07-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: awesome_print
@@ -94,9 +94,10 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: Ruby Gem for smarter importing of CSV Files as Array(s) of Hashes, with
-  optional features for processing large files in parallel, embedded comments, unusual
-  field- and record-separators, flexible mapping of CSV-headers to Hash-keys
+description: 'Ruby Gem for convenient reading and writing: importing of CSV Files
+  as Array(s) of Hashes, with lots of features for processing large files in parallel,
+  embedded comments, unusual field- and record-separators, flexible mapping of CSV-headers
+  to Hash-keys'
 email:
 - tilo.sloboda@gmail.com
 executables: []
@@ -104,6 +105,7 @@ extensions:
 - ext/smarter_csv/extconf.rb
 extra_rdoc_files: []
 files:
+- ".rspec"
 - ".rubocop.yml"
 - ".rvmrc"
 - CHANGELOG.md
@@ -117,6 +119,7 @@ files:
 - ext/smarter_csv/smarter_csv.c
 - lib/smarter_csv.rb
 - lib/smarter_csv/auto_detection.rb
+- lib/smarter_csv/errors.rb
 - lib/smarter_csv/file_io.rb
 - lib/smarter_csv/hash_transformations.rb
 - lib/smarter_csv/header_transformations.rb
@@ -127,6 +130,7 @@ files:
 - lib/smarter_csv/smarter_csv.rb
 - lib/smarter_csv/variables.rb
 - lib/smarter_csv/version.rb
+- lib/smarter_csv/writer.rb
 - smarter_csv.gemspec
 homepage: https://github.com/tilo/smarter_csv
 licenses:
@@ -147,13 +151,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 2.5.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubygems_version: 3.2.3
 signing_key:
 specification_version: 4
-summary: Ruby Gem for smarter importing of CSV Files (and CSV-like files), with lots
-  of optional features, e.g. chunked processing for huge CSV files
+summary: CSV Reading and Writing
 test_files: []