smarter_csv 1.10.3 → 1.11.0.pre1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dbc3d3e97a602c4b67a88a94eea6bb66feed23c061da6a1c07b04f84b820e8b0
-  data.tar.gz: be5eea03fa0baeec95557772aef9d2bfd6c3e200103e9b686078c05c392a0a74
+  metadata.gz: f4f709c1e38fd4e755802a17103231bd17ba58f54ce2d3fd9f8ad57222704114
+  data.tar.gz: 3ac00625a1478793b9cf9272862e448de4ca80784db71ebce3060d0ae34623ad
 SHA512:
-  metadata.gz: bb204a933e3f8ae93b626cd6c4c45e6296ece98f136b96389ae3d3459bb8826de3d128481faedb3e10fa85a547483568cce7fbb54de38ba6c8074b0276775123
-  data.tar.gz: 724279536101f6561299ffc619c12393395c019ed771cc1e7b37b376804a05bee22b179fbd30770f29f6a9c099864268ecb222a2af7497a254b43982942aec90
+  metadata.gz: dec4c77b2b4019788ff576e0d58c913e4eb5d284f7c06bc95554dc3a83985a6a5ce38fbd3fbe851d7a5944ea53621bab7dc70d270b0fdf786624d7b32a631395
+  data.tar.gz: 3b9cdea9f632698d78c749196c225726ff37bc807a0f10ca6b9f1df5c9e4cd5ad15415c81042277461bd1a2449f1f07e24ecebb1b2e1dcff63759c26cc92f67f

data/.rubocop.yml

@@ -88,12 +88,18 @@ Style/IfInsideElse:
 Style/IfUnlessModifier:
   Enabled: false
 
+Style/InverseMethods:
+  Enabled: false
+
 Style/NestedTernaryOperator:
   Enabled: false
 
 Style/PreferredHashMethods:
   Enabled: false
 
+Style/Proc:
+  Enabled: false
+
 Style/NumericPredicate:
   Enabled: false
 
@@ -129,6 +135,9 @@ Style/SymbolProc: # old Ruby versions can't do this
 Style/TrailingCommaInHashLiteral:
   Enabled: false
 
+Style/TrailingCommaInArrayLiteral:
+  Enabled: false
+
 Style/TrailingUnderscoreVariable:
   Enabled: false
 
@@ -138,6 +147,9 @@ Style/TrivialAccessors:
 # Style/UnlessModifier:
 #   Enabled: false
 
+Style/WordArray:
+  Enabled: false
+
 Style/ZeroLengthPredicate:
   Enabled: false
 

data/CHANGELOG.md

@@ -1,13 +1,42 @@
 
 # SmarterCSV 1.x Change Log
 
-## 1.10.3 (2024-03-10)
-  * fixed issue when frozen options are handed in (thanks to Daniel Pepper)
-  * cleaned-up rspec tests (thanks to Daniel Pepper)
-  * fixed link in README (issue #251)
+## T.B.D.
 
-## 1.10.2 (2024-02-11)
-  * improve error message for missing keys
+  * code refactor
+  
+  * NEW BEHAVIOR:
+    - hidden `:v2_mode` options (incomplete!)
+    - pre-processing for v2 options
+    - implemented v2 `:header_transformations` (DO NOT USE YET!)
+      + -> check if all v1 transformations are correctly done
+        How are we going to 
+        * disambiguate headers?
+
+
+        * do key_mapping? -> seems to work
+          - remove_unmapped_keys  ? 
+          - silence missing keys ... a missing mapped key should raise an exception, except when silenced
+          - required_keys needs to be a header-validation
+
+
+        * keep original headers? -> :none
+        * do strings_as_* ? -> either :keys_as_symbols, :keys_as_strings
+        * remove quote_chars? -> included in keys_as_*
+        * strip whitespace? -> included in keys_as_*
+
+        TODO:
+        
+        - add tests for header_validations
+        
+        - modify options to handle v1 and v2 options
+        - add v1 defaults in v2 processing
+        - add tests for all options processing
+        - 100% backwards compatibility when working in v1 mode
+
+
+## 1.10.1 (2024-01-07)
+  * fix incorrect warning about UTF-8 (issue #268, thanks hirowatari)
 
 ## 1.10.1 (2024-01-07)
   * fix incorrect warning about UTF-8 (issue #268, thanks hirowatari)

data/CONTRIBUTORS.md

@@ -51,5 +51,4 @@ A Big Thank you to everyone who filed issues, sent comments, and who contributed
  * [Rahul Chaudhary](https://github.com/rahulch95)
  * [Alessandro Fazzi](https://github.com/pioneerskies)
  * [JP Camara](https://github.com/jpcamara)
- * [Kenton Hirowatari](https://github.com/hirowatari)
- * [Daniel Pepper](https://github.com/dpep)
+ * [Hiro Watari](https://github.com/hirowatari)

data/README.md

@@ -23,8 +23,13 @@
 
 * default branch is `main` for 1.x development
   
-* 2.x development is [MOVED TO THIS PR](https://github.com/tilo/smarter_csv/pull/267)
-  - 2.x behavior is still EXPERIMENTAL - DO NOT USE in production
+* 2.x development is on `2.0-development` (check this branch for 2.0 documentation) 
+  - This is an EXPERIMENTAL branch - DO NOT USE in production
+
+#### Work towards Future Version 2.x
+
+* Work towards SmarterCSV 2.x is still ongoing, with improved features, and more streamlined options, but consider it as experimental at this time.
+  Please check the [2.0-develop branch](https://github.com/tilo/smarter_csv/tree/2.0-develop), open any issues and pull requests with mention of tag v2.0.
 
 ---------------
 
@@ -389,9 +394,10 @@ And header and data validations will also be supported in 2.x
  * some CSV files use un-escaped quotation characters inside fields. This can cause the import to break. To get around this, use the `:force_simple_split => true` option in combination with `:strip_chars_from_headers => /[\-"]/` . This will also significantly speed up the import.
    If you would force a different :quote_char instead (setting it to a non-used character), then the import would be up to 5-times slower than using `:force_simple_split`.
 
-## The original post that started SmarterCSV:
+## See also:
+
+  http://www.unixgods.org/~tilo/Ruby/process_csv_as_hashes.html
 
-  http://www.unixgods.org/Ruby/process_csv_as_hashes.html
 
 
 ## Installation

data/lib/smarter_csv/hash_transformations.rb

@@ -2,7 +2,16 @@
 
 module SmarterCSV
   class << self
+    # this is processing the headers from the input file
     def hash_transformations(hash, options)
+      if options[:v2_mode]
+        hash_transformations_v2(hash, options)
+      else
+        hash_transformations_v1(hash, options)
+      end
+    end
+
+    def hash_transformations_v1(hash, options)
       # there may be unmapped keys, or keys purposedly mapped to nil or an empty key..
       # make sure we delete any key/value pairs from the hash, which the user wanted to delete:
       remove_empty_values = options[:remove_empty_values] == true
@@ -33,46 +42,117 @@ module SmarterCSV
       end
     end
 
-    # def hash_transformations(hash, options)
-    #   # there may be unmapped keys, or keys purposedly mapped to nil or an empty key..
-    #   # make sure we delete any key/value pairs from the hash, which the user wanted to delete:
-    #   hash.delete(nil)
-    #   hash.delete('')
-    #   hash.delete(:"")
-
-    #   if options[:remove_empty_values] == true
-    #     hash.delete_if{|_k, v| has_rails ? v.blank? : blank?(v)}
-    #   end
-
-    #   hash.delete_if{|_k, v| !v.nil? && v =~ /^(0+|0+\.0+)$/} if options[:remove_zero_values] # values are Strings
-    #   hash.delete_if{|_k, v| v =~ options[:remove_values_matching]} if options[:remove_values_matching]
-
-    #   if options[:convert_values_to_numeric]
-    #     hash.each do |k, v|
-    #       # deal with the :only / :except options to :convert_values_to_numeric
-    #       next if limit_execution_for_only_or_except(options, :convert_values_to_numeric, k)
-
-    #       # convert if it's a numeric value:
-    #       case v
-    #       when /^[+-]?\d+\.\d+$/
-    #         hash[k] = v.to_f
-    #       when /^[+-]?\d+$/
-    #         hash[k] = v.to_i
-    #       end
-    #     end
-    #   end
-
-    #   if options[:value_converters]
-    #     hash.each do |k, v|
-    #       converter = options[:value_converters][k]
-    #       next unless converter
-
-    #       hash[k] = converter.convert(v)
-    #     end
-    #   end
-
-    #   hash
-    # end
+    def hash_transformations_v2(hash, options)
+      return hash if options[:hash_transformations].nil? || options[:hash_transformations].empty?
+
+      # do the header transformations the user requested:
+      if options[:hash_transformations]
+        options[:hash_transformations].each do |transformation|
+          if transformation.respond_to?(:call) # this is used when a user-provided Proc is passed in
+            hash = transformation.call(hash, options)
+          else
+            case transformation
+            when Symbol # this is used for pre-defined transformations that are defined in the SmarterCSV module
+              hash = public_send(transformation, hash, options)
+            when Hash # this is called for hash arguments, e.g. hash_transformations
+              trans, args = transformation.first # .first treats the hash first element as an array
+              hash = apply_transformation(trans, hash, args, options)
+            when Array # this can be used for passing additional arguments in array form (e.g. into a Proc)
+              trans, *args = transformation
+              hash = apply_transformation(trans, hash, args, options)
+            else
+              raise SmarterCSV::IncorrectOption, "Invalid transformation type: #{transformation.class}"
+            end
+          end
+        end
+      end
+
+      hash
+    end
+
+    #
+    # To handle v1-backward-compatible behavior, it is faster to roll all behavior into one method
+    #
+    def v1_backwards_compatibility(hash, options)
+      hash.each_with_object({}) do |(k, v), new_hash|
+        next if k.nil? || k == '' || k == :"" # remove_empty_keys
+        next if has_rails ? v.blank? : blank?(v) # remove_empty_values
+
+        # convert_values_to_numeric:
+        # deal with the :only / :except options to :convert_values_to_numeric
+        unless limit_execution_for_only_or_except(options, :convert_values_to_numeric, k)
+          if v =~ /^[+-]?\d+\.\d+$/
+            v = v.to_f
+          elsif v =~ /^[+-]?\d+$/
+            v = v.to_i
+          end
+        end
+
+        new_hash[k] = v
+      end
+    end
+
+    #
+    # Building Blocks in case you want to build your own flow:
+    #
+
+    def value_converters(hash, _options)
+      #
+      # TO BE IMPLEMENTED
+      #
+    end
+
+    def strip_spaces(hash, _options)
+      hash.each_key {|key| hash[key].strip! unless hash[key].nil? } # &. syntax was introduced in Ruby 2.3 - need to stay backwards compatible
+    end
+
+    def remove_blank_values(hash, _options)
+      hash.each_key {|key| hash.delete(key) if hash[key].nil? || hash[key].is_a?(String) && hash[key] !~ /[^[:space:]]/ }
+    end
+
+    def remove_zero_values(hash, _options)
+      hash.each_key {|key| hash.delete(key) if hash[key].is_a?(Numeric) && hash[key].zero? }
+    end
+
+    def remove_empty_keys(hash, _options)
+      hash.reject!{|key, _v| key.nil? || key.empty?}
+    end
+
+    def convert_values_to_numeric(hash, _options)
+      hash.each_key do |k|
+        case hash[k]
+        when /^[+-]?\d+\.\d+$/
+          hash[k] = hash[k].to_f
+        when /^[+-]?\d+$/
+          hash[k] = hash[k].to_i
+        end
+      end
+    end
+
+    def convert_values_to_numeric_unless_leading_zeroes(hash, _options)
+      hash.each_key do |k|
+        case hash[k]
+        when /^[+-]?[1-9]\d*\.\d+$/
+          hash[k] = hash[k].to_f
+        when /^[+-]?[1-9]\d*$/
+          hash[k] = hash[k].to_i
+        end
+      end
+    end
+
+    # IMPORTANT NOTE:
+    # this can lead to cases where a nil or empty value gets converted into 0 or 0.0,
+    # and can then not be properly removed!
+    #
+    # you should first try to use convert_values_to_numeric or convert_values_to_numeric_unless_leading_zeroes
+    #
+    def convert_to_integer(hash, _options)
+      hash.each_key {|key| hash[key] = hash[key].to_i }
+    end
+
+    def convert_to_float(hash, _options)
+      hash.each_key {|key| hash[key] = hash[key].to_f }
+    end
 
     protected
 

data/lib/smarter_csv/header_transformations.rb

@@ -2,8 +2,18 @@
 
 module SmarterCSV
   class << self
-    # transform the headers that were in the file:
+    # this is processing the headers from the input file
     def header_transformations(header_array, options)
+      if options[:v2_mode]
+        header_transformations_v2(header_array, options)
+      else
+        header_transformations_v1(header_array, options)
+      end
+    end
+
+    # ---- V1.x Version: transform the headers that were in the file: ------------------------------------------
+    #
+    def header_transformations_v1(header_array, options)
       header_array.map!{|x| x.gsub(%r/#{options[:quote_char]}/, '')}
       header_array.map!{|x| x.strip} if options[:strip_whitespace]
 
@@ -57,7 +67,99 @@ module SmarterCSV
           header
         end
       end
+
       headers
     end
+
+    # ---- V2.x Version: transform the headers that were in the file: ------------------------------------------
+    #
+    def header_transformations_v2(header_array, options)
+      return header_array if options[:header_transformations].nil? || options[:header_transformations].empty?
+
+      # do the header transformations the user requested:
+      if options[:header_transformations]
+        options[:header_transformations].each do |transformation|
+          if transformation.respond_to?(:call) # this is used when a user-provided Proc is passed in
+            header_array = transformation.call(header_array, options)
+          else
+            case transformation
+            when Symbol # this is used for pre-defined transformations that are defined in the SmarterCSV module
+              header_array = public_send(transformation, header_array, options)
+            when Hash # this is called for hash arguments, e.g. header_transformations
+              trans, args = transformation.first # .first treats the hash first element as an array
+              header_array = apply_transformation(trans, header_array, args, options)
+            when Array # this can be used for passing additional arguments in array form (e.g. into a Proc)
+              trans, *args = transformation
+              header_array = apply_transformation(trans, header_array, args, options)
+            else
+              raise SmarterCSV::IncorrectOption, "Invalid transformation type: #{transformation.class}"
+            end
+          end
+        end
+      end
+
+      header_array
+    end
+
+    def apply_transformation(transformation, header_array, args, options)
+      if transformation.respond_to?(:call)
+        # If transformation is a callable object (like a Proc)
+        transformation.call(header_array, args, options)
+      else
+        # If transformation is a symbol (method name)
+        public_send(transformation, header_array, args, options)
+      end
+    end
+
+    # pre-defined v2 header transformations:
+
+    # these are some pre-defined header transformations which can be used
+    # all these take the headers array as the input
+    #
+    # the computed options can be accessed via @options
+
+    def keys_as_symbols(headers, options)
+      headers.map do |header|
+        header.strip.downcase.gsub(%r{#{options[:quote_char]}}, '').gsub(/(\s|-)+/, '_').to_sym
+      end
+    end
+
+    def keys_as_strings(headers, options)
+      headers.map do |header|
+        header.strip.gsub(%r{#{options[:quote_char]}}, '').downcase.gsub(/(\s|-)+/, '_')
+      end
+    end
+
+    def downcase_headers(headers, _options)
+      headers.map do |header|
+        header.strip.downcase!
+      end
+    end
+
+    def key_mapping(headers, mapping = {}, options)
+      raise(SmarterCSV::IncorrectOption, "ERROR: incorrect format for key_mapping! Expecting hash with from -> to mappings") if mapping.empty? || !mapping.is_a?(Hash)
+
+      headers_set = headers.to_set
+      mapping_keys_set = mapping.keys.to_set
+      silence_keys_set = (options[:silence_missing_keys] || []).to_set
+
+      # Check for missing keys
+      missing_keys = mapping_keys_set - headers_set - silence_keys_set
+      raise SmarterCSV::KeyMappingError, "ERROR: cannot map headers: #{missing_keys.to_a.join(', ')}" if missing_keys.any? && !options[:silence_missing_keys]
+
+      # Apply key mapping, retaining nils for explicitly mapped headers
+      headers.map do |header|
+        if mapping.key?(header)
+          # Maps the key according to the mapping, including nil mapping
+          mapping[header]
+        elsif options[:remove_unmapped_keys]
+          # Remove headers not specified in the mapping
+          nil
+        else
+          # Keep the original header if not specified in the mapping
+          header
+        end
+      end
+    end
   end
 end

data/lib/smarter_csv/header_validations.rb

@@ -3,11 +3,21 @@
 module SmarterCSV
   class << self
     def header_validations(headers, options)
-      check_duplicate_headers(headers, options)
-      check_required_headers(headers, options)
+      if options[:v2_mode]
+        header_validations_v2(headers, options)
+      else
+        header_validations_v1(headers, options)
+      end
+    end
+
+    # ---- V1.x Version: validate the headers -----------------------------------------------------------------
+
+    def header_validations_v1(headers, options)
+      check_duplicate_headers_v1(headers, options)
+      check_required_headers_v1(headers, options)
     end
 
-    def check_duplicate_headers(headers, _options)
+    def check_duplicate_headers_v1(headers, _options)
       header_counts = Hash.new(0)
       headers.each { |header| header_counts[header] += 1 unless header.nil? }
 
@@ -18,17 +28,109 @@ module SmarterCSV
       end
     end
 
-    require 'set'
-
-    def check_required_headers(headers, options)
+    def check_required_headers_v1(headers, options)
       if options[:required_keys] && options[:required_keys].is_a?(Array)
         headers_set = headers.to_set
         missing_keys = options[:required_keys].select { |k| !headers_set.include?(k) }
 
         unless missing_keys.empty?
-          raise SmarterCSV::MissingKeys, "ERROR: missing attributes: #{missing_keys.join(',')}. Check `SmarterCSV.headers` for original headers."
+          raise SmarterCSV::MissingKeys, "ERROR: missing attributes: #{missing_keys.join(',')}"
         end
       end
     end
+
+    # ---- V2.x Version: validate the headers -----------------------------------------------------------------
+
+    # def header_validations_v2(headers, options)
+    #   return unless options[:header_validations]
+
+    #   options[:header_validations].each do |validation|
+    #     if validation.respond_to?(:call)
+    #       # Directly call if it's a Proc or lambda
+    #       validation.call(headers)
+    #     else
+    #       binding.pry
+    #       # Handle Symbol, Hash, or Array
+    #       method_name, args = validation.is_a?(Symbol) ? [validation, []] : validation
+    #       public_send(method_name, headers, *Array(args))
+    #     end
+    #   end
+    # end
+
+    def header_validations_v2(headers, options)
+      return unless options[:header_validations]
+
+      # do the header validations the user requested:
+      # Header validations typically raise errors directly
+      #
+      options[:header_validations].each do |validation|
+        if validation.respond_to?(:call)
+          # Directly call if it's a Proc or lambda
+          validation.call(headers)
+        else
+          case validation
+          when Symbol
+            public_send(validation, headers)
+          when Hash
+            val, args = validation.first
+            public_send(val, headers, args)
+          when Array
+            val, *args = validation
+            public_send(val, headers, args)
+          else
+            raise SmarterCSV::IncorrectOption, "Invalid validation type: #{validation.class}"
+          end
+        end
+      end
+    end
+
+    # def header_validations_v2_orig(headers, options)
+    #   # do the header validations the user requested:
+    #   # Header validations typically raise errors directly
+    #   #
+    #   if options[:header_validations]
+    #     options[:header_validations].each do |validation|
+    #       case validation
+    #       when Symbol
+    #         public_send(validation, headers)
+    #       when Hash
+    #         val, args = validation.first
+    #         public_send(val, headers, args)
+    #       when Array
+    #         val, args = validation
+    #         public_send(val, headers, args)
+    #       else
+    #         validation.call(headers) unless validation.nil?
+    #       end
+    #     end
+    #   end
+    # end
+
+    # these are some pre-defined header validations which can be used
+    # all these take the headers array as the input
+    #
+    # the computed options can be accessed via @options
+
+    def unique_headers(headers)
+      header_counts = Hash.new(0)
+      headers.each { |header| header_counts[header] += 1 unless header.nil? }
+
+      duplicates = header_counts.select { |_, count| count > 1 }
+
+      unless duplicates.empty?
+        raise(SmarterCSV::DuplicateHeaders, "Duplicate Headers in CSV: #{duplicates.inspect}")
+      end
+    end
+
+    def required_headers(headers, required = [])
+      raise(SmarterCSV::IncorrectOption, "ERROR: required_headers validation needs an array argument") unless required.is_a?(Array)
+
+      headers_set = headers.to_set
+      missing = required.select { |r| !headers_set.include?(r) }
+
+      unless missing.empty?
+        raise(SmarterCSV::MissingKeys, "Missing Headers in CSV: #{missing.inspect}")
+      end
+    end
   end
 end

data/lib/smarter_csv/options_processing.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module SmarterCSV
-  DEFAULT_OPTIONS = {
+  COMMON_OPTIONS = {
     acceleration: true,
     auto_row_sep_chars: 500,
     chunk_size: nil,
@@ -15,39 +15,66 @@ module SmarterCSV
     force_utf8: false,
     headers_in_file: true,
     invalid_byte_sequence: '',
+    quote_char: '"',
+    remove_unmapped_keys: false,
+    row_sep: :auto, # was: $/,
+    silence_deprecations: false, # new in 1.11
+    silence_missing_keys: false,
+    skip_lines: nil,
+    user_provided_headers: nil,
+    verbose: false,
+    with_line_numbers: false,
+    v2_mode: false,
+  }.freeze
+
+  V1_DEFAULT_OPTIONS = {
     keep_original_headers: false,
     key_mapping: nil,
-    quote_char: '"',
     remove_empty_hashes: true,
     remove_empty_values: true,
-    remove_unmapped_keys: false,
     remove_values_matching: nil,
     remove_zero_values: false,
     required_headers: nil,
     required_keys: nil,
-    row_sep: :auto, # was: $/,
-    silence_missing_keys: false,
-    skip_lines: nil,
     strings_as_keys: false,
     strip_chars_from_headers: nil,
     strip_whitespace: true,
-    user_provided_headers: nil,
     value_converters: nil,
-    verbose: false,
-    with_line_numbers: false,
+    v2_mode: false,
   }.freeze
 
+  DEPRECATED_OPTIONS = [
+    :convert_values_to_numeric,
+    :downcase_headers,
+    :keep_original_headers,
+    :key_mapping,
+    :remove_empty_hashes,
+    :remove_empty_values,
+    :remove_values_matching,
+    :remove_zero_values,
+    :required_headers,
+    :required_keys,
+    :stirngs_as_keys,
+    :strip_cars_from_headers,
+    :strip_whitespace,
+    :value_converters,
+  ].freeze
+
   class << self
     # NOTE: this is not called when "parse" methods are tested by themselves
     def process_options(given_options = {})
       puts "User provided options:\n#{pp(given_options)}\n" if given_options[:verbose]
 
-      @options = DEFAULT_OPTIONS.dup.merge!(given_options)
-
       # fix invalid input
-      @options[:invalid_byte_sequence] ||= ''
+      given_options[:invalid_byte_sequence] = '' if given_options[:invalid_byte_sequence].nil?
+
+      # warn about deprecated options / raises error for v2_mode
+      handle_deprecations(given_options)
+
+      given_options = preprocess_v2_options(given_options) if given_options[:v2_mode]
 
-      puts "Computed options:\n#{pp(@options)}\n" if @options[:verbose]
+      @options = compute_default_options(given_options).merge!(given_options)
+      puts "Computed options:\n#{pp(@options)}\n" if given_options[:verbose]
 
       validate_options!(@options)
       @options
@@ -57,11 +84,35 @@ module SmarterCSV
     #
     # ONLY FOR BACKWARDS-COMPATIBILITY
     def default_options
-      DEFAULT_OPTIONS
+      COMMON_OPTIONS.merge(V1_DEFAULT_OPTIONS)
     end
 
     private
 
+    def compute_default_options(options = {})
+      return COMMON_OPTIONS.merge(V1_DEFAULT_OPTIONS) unless options[:v2_mode]
+
+      default_options = {}
+      if options[:defaults].to_s != 'none'
+        default_options = COMMON_OPTIONS.dup.merge(V2_DEFAULT_OPTIONS)
+        if options[:defaults].to_s == 'v1'
+          default_options.merge(V1_TRANSFORMATIONS)
+        else
+          default_options.merge(V2_TRANSFORMATIONS)
+        end
+      end
+    end
+
+    def handle_deprecations(options)
+      used_deprecated_options = DEPRECATED_OPTIONS & options.keys
+      message = "SmarterCSV #{VERSION} DEPRECATED OPTIONS: #{pp(used_deprecated_options)}"
+      if options[:v2_mode]
+        raise(SmarterCSV::DeprecatedOptions, "ERROR: #{message}") unless used_deprecated_options.empty? || options[:silence_deprecations]
+      else
+        puts "DEPRECATION WARNING: #{message}" unless used_deprecated_options.empty? || options[:silence_deprecations]
+      end
+    end
+
     def validate_options!(options)
       # deprecate required_headers
       unless options[:required_headers].nil?
@@ -90,5 +141,57 @@ module SmarterCSV
     def pp(value)
       defined?(AwesomePrint) ? value.awesome_inspect(index: nil) : value.inspect
     end
+
+    # ---- V2 code ----------------------------------------------------------------------------------------
+
+    V2_DEFAULT_OPTIONS = {
+      # These need to go to the COMMON_OPTIONS:
+      remove_empty_hashes: true, # this might need a transformation or move to common options
+      # ------------
+      header_transformations: [:keys_as_symbols],
+      header_validations: [:unique_headers],
+      # data_transformations: [:replace_blank_with_nil],
+      # data_validations: [],
+      hash_transformations: [:strip_spaces, :remove_blank_values],
+      hash_validations: [],
+      v2_mode: true,
+    }.freeze
+
+    V2_TRANSFORMATIONS = {
+      header_transformations: [:keys_as_symbols],
+      header_validations: [:unique_headers],
+      # data_transformations: [:replace_blank_with_nil],
+      # data_validations: [],
+      hash_transformations: [:v1_backwards_compatibility],
+      # hash_transformations: [:remove_empty_keys, :strip_spaces, :remove_blank_values, :convert_values_to_numeric], # ??? :convert_values_to_numeric]
+      hash_validations: [],
+    }.freeze
+
+    V1_TRANSFORMATIONS = {
+      header_transformations: [:keys_as_symbols],
+      header_validations: [:unique_headers],
+      # data_transformations: [:replace_blank_with_nil],
+      # data_validations: [],
+      hash_transformations: [:strip_spaces, :remove_blank_values, :convert_values_to_numeric],
+      hash_validations: [],
+    }.freeze
+
+    def preprocess_v2_options(options)
+      return options unless options[:v2_mode] || options[:header_transformations]
+
+      # We want to provide safe defaults for easy processing, that is why we have a special keyword :none
+      # to not do any header transformations..
+      #
+      # this is why we need to remove the 'none' here:
+      #
+      requested_header_transformations = options[:header_transformations]
+      if requested_header_transformations.to_s == 'none'
+        requested_header_transformations = []
+      else
+        requested_header_transformations = requested_header_transformations.reject {|x| x.to_s == 'none'} unless requested_header_transformations.nil?
+      end
+      options[:header_transformations] = requested_header_transformations || []
+      options
+    end
   end
 end

data/lib/smarter_csv/smarter_csv.rb

@@ -2,6 +2,7 @@
 
 module SmarterCSV
   class SmarterCSVException < StandardError; end
+  class DeprecatedOptions < SmarterCSVException; end
   class HeaderSizeMismatch < SmarterCSVException; end
   class IncorrectOption < SmarterCSVException; end
   class ValidationError < SmarterCSVException; end
@@ -108,6 +109,10 @@ module SmarterCSV
 
         next if options[:remove_empty_hashes] && hash.empty?
 
+        #
+        # should HASH VALIDATIONS go here instead?
+        #
+
         puts "CSV Line #{@file_line_count}: #{pp(hash)}" if @verbose == '2' # very verbose setting
         # optional adding of csv_line_number to the hash to help debugging
         hash[:csv_line_number] = @csv_line_count if options[:with_line_numbers]

data/lib/smarter_csv/variables.rb

@@ -15,6 +15,7 @@ module SmarterCSV
       @raw_header = nil # header as it appears in the file
       @result = []
       @warnings = {}
+      @v2_mode = false
       @enforce_utf8 = false # only set to true if needed (after options parsing)
     end
 

data/lib/smarter_csv/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SmarterCSV
-  VERSION = "1.10.3"
+  VERSION = "1.11.0.pre1"
 end

data/lib/smarter_csv.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'set'
+
 require "smarter_csv/version"
 require "smarter_csv/file_io"
 require "smarter_csv/options_processing"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: smarter_csv
 version: !ruby/object:Gem::Version
-  version: 1.10.3
+  version: 1.11.0.pre1
 platform: ruby
 authors:
 - Tilo Sloboda
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-03-10 00:00:00.000000000 Z
+date: 2024-01-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: awesome_print
@@ -104,7 +104,6 @@ extensions:
 - ext/smarter_csv/extconf.rb
 extra_rdoc_files: []
 files:
-- ".rspec"
 - ".rubocop.yml"
 - ".rvmrc"
 - CHANGELOG.md
@@ -148,9 +147,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 2.5.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubygems_version: 3.2.3
 signing_key:

data/.rspec

@@ -1 +0,0 @@
---require spec_helper