smarter_csv 1.8.5 → 1.9.2.pre01

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8a812edc2e7a7778b0722a120acf8432eb844895b09668c3adf37184cfa08408
-  data.tar.gz: e78756cef3558b32cfa2788fdbfdd8723cc7a06872c97916fc2c95e16cf363d7
+  metadata.gz: 75d9d441d771c2fbe0861e0bc5f84dbf05d010b4844a867fc4679df002822d07
+  data.tar.gz: 0d5d37d4f2654fd354a2adac23019b2955540c1356c57f72052c01220598ffa2
 SHA512:
-  metadata.gz: e81dfd9e713a301f58c64311a02e5047aa3678652e7c410fd8bfaaa5f49459faa3766f5e152d27b080d8677fb3cb452b45ae76086f9feba5caf5f1cd57220260
-  data.tar.gz: c2a6206128b0860138ca739a70b9546fe6ddbb37b4d09069607abc17a08fd51a2f2070d9ab2a6f55955a5f6a98f9def1dad7ce5ac03d0d6aabe67c9593be792f
+  metadata.gz: b24e2b09ea919994da347eb52b781868a19e0f28dc367bfeb43b8a254619ab8dd882d3035f0546683c2ebc893fd600ce05a3abb800e4b124c7369d314607ee3f
+  data.tar.gz: 3a1115ac4937c2fedf469d1f45e3aa1cf7ed03f1f55d66f6cb310c767a3b5e8cb4966a19ba968f065c5d1de2d7074f479b5eeb0686dbab8012e5e6b8ed0f2628

data/.rubocop.yml

@@ -22,6 +22,9 @@ Metrics/BlockLength:
 Metrics/BlockNesting:
   Enabled: false
 
+Metrics/ClassLength:
+  Enabled: false
+
 Metrics/CyclomaticComplexity: # BS rule
   Enabled: false
 
@@ -46,6 +49,9 @@ Naming/VariableNumber:
 Style/ClassEqualityComparison:
   Enabled: false
 
+Style/ClassMethods:
+  Enabled: false
+
 Style/ConditionalAssignment:
   Enabled: false
 
@@ -114,6 +120,9 @@ Style/StringLiteralsInInterpolation:
   Enabled: false
   EnforcedStyle: double_quotes
 
+Style/SymbolArray:
+  Enabled: false
+
 Style/SymbolProc: # old Ruby versions can't do this
   Enabled: false
 
@@ -123,6 +132,9 @@ Style/TrailingCommaInHashLiteral:
 Style/TrailingUnderscoreVariable:
   Enabled: false
 
+Style/TrivialAccessors:
+  Enabled: false
+
 # Style/UnlessModifier:
 #   Enabled: false
 
@@ -130,4 +142,4 @@ Style/ZeroLengthPredicate:
   Enabled: false
 
 Layout/LineLength:
-  Max: 240
+  Max: 256

data/CHANGELOG.md

@@ -1,6 +1,40 @@
 
 # SmarterCSV 1.x Change Log
 
+## 1.9.2.pre01 (2023-11-11)
+  * fixed bug with '\\' at end of line (issue #252)
+  * fixed require statements
+    
+## 1.9.1 (2023-10-30) (YANKED)
+  * yanked
+  * no functional changes
+  * refactored directory structure
+  * re-added JRuby and TruffleRuby to CI tests
+  * no C-accelleration for JRuby
+  * refactored options parsing
+  * code coverage / rubocop
+
+## 1.9.0 (2023-09-04)
+  * fixed issue #139
+
+  * Error `SmarterCSV::MissingHeaders` was renamed to `SmarterCSV::MissingKeys`
+    
+  * CHANGED BEHAVIOR:
+    When `key_mapping` option is used. (issue #139)
+    Previous versions just printed an error message when a CSV header was missing during key mapping.
+    Versions >= 1.9 will throw `SmarterCSV::MissingHeaders` listing all headers that were missing during mapping.
+
+  * Notable details for `key_mapping` and `required_headers`:
+
+    * `key_mapping` is applied to the headers early on during `SmarterCSV.process`, and raises an error if a header in the input CSV file is missing, and we can not map that header to its desired name.
+
+    Mapping errors can be surpressed by using:
+    * `silence_missing_keys` set to `true`, which silence all such errors, making all headers for mapping optional.
+    * `silence_missing_keys` given an Array with the specific header keys that are optional
+    The use case is that some header fields are optional, but we still want them renamed if they are present.
+
+    * `required_headers` checks which headers are present **after** `key_mapping` was applied.
+
 ## 1.8.5 (2023-06-25)
   * fix parsing of escaped quote characters (thanks to JP Camara)
     

data/Gemfile

@@ -5,10 +5,13 @@ source 'https://rubygems.org'
 # Specify your gem's dependencies in smarter_csv.gemspec
 gemspec
 
-gem "rake" # , "< 11"
+gem "rake"
 gem "rake-compiler"
 
 gem 'pry'
-
-gem "rspec"
 gem "rubocop"
+
+group :test do
+  gem "rspec"
+  gem "simplecov"
+end

data/README.md

@@ -46,7 +46,11 @@ One `smarter_csv` user wrote:
  * able to ignore "columns" in the input (delete columns)
  * able to eliminate nil or empty fields from the result hashes (default)
 
-NOTE; This Gem is only for importing CSV files - writing of CSV files is not supported at this time.
+#### Assumptions / Limitations
+* It is assumed that the escape character is `\`, as on UNIX and Windows systems.
+* It is assumed that quote charcters around fields are balanced, e.g. valid: `"field"`, invalid: `"field\"`
+  e.g. an escaped `quote_char` does not denote the end of a field.
+* This Gem is only for importing CSV files - writing of CSV files is not supported at this time.
 
 ### Why?
 
@@ -161,7 +165,22 @@ and how the `process` method returns the number of chunks when called with a blo
      => returns number of chunks / rows we processed
 ```
 
-#### Example 4: Reading a CSV-like File, and Processing it with Sidekiq:
+#### Example 4: Processing a CSV File, and inserting batch jobs in Sidekiq:
+```ruby
+    filename = '/tmp/input.csv' # CSV file containing ids or data to process
+    options = { :chunk_size => 100 }
+    n = SmarterCSV.process(filename, options) do |chunk|
+      Sidekiq::Client.push_bulk(
+        'class' => SidekiqIndividualWorkerClass,
+        'args' => chunk,
+      )
+      # OR:
+      # SidekiqBatchWorkerClass.process_async(chunk ) # pass an array of hashes to Sidekiq workers for parallel processing
+    end
+    => returns number of chunks
+```
+
+#### Example 4b: Reading a CSV-like File, and Processing it with Sidekiq:
 ```ruby
     filename = '/tmp/strange_db_dump'   # a file with CRTL-A as col_separator, and with CTRL-B\n as record_separator (hello iTunes!)
     options = {
@@ -173,7 +192,6 @@ and how the `process` method returns the number of chunks when called with a blo
     end
     => returns number of chunks
 ```
-
 #### Example 5: Populate a MongoDB Database in Chunks of 100 records with SmarterCSV:
 ```ruby
     # using chunks:
@@ -282,7 +300,9 @@ And header and data validations will also be supported in 2.x
      | Option                      | Default  |  Explanation                                                                         |
      ---------------------------------------------------------------------------------------------------------------------------------
      | :key_mapping                |   nil    | a hash which maps headers from the CSV file to keys in the result hash               |
-     | :silence_missing_key        |   false  | ignore missing keys in `key_mapping` if true                                         |
+     | :silence_missing_key        |   false  | ignore missing keys in `key_mapping`                                   | 
+     |                             |          | if set to true: makes all mapped keys optional                         |
+     |                             |          | if given an array, makes only the keys listed in it optional                         |
      | :required_keys              |   nil    | An array. Specify the required names AFTER header transformation.                  |
      | :required_headers           |   nil    | (DEPRECATED / renamed) Use `required_keys` instead                          |
      |                             |          | or an exception is raised   No validation if nil is given.                           |

data/Rakefile

@@ -3,16 +3,15 @@
 require "bundler/gem_tasks"
 require 'rspec/core/rake_task'
 
-
-# temp fix for NoMethodError: undefined method `last_comment'
-# remove when fixed in Rake 11.x and higher
-module TempFixForRakeLastComment
-  def last_comment
-    last_description
-  end
-end
-Rake::Application.send :include, TempFixForRakeLastComment
-### end of tempfix
+# # temp fix for NoMethodError: undefined method `last_comment'
+# # remove when fixed in Rake 11.x and higher
+# module TempFixForRakeLastComment
+#   def last_comment
+#     last_description
+#   end
+# end
+# Rake::Application.send :include, TempFixForRakeLastComment
+# ### end of tempfix
 
 RSpec::Core::RakeTask.new(:spec)
 
@@ -22,11 +21,26 @@ RuboCop::RakeTask.new
 
 require "rake/extensiontask"
 
-task build: :compile
+if RUBY_ENGINE == 'jruby'
+
+  task default: %i[spec]
 
-Rake::ExtensionTask.new("smarter_csv") do |ext|
-  ext.ext_dir = "ext/smarter_csv"
+else
+  task build: :compile
+
+  Rake::ExtensionTask.new("smarter_csv") do |ext|
+    ext.lib_dir = "lib/smarter_csv"
+    ext.ext_dir = "ext/smarter_csv"
+    ext.source_pattern = "*.{c,h}"
+  end
+
+  # task default: %i[clobber compile spec rubocop]
+  task default: %i[clobber compile spec]
 end
 
-# task default: %i[clobber compile spec rubocop]
-task default: %i[clobber compile spec]
+desc 'Run spec with coverage'
+task :coverage do
+  ENV['COVERAGE'] = 'true'
+  Rake::Task['spec'].execute
+  `open coverage/index.html`
+end

data/ext/smarter_csv/extconf.rb

@@ -1,10 +1,10 @@
 # frozen_string_literal: true
 
 require 'mkmf'
-
 require "rbconfig"
+
 if RbConfig::MAKEFILE_CONFIG["CFLAGS"].include?("-g -O3")
-  fixed_CFLAGS = RbConfig::MAKEFILE_CONFIG["CFLAGS"].sub("-g -O3", "$(cflags)")
+  fixed_CFLAGS = RbConfig::MAKEFILE_CONFIG["CFLAGS"].sub("-g -O3", "-O3 $(cflags)")
   puts("Fix CFLAGS: #{RbConfig::MAKEFILE_CONFIG["CFLAGS"]} -> #{fixed_CFLAGS}")
   RbConfig::MAKEFILE_CONFIG["CFLAGS"] = fixed_CFLAGS
 end

data/ext/smarter_csv/smarter_csv.c

@@ -40,6 +40,7 @@ static VALUE rb_parse_csv_line(VALUE self, VALUE line, VALUE col_sep, VALUE quot
   long i;
 
   char prev_char = '\0'; // Store the previous character for comparison against an escape character
+  long backslash_count = 0; // to count consecutive backslash characters
 
   while (p < endP) {
     /* does the remaining string start with col_sep ? */
@@ -61,8 +62,13 @@ static VALUE rb_parse_csv_line(VALUE self, VALUE line, VALUE col_sep, VALUE quot
         startP = p;
       }
     } else {
-      if (*p == *quoteP && prev_char != '\\') {
-        quote_count += 1;
+      if (*p == '\\') {
+        backslash_count++;
+      } else {
+        if (*p == *quoteP && (backslash_count % 2 == 0)) {
+          quote_count++;
+        }
+        backslash_count = 0; // no more consecutive backslash characters
       }
       p++;
     }

data/lib/smarter_csv/options_processing.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module SmarterCSV
+  DEFAULT_OPTIONS = {
+    acceleration: true,
+    auto_row_sep_chars: 500,
+    chunk_size: nil,
+    col_sep: :auto, # was: ',',
+    comment_regexp: nil, # was: /\A#/,
+    convert_values_to_numeric: true,
+    downcase_header: true,
+    duplicate_header_suffix: nil,
+    file_encoding: 'utf-8',
+    force_simple_split: false,
+    force_utf8: false,
+    headers_in_file: true,
+    invalid_byte_sequence: '',
+    keep_original_headers: false,
+    key_mapping: nil,
+    quote_char: '"',
+    remove_empty_hashes: true,
+    remove_empty_values: true,
+    remove_unmapped_keys: false,
+    remove_values_matching: nil,
+    remove_zero_values: false,
+    required_headers: nil,
+    required_keys: nil,
+    row_sep: :auto, # was: $/,
+    silence_missing_keys: false,
+    skip_lines: nil,
+    strings_as_keys: false,
+    strip_chars_from_headers: nil,
+    strip_whitespace: true,
+    user_provided_headers: nil,
+    value_converters: nil,
+    verbose: false,
+    with_line_numbers: false,
+  }.freeze
+
+  class << self
+    # NOTE: this is not called when "parse" methods are tested by themselves
+    def process_options(given_options = {})
+      puts "User provided options:\n#{pp(given_options)}\n" if given_options[:verbose]
+
+      # fix invalid input
+      given_options[:invalid_byte_sequence] = '' if given_options[:invalid_byte_sequence].nil?
+
+      @options = DEFAULT_OPTIONS.dup.merge!(given_options)
+      puts "Computed options:\n#{pp(@options)}\n" if given_options[:verbose]
+
+      validate_options!(@options)
+      @options
+    end
+
+    # NOTE: this is not called when "parse" methods are tested by themselves
+    #
+    # ONLY FOR BACKWARDS-COMPATIBILITY
+    def default_options
+      DEFAULT_OPTIONS
+    end
+
+    private
+
+    def validate_options!(options)
+      keys = options.keys
+      errors = []
+      errors << "invalid row_sep" if keys.include?(:row_sep) && !option_valid?(options[:row_sep])
+      errors << "invalid col_sep" if keys.include?(:col_sep) && !option_valid?(options[:col_sep])
+      errors << "invalid quote_char" if keys.include?(:quote_char) && !option_valid?(options[:quote_char])
+      raise SmarterCSV::ValidationError, errors.inspect if errors.any?
+    end
+
+    def option_valid?(str)
+      return true if str.is_a?(Symbol) && str == :auto
+      return true if str.is_a?(String) && !str.empty?
+
+      false
+    end
+
+    def pp(value)
+      defined?(AwesomePrint) ? value.awesome_inspect(index: nil) : value.inspect
+    end
+  end
+end