smarter_csv 1.4.2 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3be724101d41326ff480bcb723c1b40a3cabd879eb55e0c2f044372f8e5a57d0
-  data.tar.gz: 657db1421352f449bf042f8df4d5178167af048ad37836e4f2f2f8a6aea3ece0
+  metadata.gz: 23032eface2d1d918bcd6daabb4ca79e03096612bda1017d06f1b0542d0c4619
+  data.tar.gz: 12b68eeafc4f83c06b66da45b27da5e716675bff1e77be2362c2c10006821d9c
 SHA512:
-  metadata.gz: 3430649df35ac8139d35b04b85e8691ca5fc3d98b7b15f0d3987855f571987bdb742e0ed6f807ddb7a2e61e61d696d529ac311bc58e30188325f1c4bb78098a4
-  data.tar.gz: 1b386af7cc7c39bc7ea934875e16f6641a2cc0c2bb5dfaa3b1f298739b1b355b2f41570e42998a2d7790a17f96feb07118b69c23d913acc634aae5901f0c9229
+  metadata.gz: 5b84337de25ed7a8492088b82342e6d3b16d1fdc95120f9699986aee7d9416a51cfec981eb125e0d4b17600bc1c06c52eb3b2251857668210d9402e95bb75860
+  data.tar.gz: b26b40b49bf6d739df9cd5deb477c33fdc22c54ed88c96f87e401fef789aedf3f9c55d25df60e4900a1e2a3c8bc0fc6e78018b128ab6ac14062fd97f694f3568

data/CHANGELOG.md

@@ -1,7 +1,19 @@
 
 # SmarterCSV 1.x Change Log
 
-## 1.4.1 (2022-02-12)
+## 1.5.0 (2022-04-25)
+  * fixed bug with trailing col_sep characters, introduced in 1.4.0
+  * Fix deprecation warning in Ruby 3.0.3 / $INPUT_RECORD_SEPARATOR (thanks to Joel Fouse )
+
+  * changed default for `comment_regexp` to be `nil` for a safer default behavior (thanks to David Lazar)
+  **Note**
+    This no longer assumes that lines starting with `#` are comments.
+    If you want to treat lines starting with '#' as comments, use `comment_regexp: /\A#/`
+
+## 1.4.2 (2022-02-12)
+  * fixed issue with simplecov
+
+## 1.4.1 (2022-02-12) (PULLED)
   * minor fix: also support `col_sep: :auto`
   * added simplecov
 

data/CONTRIBUTORS.md

@@ -43,3 +43,4 @@ A Big Thank you to everyone who filed issues, sent comments, and who contributed
  * [Olle Jonsson](https://github.com/olleolleolle)
  * [Nicolas Guillemain](https://github.com/Viiruus)
  * [Sp6](https://github.com/sp6)
+ * [Joel Fouse](https://github.com/jfouse)

data/README.md

@@ -215,7 +215,7 @@ The options and the block are optional.
      | :invalid_byte_sequence      |   ''     | what to replace invalid byte sequences with                                          |
      | :force_utf8                 |   false  | force UTF-8 encoding of all lines (including headers) in the CSV file                |
      | :skip_lines                 |   nil    | how many lines to skip before the first line or header line is processed             |
-     | :comment_regexp             |   /^#/   | regular expression which matches comment lines (see NOTE about the CSV header)       |
+     | :comment_regexp             |   nil    | regular expression to ignore comment lines (see NOTE on CSV header), e.g./\A#/       |
      ---------------------------------------------------------------------------------------------------------------------------------
      | :col_sep                    |   ','    | column separator, can be set to :auto                                                |
      | :force_simple_split         |   false  | force simple splitting on :col_sep character for non-standard CSV-files.             |
@@ -284,7 +284,8 @@ And header and data validations will also be supported in 2.x
 ```
 #### NOTES about CSV Headers:
  * as this method parses CSV files, it is assumed that the first line of any file will contain a valid header
- * the first line with the CSV header may or may not be commented out according to the :comment_regexp
+ * the first line with the header might be commented out, in which case you will need to set `comment_regexp: /\A#/`
+   This is no longer handled automatically since 1.5.0.
  * any occurences of :comment_regexp or :row_sep will be stripped from the first line with the CSV header
  * any of the keys in the header line will be downcased, spaces replaced by underscore, and converted to Ruby symbols before being used as keys in the returned Hashes
  * you can not combine the :user_provided_headers and :key_mapping options

data/lib/smarter_csv/smarter_csv.rb

@@ -12,7 +12,6 @@ module SmarterCSV
 
     headerA = []
     result = []
-    old_row_sep = $INPUT_RECORD_SEPARATOR
     file_line_count = 0
     csv_line_count = 0
     has_rails = !! defined?(Rails)
@@ -21,9 +20,8 @@ module SmarterCSV
 
       # auto-detect the row separator
       options[:row_sep] = SmarterCSV.guess_line_ending(f, options) if options[:row_sep].to_sym == :auto
-      $INPUT_RECORD_SEPARATOR = options[:row_sep]
       # attempt to auto-detect column separator
-      options[:col_sep] = guess_column_separator(f) if options[:col_sep].to_sym == :auto
+      options[:col_sep] = guess_column_separator(f, options) if options[:col_sep].to_sym == :auto
       # preserve options, in case we need to call the CSV class
       csv_options = options.select{|k,v| [:col_sep, :row_sep, :quote_char].include?(k)} # options.slice(:col_sep, :row_sep, :quote_char)
       csv_options.delete(:row_sep) if [nil, :auto].include?( options[:row_sep].to_sym )
@@ -33,14 +31,15 @@ module SmarterCSV
         puts 'WARNING: you are trying to process UTF-8 input, but did not open the input with "b:utf-8" option. See README file "NOTES about File Encodings".'
       end
 
-      options[:skip_lines].to_i.times{f.readline} if options[:skip_lines].to_i > 0
+      options[:skip_lines].to_i.times{f.readline(options[:row_sep])} if options[:skip_lines].to_i > 0
 
       if options[:headers_in_file]        # extract the header line
         # process the header line in the CSV file..
         # the first line of a CSV file contains the header .. it might be commented out, so we need to read it anyhow
-        header = f.readline
+        header = f.readline(options[:row_sep])
         header = header.force_encoding('utf-8').encode('utf-8', invalid: :replace, undef: :replace, replace: options[:invalid_byte_sequence]) if options[:force_utf8] || options[:file_encoding] !~ /utf-8/i
-        header = header.sub(options[:comment_regexp],'').chomp(options[:row_sep])
+        header = header.sub(options[:comment_regexp],'') if options[:comment_regexp]
+        header = header.chomp(options[:row_sep])
 
         file_line_count += 1
         csv_line_count += 1
@@ -120,7 +119,7 @@ module SmarterCSV
 
       # now on to processing all the rest of the lines in the CSV file:
       while ! f.eof?    # we can't use f.readlines() here, because this would read the whole file into memory at once, and eof => true
-        line = f.readline  # read one line.. this uses the input_record_separator $INPUT_RECORD_SEPARATOR which we set previously!
+        line = f.readline(options[:row_sep])  # read one line
 
         # replace invalid byte sequence in UTF-8 with question mark to avoid errors
         line = line.force_encoding('utf-8').encode('utf-8', invalid: :replace, undef: :replace, replace: options[:invalid_byte_sequence]) if options[:force_utf8] || options[:file_encoding] !~ /utf-8/i
@@ -128,21 +127,22 @@ module SmarterCSV
         file_line_count += 1
         csv_line_count += 1
         print "processing file line %10d, csv line %10d\r" % [file_line_count, csv_line_count] if options[:verbose]
-        next  if  line =~ options[:comment_regexp]  # ignore all comment lines if there are any
+
+        next if options[:comment_regexp] && line =~ options[:comment_regexp] # ignore all comment lines if there are any
 
         # cater for the quoted csv data containing the row separator carriage return character
         # in which case the row data will be split across multiple lines (see the sample content in spec/fixtures/carriage_returns_rn.csv)
         # by detecting the existence of an uneven number of quote characters
         multiline = line.count(options[:quote_char])%2 == 1 # should handle quote_char nil
         while line.count(options[:quote_char])%2 == 1 # should handle quote_char nil
-          next_line = f.readline
+          next_line = f.readline(options[:row_sep])
           next_line = next_line.force_encoding('utf-8').encode('utf-8', invalid: :replace, undef: :replace, replace: options[:invalid_byte_sequence]) if options[:force_utf8] || options[:file_encoding] !~ /utf-8/i
           line += next_line
           file_line_count += 1
         end
         print "\nline contains uneven number of quote chars so including content through file line %d\n" % file_line_count if options[:verbose] && multiline
 
-        line.chomp!    # will use $INPUT_RECORD_SEPARATOR which is set to options[:col_sep]
+        line.chomp!(options[:row_sep])
 
         if (line =~ %r{#{options[:quote_char]}}) and (! options[:force_simple_split])
           dataA = begin
@@ -151,10 +151,10 @@ module SmarterCSV
             raise $!, "#{$!} [SmarterCSV: csv line #{csv_line_count}]", $!.backtrace
           end
         else
-          dataA =  line.split(options[:col_sep], header_size)
+          dataA = line.split(options[:col_sep], header_size)
         end
-####     dataA.map!{|x| x.gsub(%r/#{options[:quote_char]}/,'') }  # this is actually not a good idea as a default
-        dataA.map!{|x| x.strip}  if options[:strip_whitespace]
+        dataA.map!{|x| x.sub(/(#{options[:col_sep]})+\z/, '')} # remove any unwanted trailing col_sep characters at the end
+        dataA.map!{|x| x.strip} if options[:strip_whitespace]
 
         # if all values are blank, then ignore this line
         # SEE: https://github.com/rails/rails/blob/32015b6f369adc839c4f0955f2d9dce50c0b6123/activesupport/lib/active_support/core_ext/object/blank.rb#L121
@@ -249,7 +249,6 @@ module SmarterCSV
         chunk = []  # initialize for next chunk of data
       end
     ensure
-      $INPUT_RECORD_SEPARATOR = old_row_sep   # make sure this stupid global variable is always reset to it's previous value after we're done!
       f.close if f.respond_to?(:close)
     end
     if block_given?
@@ -266,7 +265,7 @@ module SmarterCSV
       auto_row_sep_chars: 500,
       chunk_size: nil ,
       col_sep: ',',
-      comment_regexp: /\A#/,
+      comment_regexp: nil, # was: /\A#/,
       convert_values_to_numeric: true,
       downcase_header: true,
       file_encoding: 'utf-8',
@@ -329,11 +328,11 @@ module SmarterCSV
   end
 
   # raise exception if none is found
-  def self.guess_column_separator(filehandle)
+  def self.guess_column_separator(filehandle, options)
     del = [',', "\t", ';', ':', '|']
     n = Hash.new(0)
     5.times do
-      line = filehandle.readline
+      line = filehandle.readline(options[:row_sep])
       del.each do |d|
         n[d] += line.scan(d).count
       end

data/lib/smarter_csv/version.rb

@@ -1,3 +1,3 @@
 module SmarterCSV
-  VERSION = "1.4.2"
+  VERSION = "1.5.0"
 end

data/spec/fixtures/additional_separator.csv

@@ -0,0 +1,6 @@
+col1,col2
+eins,zwei
+uno,dos,
+one,two ,,,
+ichi, ,,,,,
+un

data/spec/fixtures/hard_sample.csv

@@ -0,0 +1,2 @@
+Name,Email,Financial Status,Paid at,Fulfillment Status,Fulfilled at,Accepts Marketing,Currency,Subtotal,Shipping,Taxes,Total,Discount Code,Discount Amount,Shipping Method,Created at,Lineitem quantity,Lineitem name,Lineitem price,Lineitem compare at price,Lineitem sku,Lineitem requires shipping,Lineitem taxable,Lineitem fulfillment status,Billing Name,Billing Street,Billing Address1,Billing Address2,Billing Company,Billing City,Billing Zip,Billing Province,Billing Country,Billing Phone,Shipping Name,Shipping Street,Shipping Address1,Shipping Address2,Shipping Company,Shipping City,Shipping Zip,Shipping Province,Shipping Country,Shipping Phone,Notes,Note Attributes,Cancelled at,Payment Method,Payment Reference,Refunded Amount,Vendor,        rece,Tags,Risk Level,Source,Lineitem discount,Tax 1 Name,Tax 1 Value,Tax 2 Name,Tax 2 Value,Tax 3 Name,Tax 3 Value,Tax 4 Name,Tax 4 Value,Tax 5 Name,Tax 5 Value,Phone,Receipt Number,Duties,Billing Province Name,Shipping Province Name,Payment ID,Payment Terms Name,Next Payment Due At
+#MR1220817,foo@bar.com,paid,2022-02-08 22:31:28 +0100,unfulfilled,,yes,EUR,144,0,24,144,VIP,119.6,"Livraison Standard GRATUITE, 2-5 jours avec suivi",2022-02-08 22:31:26 +0100,2,Cire Épilation Nacrée,37,,WAX-200-NAC,true,true,pending,French Fry,64 Boulevard Budgié,64 Boulevard Budgié,,,dootdoot’,'49100,,FR,06 12 34 56 78,French Fry,64 Boulevard Budgi,64 Boulevard Budgié,,,dootdoot,'49100,,FR,06 12 34 56 78,,,,Stripe,c23800013619353.2,0,Goober  Rég,4331065802905,902,Low,web,0,FR TVA 20%,24,,,,,,,,,3366012111111,,,,,,,

data/spec/smarter_csv/additional_separator_spec.rb

@@ -0,0 +1,45 @@
+require 'spec_helper'
+
+fixture_path = 'spec/fixtures'
+
+describe 'handling of additional trailing column separators' do
+  let(:file) { "#{fixture_path}/additional_separator.csv" }
+
+  describe '' do
+    let(:data) { SmarterCSV.process(file) }
+
+    it 'reads all lines' do
+      data.size.should eq 5
+    end
+
+    it 'reads regular lines' do
+      item = data[0]
+      item[:col1].should == 'eins'
+      item[:col2].should == 'zwei'
+    end
+
+    it 'strips single trailing col_sep character' do
+      item = data[1]
+      item[:col1].should == 'uno'
+      item[:col2].should == 'dos'
+    end
+
+    it 'strips multiple trailing col_sep characters' do
+      item = data[2]
+      item[:col1].should == 'one'
+      item[:col2].should == 'two'
+    end
+
+    it 'strips multiple trailing col_sep chars' do
+      item = data[3]
+      item[:col1].should == 'ichi'
+      item[:col2].should == nil
+    end
+
+    it 'strips multiple trailing col_sep chars' do
+      item = data[4]
+      item[:col1].should == 'un'
+      item[:col2].should == nil
+    end
+  end
+end

data/spec/smarter_csv/binary_file2_spec.rb

@@ -12,7 +12,7 @@ describe 'be_able_to' do
   it 'loads_binary_file_with_strings_as_keys' do 
     options = {:col_sep => "\cA", :row_sep => "\cB", :comment_regexp => /^#/, :strings_as_keys => true}
     data = SmarterCSV.process("#{fixture_path}/binary.csv", options)
-    data.flatten.size.should == 8
+    data.size.should == 8
     data.each do |item|
       # all keys should be strings
       item.keys.each{|x| x.class.should be == String}

data/spec/smarter_csv/hard_sample_spec.rb

@@ -0,0 +1,24 @@
+require 'spec_helper'
+
+fixture_path = 'spec/fixtures'
+
+describe 'can handle the difficult CSV file' do
+
+  it 'loads the data with default values' do
+    data = SmarterCSV.process("#{fixture_path}/hard_sample.csv")
+    data.size.should eq 1
+    item = data.first
+    item.keys.count.should == 48
+    item[:name].should == '#MR1220817'
+    item[:shipping_method].should == 'Livraison Standard GRATUITE, 2-5 jours avec suivi'
+    item[:lineitem_name].should == 'Cire Épilation Nacrée'
+    item[:phone].should == 3366012111111
+  end
+
+  # the main problem is the data line starting with a # character, but not being a comment
+  it 'fails to load the CSV file with incorrectly set comment_regexp' do
+    options = {comment_regexp: /\A#/ }
+    data = SmarterCSV.process("#{fixture_path}/hard_sample.csv", options)
+    data.size.should eq 0
+  end
+end

data/spec/smarter_csv/ignore_comments_spec.rb

@@ -1,30 +1,45 @@
-require 'spec_helper'
-
-fixture_path = 'spec/fixtures'
-
-describe 'be_able_to' do
-  it 'ignore comments in CSV files' do
-    options = {}
-    data = SmarterCSV.process("#{fixture_path}/ignore_comments.csv", options)
-
-    data.size.should eq 5
-
-    # all the keys should be symbols
-    data.each{|item| item.keys.each{|x| x.is_a?(Symbol).should be_truthy}}
-    data.each do |h|
-      h.keys.each do |key|
-        [:"not_a_comment#first_name", :last_name, :dogs, :cats, :birds, :fish].should include( key )
-      end
-    end
-  end
-
-  it 'ignore comments in CSV files with CRLF' do
-    options = {row_sep: "\r\n"}
-    data = SmarterCSV.process("#{fixture_path}/ignore_comments2.csv", options)
-
-    # all the keys should be symbols
-    data.size.should eq 1
-    data.first[:h1].should eq 'a'
-    data.first[:h2].should eq "b\r\n#c"
-  end
-end
+require 'spec_helper'
+
+fixture_path = 'spec/fixtures'
+
+describe 'be_able_to' do
+  it 'by default does not ignore comments in CSV files' do
+    options = {}
+    data = SmarterCSV.process("#{fixture_path}/ignore_comments.csv", options)
+
+    data.size.should eq 8
+
+    # all the keys should be symbols
+    data.each{|item| item.keys.each{|x| x.is_a?(Symbol).should be_truthy}}
+    data.each do |h|
+      h.keys.each do |key|
+        [:"not_a_comment#first_name", :last_name, :dogs, :cats, :birds, :fish].should include( key )
+      end
+    end
+  end
+
+  it 'ignore comments in CSV files using comment_regexp' do
+    options = {comment_regexp: /\A#/}
+    data = SmarterCSV.process("#{fixture_path}/ignore_comments.csv", options)
+
+    data.size.should eq 5
+
+    # all the keys should be symbols
+    data.each{|item| item.keys.each{|x| x.is_a?(Symbol).should be_truthy}}
+    data.each do |h|
+      h.keys.each do |key|
+        [:"not_a_comment#first_name", :last_name, :dogs, :cats, :birds, :fish].should include( key )
+      end
+    end
+  end
+
+  it 'ignore comments in CSV files with CRLF' do
+    options = {row_sep: "\r\n"}
+    data = SmarterCSV.process("#{fixture_path}/ignore_comments2.csv", options)
+
+    # all the keys should be symbols
+    data.size.should eq 1
+    data.first[:h1].should eq 'a'
+    data.first[:h2].should eq "b\r\n#c"
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: smarter_csv
 version: !ruby/object:Gem::Version
-  version: 1.4.2
+  version: 1.5.0
 platform: ruby
 authors:
 - Tilo Sloboda
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-02-15 00:00:00.000000000 Z
+date: 2022-04-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -62,6 +62,7 @@ files:
 - lib/smarter_csv/smarter_csv.rb
 - lib/smarter_csv/version.rb
 - smarter_csv.gemspec
+- spec/fixtures/additional_separator.csv
 - spec/fixtures/basic.csv
 - spec/fixtures/binary.csv
 - spec/fixtures/carriage_returns_n.csv
@@ -73,6 +74,7 @@ files:
 - spec/fixtures/empty.csv
 - spec/fixtures/empty_columns_1.csv
 - spec/fixtures/empty_columns_2.csv
+- spec/fixtures/hard_sample.csv
 - spec/fixtures/ignore_comments.csv
 - spec/fixtures/ignore_comments2.csv
 - spec/fixtures/key_mapping.csv
@@ -101,6 +103,7 @@ files:
 - spec/fixtures/valid_unicode.csv
 - spec/fixtures/with_dashes.csv
 - spec/fixtures/with_dates.csv
+- spec/smarter_csv/additional_separator_spec.rb
 - spec/smarter_csv/binary_file2_spec.rb
 - spec/smarter_csv/binary_file_spec.rb
 - spec/smarter_csv/blank_spec.rb
@@ -111,6 +114,7 @@ files:
 - spec/smarter_csv/convert_values_to_numeric_spec.rb
 - spec/smarter_csv/empty_columns_spec.rb
 - spec/smarter_csv/extenstions_spec.rb
+- spec/smarter_csv/hard_sample_spec.rb
 - spec/smarter_csv/header_transformation_spec.rb
 - spec/smarter_csv/ignore_comments_spec.rb
 - spec/smarter_csv/invalid_headers_spec.rb
@@ -164,6 +168,7 @@ specification_version: 4
 summary: Ruby Gem for smarter importing of CSV Files (and CSV-like files), with lots
   of optional features, e.g. chunked processing for huge CSV files
 test_files:
+- spec/fixtures/additional_separator.csv
 - spec/fixtures/basic.csv
 - spec/fixtures/binary.csv
 - spec/fixtures/carriage_returns_n.csv
@@ -175,6 +180,7 @@ test_files:
 - spec/fixtures/empty.csv
 - spec/fixtures/empty_columns_1.csv
 - spec/fixtures/empty_columns_2.csv
+- spec/fixtures/hard_sample.csv
 - spec/fixtures/ignore_comments.csv
 - spec/fixtures/ignore_comments2.csv
 - spec/fixtures/key_mapping.csv
@@ -203,6 +209,7 @@ test_files:
 - spec/fixtures/valid_unicode.csv
 - spec/fixtures/with_dashes.csv
 - spec/fixtures/with_dates.csv
+- spec/smarter_csv/additional_separator_spec.rb
 - spec/smarter_csv/binary_file2_spec.rb
 - spec/smarter_csv/binary_file_spec.rb
 - spec/smarter_csv/blank_spec.rb
@@ -213,6 +220,7 @@ test_files:
 - spec/smarter_csv/convert_values_to_numeric_spec.rb
 - spec/smarter_csv/empty_columns_spec.rb
 - spec/smarter_csv/extenstions_spec.rb
+- spec/smarter_csv/hard_sample_spec.rb
 - spec/smarter_csv/header_transformation_spec.rb
 - spec/smarter_csv/ignore_comments_spec.rb
 - spec/smarter_csv/invalid_headers_spec.rb