smarter_csv 1.2.3 → 1.2.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c0efcd4dbad2546469ba99c1a133bc7548382e147fca66207d11065607fdb5ee
-  data.tar.gz: 98024633925ed73251dd00606f23f73ca6cbda5517c24882b95935138ab4d7e3
+  metadata.gz: 90df8a442690ff9202ee2762a199e652ebca4dc6e5899d24ee0a72d192cf539f
+  data.tar.gz: 16d4826bc7840a112ef5d5ea3b7acf0997d5fbeb46f8f5289ffa31a7356aa71c
 SHA512:
-  metadata.gz: ac64f3a688f9b5b4bce09e26d8ee140994bf585578ea6eed1d998cec434535f3afb4aa8c2ea222b873cb5ee113fba748412a96e1b3b23a8b5425d13aa7a6436c
-  data.tar.gz: f8c3132f1f2cf60f8f67d2278d0b3660d7aab456581256e1c2508d1397c1cc31d455035c68656777c34e90ad6238c239699b1c59cdd918cdcd63c9f5a264aaba
+  metadata.gz: 85cf20915d26794ba37e508379bc8460d65f054771803779df3b67cd907a61da9ea195d8b51da5a9942bb7f3a5866325114a7a107cbbfcc269f47f6bf6dbfff5
+  data.tar.gz: e2508804722db4bd76e63a9e459b180fb75f7215e3e6d53970e5a83c120d0dab3cdd639c562abfe91d5b9458c3a990aa3959e85000491aef8529442ade5a3f90

data/.travis.yml

@@ -6,10 +6,14 @@ before_install:
 
 matrix:
   include:
-    - rvm: 2.2.8
-    - rvm: 2.3.5
-    - rvm: 2.4.2
-    - rvm: jruby-9.1.13.0
+    - rvm: 2.2.10
+    - rvm: 2.3.8
+    - rvm: 2.4.6
+    - rvm: 2.5.7
+    - rvm: 2.6.3
+    - rvm: 2.7.2
+    - rvm: 3.0.0
+    - rvm: jruby-9.2.7.0
       env:
         - JRUBY_OPTS="--server -Xcompile.invokedynamic=false -J-XX:+TieredCompilation -J-XX:TieredStopAtLevel=1 -J-noverify -J-Xms512m -J-Xmx1024m"
     - rvm: ruby-head
@@ -17,3 +21,5 @@ matrix:
 branches:
   only:
     - master
+    - 1.2-stable
+    - 2.0-develop

data/Gemfile

@@ -3,7 +3,6 @@ source 'https://rubygems.org'
 # Specify your gem's dependencies in smarter_csv.gemspec
 gemspec
 
-
 gem "rake", "< 11"
 gem 'pry'
 

data/README.md

@@ -324,11 +324,24 @@ Planned in the next releases:
 
 ## Changes
 
+#### 1.2.8 (2020-02-04)
+ * fix deprecation warnings on Ruby 2.7 (thank to Diego Salido)
+
+#### 1.2.7 (2020-02-03)
+
+#### 1.2.6 (2018-11-13)
+ * fixing error caused by calling f.close when we do not hand in a file
+
+#### 1.2.5 (2018-09-16)
+ * fixing issue #136 with comments in CSV files
+ * fixing error class hierarchy
+
+#### 1.2.4 (2018-08-06)
+ * using Rails blank? if it's available
+
 #### 1.2.3 (2018-01-27)
  * fixed regression / test
  * fuxed quote_char interpolation for headers, but not data (thanks to Colin Petruno)
-
-#### 1.2.1 (2018-01-25)    ### YANKED!
  * bugfix (thanks to Joshua Smith for reporting)
 
 #### 1.2.0 (2018-01-20)
@@ -385,9 +398,6 @@ Planned in the next releases:
 #### 1.0.14 (2013-11-01)
  * added GPL-2 and MIT license to GEM spec file; if you need another license contact me
 
-#### 1.0.13 (2013-11-01)    ### YANKED!
- * added GPL-2 license to GEM spec file; if you need another license contact me
-
 #### 1.0.12 (2013-10-15)
  * added RSpec tests
 
@@ -503,6 +513,7 @@ And a special thanks to those who contributed pull requests:
  * [Rohit Amarnath](https://github.com/ramarnat)
  * [Joshua Smith](https://github.com/enviable)
  * [Colin Petruno](https://github.com/colinpetruno)
+ * [Diego Salido](https://github.com/salidux)
 
 
 ## Contributing

data/lib/smarter_csv/smarter_csv.rb

@@ -1,16 +1,16 @@
 module SmarterCSV
-
-  class HeaderSizeMismatch < Exception; end
-  class IncorrectOption < Exception; end
-  class DuplicateHeaders < Exception; end
-  class MissingHeaders < Exception; end
+  class SmarterCSVException < StandardError; end
+  class HeaderSizeMismatch < SmarterCSVException; end
+  class IncorrectOption < SmarterCSVException; end
+  class DuplicateHeaders < SmarterCSVException; end
+  class MissingHeaders < SmarterCSVException; end
 
 
   def SmarterCSV.process(input, options={}, &block)   # first parameter: filename or input object with readline method
     default_options = {:col_sep => ',' , :row_sep => $/ , :quote_char => '"', :force_simple_split => false , :verbose => false ,
       :remove_empty_values => true, :remove_zero_values => false , :remove_values_matching => nil , :remove_empty_hashes => true , :strip_whitespace => true,
       :convert_values_to_numeric => true, :strip_chars_from_headers => nil , :user_provided_headers => nil , :headers_in_file => true,
-      :comment_regexp => /^#/, :chunk_size => nil , :key_mapping_hash => nil , :downcase_header => true, :strings_as_keys => false, :file_encoding => 'utf-8',
+      :comment_regexp => /\A#/, :chunk_size => nil , :key_mapping_hash => nil , :downcase_header => true, :strings_as_keys => false, :file_encoding => 'utf-8',
       :remove_unmapped_keys => false, :keep_original_headers => false, :value_converters => nil, :skip_lines => nil, :force_utf8 => false, :invalid_byte_sequence => '',
       :auto_row_sep_chars => 500, :required_headers => nil
     }
@@ -22,6 +22,7 @@ module SmarterCSV
     old_row_sep = $/
     file_line_count = 0
     csv_line_count = 0
+    has_rails = !! defined?(Rails)
     begin
       f = input.respond_to?(:readline) ? input : File.open(input, "r:#{options[:file_encoding]}")
 
@@ -52,7 +53,7 @@ module SmarterCSV
 
         if (header =~ %r{#{options[:quote_char]}}) and (! options[:force_simple_split])
           file_headerA = begin
-            CSV.parse( header, csv_options ).flatten.collect!{|x| x.nil? ? '' : x} # to deal with nil values from CSV.parse
+            CSV.parse( header, **csv_options ).flatten.collect!{|x| x.nil? ? '' : x} # to deal with nil values from CSV.parse
           rescue CSV::MalformedCSVError => e
             raise $!, "#{$!} [SmarterCSV: csv line #{csv_line_count}]", $!.backtrace
           end
@@ -148,7 +149,7 @@ module SmarterCSV
 
         if (line =~ %r{#{options[:quote_char]}}) and (! options[:force_simple_split])
           dataA = begin
-            CSV.parse( line, csv_options ).flatten.collect!{|x| x.nil? ? '' : x} # to deal with nil values from CSV.parse
+            CSV.parse( line, **csv_options ).flatten.collect!{|x| x.nil? ? '' : x} # to deal with nil values from CSV.parse
           rescue CSV::MalformedCSVError => e
             raise $!, "#{$!} [SmarterCSV: csv line #{csv_line_count}]", $!.backtrace
           end
@@ -167,7 +168,13 @@ module SmarterCSV
 
         # remove empty values using the same regexp as used by the rails blank? method
         # which caters for double \n and \r\n characters such as "1\r\n\r\n2" whereas the original check (v =~ /^\s*$/) does not
-        hash.delete_if{|k,v| v.nil? || v !~ /[^[:space:]]/}  if options[:remove_empty_values]
+        if options[:remove_empty_values]
+          if has_rails
+            hash.delete_if{|k,v| v.blank?}
+          else
+            hash.delete_if{|k,v| v.nil? || v !~ /[^[:space:]]/}
+          end
+        end
 
         hash.delete_if{|k,v| ! v.nil? && v =~ /^(\d+|\d+\.\d+)$/ && v.to_f == 0} if options[:remove_zero_values]   # values are typically Strings!
         hash.delete_if{|k,v| v =~ options[:remove_values_matching]} if options[:remove_values_matching]
@@ -241,7 +248,7 @@ module SmarterCSV
       end
     ensure
       $/ = old_row_sep   # make sure this stupid global variable is always reset to it's previous value after we're done!
-      f.close
+      f.close if f.respond_to?(:close)
     end
     if block_given?
       return chunk_count  # when we do processing through a block we only care how many chunks we processed

data/lib/smarter_csv/version.rb

@@ -1,3 +1,3 @@
 module SmarterCSV
-  VERSION = "1.2.3"
+  VERSION = "1.2.8"
 end

data/spec/fixtures/ignore_comments.csv

@@ -0,0 +1,11 @@
+not a comment#First Name,Last Name,Dogs,Cats,Birds,Fish
+# comment two
+Dan,McAllister,2,0,,
+Lucy#L,Laweless,,5,0,
+# anothter comment
+,,,,,
+Miles,O'Brian,0,0,0,21
+Nancy,Homes,2,0,1,
+Hernán,Curaçon,3,0,0,
+#comment,comment,1,2,3,4
+,,,,,

data/spec/fixtures/ignore_comments2.csv

@@ -0,0 +1,3 @@
+h1,h2
+a,"b
+#c"

data/spec/fixtures/problematic.csv

@@ -0,0 +1,8 @@
+Compte;Date de comptabilisation;Date op�ration;Libell�;R�f�rence;Date valeur;Montant
+22215449203;02/06/2018;01/06/2018;ECHEANCE PRET DONT CAP    410,33 ASS.    8,00E INT.    21,87 COM.    0,00E;8711552;01/06/2018;-440,20;
+22215449203;04/06/2018;04/06/2018;EVI Gaultier Laperche remboursement compte courant;1038326;04/06/2018;-144,07;
+22215449203;04/06/2018;04/06/2018;EVI Guillemain Nicolas remboursement CC pret d'honneur;1038328;04/06/2018;-144,07;
+22215449203;01/06/2018;01/06/2018;310518 SC****5448 INTERMARCHE    95ERMONT;701JQ1K;01/06/2018;-16,00;
+22215449203;01/06/2018;01/06/2018;EVI Stripe Payments UK L STRIPE E7U0R1;706AO1Q;01/06/2018;45,89;
+22215449203;01/06/2018;01/06/2018;EVI Compte N26 Heroku;1100653;01/06/2018;-700,00;
+22215449203;31/05/2018;31/05/2018;EVI Stripe Payments UK L STRIPE L2J1N7;6YISBWF;31/05/2018;465,89;

data/spec/smarter_csv/ignore_comments_spec.rb

@@ -0,0 +1,30 @@
+require 'spec_helper'
+
+fixture_path = 'spec/fixtures'
+
+describe 'be_able_to' do
+  it 'ignore comments in CSV files' do
+    options = {}
+    data = SmarterCSV.process("#{fixture_path}/ignore_comments.csv", options)
+
+    data.size.should eq 5
+
+    # all the keys should be symbols
+    data.each{|item| item.keys.each{|x| x.is_a?(Symbol).should be_truthy}}
+    data.each do |h|
+      h.keys.each do |key|
+        [:"not_a_comment#first_name", :last_name, :dogs, :cats, :birds, :fish].should include( key )
+      end
+    end
+  end
+
+  it 'ignore comments in CSV files with CRLF' do
+    options = {row_sep: "\r\n"}
+    data = SmarterCSV.process("#{fixture_path}/ignore_comments2.csv", options)
+
+    # all the keys should be symbols
+    data.size.should eq 1
+    data.first[:h1].should eq 'a'
+    data.first[:h2].should eq "b\r\n#c"
+  end
+end

data/spec/smarter_csv/problematic.rb

@@ -0,0 +1,34 @@
+require 'spec_helper'
+
+fixture_path = 'spec/fixtures'
+
+describe 'loading file with UTF-8 characters in the header' do
+
+  # file which caused issues because of UTF-8 characters in the header
+  it 'loads the file with force_utf8 flag set' do
+    options = {col_sep:  ";", force_utf8: true}
+    data = SmarterCSV.process("#{fixture_path}/problematic.csv", options)
+
+    data.length.should eq 7
+  end
+
+  it 'loads the file with strings as keys' do
+    options = {
+      file_encoding: 'iso-8859-1:UTF-8', # important!
+      col_sep:  ";", strings_as_keys: true,
+    }
+    data = SmarterCSV.process("#{fixture_path}/problematic.csv", options)
+
+    data.length.should eq 7
+    data.first.keys.sort.should eq [
+      "compte",
+      "date_de_comptabilisation",
+      "date_opération",
+      "date_valeur",
+      "libellé",
+      "montant",
+      "référence"
+    ]
+  end
+
+end

metadata

@@ -1,16 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: smarter_csv
 version: !ruby/object:Gem::Version
-  version: 1.2.3
+  version: 1.2.8
 platform: ruby
 authors:
 - 'Tilo Sloboda
 
-'
-autorequire: 
+  '
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-01-27 00:00:00.000000000 Z
+date: 2021-02-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -32,7 +32,7 @@ description: Ruby Gem for smarter importing of CSV Files as Array(s) of Hashes,
 email:
 - 'tilo.sloboda@gmail.com
 
-'
+  '
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -58,6 +58,8 @@ files:
 - spec/fixtures/chunk_cornercase.csv
 - spec/fixtures/duplicate_headers.csv
 - spec/fixtures/empty.csv
+- spec/fixtures/ignore_comments.csv
+- spec/fixtures/ignore_comments2.csv
 - spec/fixtures/line_endings_n.csv
 - spec/fixtures/line_endings_r.csv
 - spec/fixtures/line_endings_rn.csv
@@ -68,6 +70,7 @@ files:
 - spec/fixtures/no_header.csv
 - spec/fixtures/numeric.csv
 - spec/fixtures/pets.csv
+- spec/fixtures/problematic.csv
 - spec/fixtures/quote_char.csv
 - spec/fixtures/quoted.csv
 - spec/fixtures/quoted2.csv
@@ -87,6 +90,7 @@ files:
 - spec/smarter_csv/convert_values_to_numeric_spec.rb
 - spec/smarter_csv/extenstions_spec.rb
 - spec/smarter_csv/header_transformation_spec.rb
+- spec/smarter_csv/ignore_comments_spec.rb
 - spec/smarter_csv/invalid_headers_spec.rb
 - spec/smarter_csv/keep_headers_spec.rb
 - spec/smarter_csv/key_mapping_spec.rb
@@ -95,6 +99,7 @@ files:
 - spec/smarter_csv/malformed_spec.rb
 - spec/smarter_csv/no_header_spec.rb
 - spec/smarter_csv/not_downcase_header_spec.rb
+- spec/smarter_csv/problematic.rb
 - spec/smarter_csv/quoted_spec.rb
 - spec/smarter_csv/remove_empty_values_spec.rb
 - spec/smarter_csv/remove_keys_from_hashes_spec.rb
@@ -115,7 +120,7 @@ licenses:
 - MIT
 - GPL-2
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -131,9 +136,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements:
 - csv
-rubyforge_project: 
-rubygems_version: 2.7.4
-signing_key: 
+rubygems_version: 3.0.6
+signing_key:
 specification_version: 4
 summary: Ruby Gem for smarter importing of CSV Files (and CSV-like files), with lots
   of optional features, e.g. chunked processing for huge CSV files
@@ -147,6 +151,8 @@ test_files:
 - spec/fixtures/chunk_cornercase.csv
 - spec/fixtures/duplicate_headers.csv
 - spec/fixtures/empty.csv
+- spec/fixtures/ignore_comments.csv
+- spec/fixtures/ignore_comments2.csv
 - spec/fixtures/line_endings_n.csv
 - spec/fixtures/line_endings_r.csv
 - spec/fixtures/line_endings_rn.csv
@@ -157,6 +163,7 @@ test_files:
 - spec/fixtures/no_header.csv
 - spec/fixtures/numeric.csv
 - spec/fixtures/pets.csv
+- spec/fixtures/problematic.csv
 - spec/fixtures/quote_char.csv
 - spec/fixtures/quoted.csv
 - spec/fixtures/quoted2.csv
@@ -176,6 +183,7 @@ test_files:
 - spec/smarter_csv/convert_values_to_numeric_spec.rb
 - spec/smarter_csv/extenstions_spec.rb
 - spec/smarter_csv/header_transformation_spec.rb
+- spec/smarter_csv/ignore_comments_spec.rb
 - spec/smarter_csv/invalid_headers_spec.rb
 - spec/smarter_csv/keep_headers_spec.rb
 - spec/smarter_csv/key_mapping_spec.rb
@@ -184,6 +192,7 @@ test_files:
 - spec/smarter_csv/malformed_spec.rb
 - spec/smarter_csv/no_header_spec.rb
 - spec/smarter_csv/not_downcase_header_spec.rb
+- spec/smarter_csv/problematic.rb
 - spec/smarter_csv/quoted_spec.rb
 - spec/smarter_csv/remove_empty_values_spec.rb
 - spec/smarter_csv/remove_keys_from_hashes_spec.rb