smarter_csv 1.2.0 → 1.2.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 251eb1eff306211d163c6be6880ba45e3eb29985
-  data.tar.gz: 9e16d1d4aaab86df7a65d73e6a23ace8f481ba4e
+SHA256:
+  metadata.gz: c912f58ae42fab60cb40e5a708e1551efda1a4abf4f5ad88e79f5803b5c84137
+  data.tar.gz: 438245fc8061b621a15ad2c5f7cc16d0ea9dbaa7049f60c806d88673eaeffbe0
 SHA512:
-  metadata.gz: e2ad758ddb7d644777df1b9f38b73c7c4c0753d76507834bf68e86394ae017b1f5caa16867efd82f4c0443d4a7c1b4c1d7f704d9c3796e71083bde689ee7aeba
-  data.tar.gz: 8f7126500628a17fda587a05678aca9140d0ebb7ca168fc3144b1d9c00d2be6ec901a43900bd3d558aceb0a8dcfc139aba7347e6dbad01b2ff83c19dc3816763
+  metadata.gz: 7ab27854092a7b93aee3f596eeb5e3757cdfe0cf15fa95ff1e131268e02ce934fbbe2e7f25d786014f04222e8e5147821122f84fed9a9178bae555a13bafb505
+  data.tar.gz: 10129052b5e02a7b48a5e37aeb073abaceb43c526e05de51f84442d9a8f059caf3db530b43c59b44716eb688d8b76a1429b4f0a1e317375f8fc480004933999c

data/.travis.yml

@@ -6,10 +6,12 @@ before_install:
 
 matrix:
   include:
-    - rvm: 2.2.8
-    - rvm: 2.3.5
-    - rvm: 2.4.2
-    - rvm: jruby-9.1.13.0
+    - rvm: 2.2.10
+    - rvm: 2.3.8
+    - rvm: 2.4.6
+    - rvm: 2.5.5
+    - rvm: 2.6.3
+    - rvm: jruby-9.2.7.0
       env:
         - JRUBY_OPTS="--server -Xcompile.invokedynamic=false -J-XX:+TieredCompilation -J-XX:TieredStopAtLevel=1 -J-noverify -J-Xms512m -J-Xmx1024m"
     - rvm: ruby-head
@@ -17,3 +19,5 @@ matrix:
 branches:
   only:
     - master
+    - 1.2-stable
+    - 2.0-develop

data/Gemfile

@@ -3,7 +3,6 @@ source 'https://rubygems.org'
 # Specify your gem's dependencies in smarter_csv.gemspec
 gemspec
 
-
 gem "rake", "< 11"
 gem 'pry'
 

data/README.md

@@ -2,6 +2,19 @@
 
 [![Build Status](https://secure.travis-ci.org/tilo/smarter_csv.svg?branch=master)](http://travis-ci.org/tilo/smarter_csv) [![Gem Version](https://badge.fury.io/rb/smarter_csv.svg)](http://badge.fury.io/rb/smarter_csv)
 
+---------------
+#### Service Announcement
+
+Work towards SmarterCSV 2.0 is on it's way, with much improved features, and more streamlined options.
+
+Please check the 2.0-develop branch, and open issues marked v2.0 and leave your comments.
+
+New versions on the 1.2 branch will soon print a deprecation warning if you set :verbose to true
+See below for list of deprecated options.
+
+---------------
+#### SmarterCSV
+
 `smarter_csv` is a Ruby Gem for smarter importing of CSV Files as Array(s) of Hashes, suitable for direct processing with Mongoid or ActiveRecord,
 and parallel processing with Resque or Sidekiq.
 
@@ -182,18 +195,48 @@ The options and the block are optional.
 
 `SmarterCSV.process` supports the following options:
 
+#### Options:
+
      | Option                      | Default  |  Explanation                                                                         |
      ---------------------------------------------------------------------------------------------------------------------------------
+     | :chunk_size                 |   nil    | if set, determines the desired chunk-size (defaults to nil, no chunk processing)     |
+     |                             |          |                                                                                      |
+     | :file_encoding              |   utf-8  | Set the file encoding eg.: 'windows-1252' or 'iso-8859-1'                            |
+     | :invalid_byte_sequence      |   ''     | what to replace invalid byte sequences with                                          |
+     | :force_utf8                 |   false  | force UTF-8 encoding of all lines (including headers) in the CSV file                |
+     | :skip_lines                 |   nil    | how many lines to skip before the first line or header line is processed             |
+     | :comment_regexp             |   /^#/   | regular expression which matches comment lines (see NOTE about the CSV header)       |
+     ---------------------------------------------------------------------------------------------------------------------------------
      | :col_sep                    |   ','    | column separator                                                                     |
+     | :force_simple_split         |   false  | force simple splitting on :col_sep character for non-standard CSV-files.             |
+     |                             |          | e.g. when :quote_char is not properly escaped                                        |
      | :row_sep                    | $/ ,"\n" | row separator or record separator , defaults to system's $/ , which defaults to "\n" |
      |                             |          | This can also be set to :auto, but will process the whole cvs file first  (slow!)    |
      | :auto_row_sep_chars         |   500    | How many characters to analyze when using `:row_sep => :auto`. nil or 0 means whole file. |
      | :quote_char                 |   '"'    | quotation character                                                                  |
-     | :comment_regexp             |   /^#/   | regular expression which matches comment lines (see NOTE about the CSV header)       |
-     | :chunk_size                 |   nil    | if set, determines the desired chunk-size (defaults to nil, no chunk processing)     |
+     ---------------------------------------------------------------------------------------------------------------------------------
+     | :headers_in_file            |   true   | Whether or not the file contains headers as the first line.                          |
+     |                             |          | Important if the file does not contain headers,                                      |
+     |                             |          | otherwise you would lose the first line of data.                                     |
+     | :user_provided_headers      |   nil    | *careful with that axe!*                                                             |
+     |                             |          | user provided Array of header strings or symbols, to define                          |
+     |                             |          | what headers should be used, overriding any in-file headers.                         |
+     |                             |          | You can not combine the :user_provided_headers and :key_mapping options              |
+     | :remove_empty_hashes        |   true   | remove / ignore any hashes which don't have any key/value pairs                      |
+     | :verbose                    |   false  | print out line number while processing (to track down problems in input files)       |
+     ---------------------------------------------------------------------------------------------------------------------------------
+
+#### Deprecated 1.x Options: to be replaced in 2.0
+
+There have been a lot of 1-offs and feature creep around these options, and going forward we'll have a simpler, but more flexible way to address these features.
+
+Instead of these options, there will be a new and more flexible way to process the header fields, as well as the fields in each line of the CSV.
+And header and data validations will also be supported in 2.x
+
+     | Option                      | Default  |  Explanation                                                                         |
      ---------------------------------------------------------------------------------------------------------------------------------
      | :key_mapping                |   nil    | a hash which maps headers from the CSV file to keys in the result hash               |
-     | :required_headers           |   nil    | An array. Eacn of the given headers must be present in the CSV  file,                |
+     | :required_headers           |   nil    | An array. Eacn of the given headers must be present after header manipulation,       |
      |                             |          | or an exception is raised   No validation if nil is given.                           |
      | :remove_unmapped_keys       |   false  | when using :key_mapping option, should non-mapped keys / columns be removed?         |
      | :downcase_header            |   true   | downcase all column headers                                                          |
@@ -201,17 +244,7 @@ The options and the block are optional.
      | :strip_whitespace           |   true   | remove whitespace before/after values and headers                                    |
      | :keep_original_headers      |   false  | keep the original headers from the CSV-file as-is.                                   |
      |                             |          | Disables other flags manipulating the header fields.                                 |
-     | :user_provided_headers      |   nil    | *careful with that axe!*                                                             |
-     |                             |          | user provided Array of header strings or symbols, to define                          |
-     |                             |          | what headers should be used, overriding any in-file headers.                         |
-     |                             |          | You can not combine the :user_provided_headers and :key_mapping options              |
      | :strip_chars_from_headers   |   nil    | RegExp to remove extraneous characters from the header line (e.g. if headers are quoted) |
-     | :headers_in_file            |   true   | Whether or not the file contains headers as the first line.                          |
-     |                             |          | Important if the file does not contain headers,                                      |
-     |                             |          | otherwise you would lose the first line of data.                                     |
-     | :skip_lines                 |   nil    | how many lines to skip before the first line or header line is processed             |
-     | :force_utf8                 |   false  | force UTF-8 encoding of all lines (including headers) in the CSV file                |
-     | :invalid_byte_sequence      |   ''     | how to replace invalid byte sequences with                                           |
      ---------------------------------------------------------------------------------------------------------------------------------
      | :value_converters           |   nil    | supply a hash of :header => KlassName; the class needs to implement self.convert(val)|
      | :remove_empty_values        |   true   | remove values which have nil or empty strings as values                              |
@@ -220,11 +253,7 @@ The options and the block are optional.
      |                             |          | /^\$0\.0+$/ to match $0.00 , or /^#VALUE!$/ to match errors in Excel spreadsheets    |
      | :convert_values_to_numeric  |   true   | converts strings containing Integers or Floats to the appropriate class              |
      |                             |          |      also accepts either {:except => [:key1,:key2]} or {:only => :key3}              |
-     | :remove_empty_hashes        |   true   | remove / ignore any hashes which don't have any key/value pairs                      |
-     | :file_encoding              |   utf-8  | Set the file encoding eg.: 'windows-1252' or 'iso-8859-1'                            |
-     | :force_simple_split         |   false  | force simple splitting on :col_sep character for non-standard CSV-files.            |
-     |                             |          | e.g. when :quote_char is not properly escaped                                        |
-     | :verbose                    |   false  | print out line number while processing (to track down problems in input files)       |
+     ---------------------------------------------------------------------------------------------------------------------------------
 
 
 #### NOTES about File Encodings:
@@ -295,6 +324,21 @@ Planned in the next releases:
 
 ## Changes
 
+#### 1.2.6 (2018-11-13)
+ * fixing error caused by calling f.close when we do not hand in a file
+
+#### 1.2.5 (2018-09-16)
+ * fixing issue #136 with comments in CSV files
+ * fixing error class hierarchy
+
+#### 1.2.4 (2018-08-06)
+ * using Rails blank? if it's available
+
+#### 1.2.3 (2018-01-27)
+ * fixed regression / test
+ * fuxed quote_char interpolation for headers, but not data (thanks to Colin Petruno)
+ * bugfix (thanks to Joshua Smith for reporting)
+
 #### 1.2.0 (2018-01-20)
  * add default validation that a header can only appear once
  * add option `required_headers`
@@ -349,9 +393,6 @@ Planned in the next releases:
 #### 1.0.14 (2013-11-01)
  * added GPL-2 and MIT license to GEM spec file; if you need another license contact me
 
-#### 1.0.13 (2013-11-01)    ### YANKED!
- * added GPL-2 license to GEM spec file; if you need another license contact me
-
 #### 1.0.12 (2013-10-15)
  * added RSpec tests
 
@@ -465,6 +506,8 @@ And a special thanks to those who contributed pull requests:
  * [Ivan Ushakov](https://github.com/IvanUshakov)
  * [Matthieu Paret](https://github.com/mtparet)
  * [Rohit Amarnath](https://github.com/ramarnat)
+ * [Joshua Smith](https://github.com/enviable)
+ * [Colin Petruno](https://github.com/colinpetruno)
 
 
 ## Contributing

data/lib/smarter_csv/smarter_csv.rb

@@ -1,16 +1,16 @@
 module SmarterCSV
-
-  class HeaderSizeMismatch < Exception; end
-  class IncorrectOption < Exception; end
-  class DuplicateHeaders < Exception; end
-  class MissingHeaders < Exception; end
+  class SmarterCSVException < StandardError; end
+  class HeaderSizeMismatch < SmarterCSVException; end
+  class IncorrectOption < SmarterCSVException; end
+  class DuplicateHeaders < SmarterCSVException; end
+  class MissingHeaders < SmarterCSVException; end
 
 
   def SmarterCSV.process(input, options={}, &block)   # first parameter: filename or input object with readline method
     default_options = {:col_sep => ',' , :row_sep => $/ , :quote_char => '"', :force_simple_split => false , :verbose => false ,
       :remove_empty_values => true, :remove_zero_values => false , :remove_values_matching => nil , :remove_empty_hashes => true , :strip_whitespace => true,
       :convert_values_to_numeric => true, :strip_chars_from_headers => nil , :user_provided_headers => nil , :headers_in_file => true,
-      :comment_regexp => /^#/, :chunk_size => nil , :key_mapping_hash => nil , :downcase_header => true, :strings_as_keys => false, :file_encoding => 'utf-8',
+      :comment_regexp => /\A#/, :chunk_size => nil , :key_mapping_hash => nil , :downcase_header => true, :strings_as_keys => false, :file_encoding => 'utf-8',
       :remove_unmapped_keys => false, :keep_original_headers => false, :value_converters => nil, :skip_lines => nil, :force_utf8 => false, :invalid_byte_sequence => '',
       :auto_row_sep_chars => 500, :required_headers => nil
     }
@@ -22,6 +22,7 @@ module SmarterCSV
     old_row_sep = $/
     file_line_count = 0
     csv_line_count = 0
+    has_rails = !! defined?(Rails)
     begin
       f = input.respond_to?(:readline) ? input : File.open(input, "r:#{options[:file_encoding]}")
 
@@ -59,7 +60,7 @@ module SmarterCSV
         else
           file_headerA =  header.split(options[:col_sep])
         end
-        file_headerA.map!{|x| x.gsub(%r/options[:quote_char]/,'') }
+        file_headerA.map!{|x| x.gsub(%r/#{options[:quote_char]}/,'') }
         file_headerA.map!{|x| x.strip}  if options[:strip_whitespace]
         unless options[:keep_original_headers]
           file_headerA.map!{|x| x.gsub(/\s+|-+/,'_')}
@@ -68,14 +69,14 @@ module SmarterCSV
 
         file_header_size = file_headerA.size
       else
-        raise SmarterCSV::IncorrectOption , "ERROR [smarter_csv]: If :headers_in_file is set to false, you have to provide :user_provided_headers" if options[:user_provided_headers].nil?
+        raise SmarterCSV::IncorrectOption , "ERROR: If :headers_in_file is set to false, you have to provide :user_provided_headers" if options[:user_provided_headers].nil?
       end
       if options[:user_provided_headers] && options[:user_provided_headers].class == Array && ! options[:user_provided_headers].empty?
         # use user-provided headers
         headerA = options[:user_provided_headers]
         if defined?(file_header_size) && ! file_header_size.nil?
           if headerA.size != file_header_size
-            raise SmarterCSV::HeaderSizeMismatch , "ERROR [smarter_csv]: :user_provided_headers defines #{headerA.size} headers !=  CSV-file #{input} has #{file_header_size} headers"
+            raise SmarterCSV::HeaderSizeMismatch , "ERROR: :user_provided_headers defines #{headerA.size} headers !=  CSV-file #{input} has #{file_header_size} headers"
           else
             # we could print out the mapping of file_headerA to headerA here
           end
@@ -100,14 +101,14 @@ module SmarterCSV
       headerA.compact.each do |k|
         duplicate_headers << k if headerA.select{|x| x == k}.size > 1
       end
-      raise SmarterCSV::DuplicateHeaders , "ERORR [smarter_csv]: duplicate headers: #{duplicate_headers.join(',')}" unless duplicate_headers.empty?
+      raise SmarterCSV::DuplicateHeaders , "ERROR: duplicate headers: #{duplicate_headers.join(',')}" unless duplicate_headers.empty?
 
       if options[:required_headers] && options[:required_headers].is_a?(Array)
         missing_headers = []
         options[:required_headers].each do |k|
           missing_headers << k unless headerA.include?(k)
         end
-        raise SmarterCSV::MissingHeaders , "ERORR [smarter_csv]: missing headers: #{missing_headers.join(',')}" unless missing_headers.empty?
+        raise SmarterCSV::MissingHeaders , "ERROR: missing headers: #{missing_headers.join(',')}" unless missing_headers.empty?
       end
 
       # in case we use chunking.. we'll need to set it up..
@@ -155,7 +156,7 @@ module SmarterCSV
         else
           dataA =  line.split(options[:col_sep])
         end
-        dataA.map!{|x| x.gsub(%r/options[:quote_char]/,'') }
+####     dataA.map!{|x| x.gsub(%r/#{options[:quote_char]}/,'') }  # this is actually not a good idea as a default
         dataA.map!{|x| x.strip}  if options[:strip_whitespace]
         hash = Hash.zip(headerA,dataA)  # from Facets of Ruby library
         # make sure we delete any key/value pairs from the hash, which the user wanted to delete:
@@ -167,7 +168,13 @@ module SmarterCSV
 
         # remove empty values using the same regexp as used by the rails blank? method
         # which caters for double \n and \r\n characters such as "1\r\n\r\n2" whereas the original check (v =~ /^\s*$/) does not
-        hash.delete_if{|k,v| v.nil? || v !~ /[^[:space:]]/}  if options[:remove_empty_values]
+        if options[:remove_empty_values]
+          if has_rails
+            hash.delete_if{|k,v| v.blank?}
+          else
+            hash.delete_if{|k,v| v.nil? || v !~ /[^[:space:]]/}
+          end
+        end
 
         hash.delete_if{|k,v| ! v.nil? && v =~ /^(\d+|\d+\.\d+)$/ && v.to_f == 0} if options[:remove_zero_values]   # values are typically Strings!
         hash.delete_if{|k,v| v =~ options[:remove_values_matching]} if options[:remove_values_matching]
@@ -241,7 +248,7 @@ module SmarterCSV
       end
     ensure
       $/ = old_row_sep   # make sure this stupid global variable is always reset to it's previous value after we're done!
-      f.close
+      f.close if f.respond_to?(:close)
     end
     if block_given?
       return chunk_count  # when we do processing through a block we only care how many chunks we processed

data/lib/smarter_csv/version.rb

@@ -1,3 +1,3 @@
 module SmarterCSV
-  VERSION = "1.2.0"
+  VERSION = "1.2.7"
 end

data/spec/fixtures/ignore_comments.csv

@@ -0,0 +1,11 @@
+not a comment#First Name,Last Name,Dogs,Cats,Birds,Fish
+# comment two
+Dan,McAllister,2,0,,
+Lucy#L,Laweless,,5,0,
+# anothter comment
+,,,,,
+Miles,O'Brian,0,0,0,21
+Nancy,Homes,2,0,1,
+Hernán,Curaçon,3,0,0,
+#comment,comment,1,2,3,4
+,,,,,

data/spec/fixtures/ignore_comments2.csv

@@ -0,0 +1,3 @@
+h1,h2
+a,"b
+#c"

data/spec/fixtures/problematic.csv

@@ -0,0 +1,8 @@
+Compte;Date de comptabilisation;Date op�ration;Libell�;R�f�rence;Date valeur;Montant
+22215449203;02/06/2018;01/06/2018;ECHEANCE PRET DONT CAP    410,33 ASS.    8,00E INT.    21,87 COM.    0,00E;8711552;01/06/2018;-440,20;
+22215449203;04/06/2018;04/06/2018;EVI Gaultier Laperche remboursement compte courant;1038326;04/06/2018;-144,07;
+22215449203;04/06/2018;04/06/2018;EVI Guillemain Nicolas remboursement CC pret d'honneur;1038328;04/06/2018;-144,07;
+22215449203;01/06/2018;01/06/2018;310518 SC****5448 INTERMARCHE    95ERMONT;701JQ1K;01/06/2018;-16,00;
+22215449203;01/06/2018;01/06/2018;EVI Stripe Payments UK L STRIPE E7U0R1;706AO1Q;01/06/2018;45,89;
+22215449203;01/06/2018;01/06/2018;EVI Compte N26 Heroku;1100653;01/06/2018;-700,00;
+22215449203;31/05/2018;31/05/2018;EVI Stripe Payments UK L STRIPE L2J1N7;6YISBWF;31/05/2018;465,89;

data/spec/fixtures/quote_char.csv

@@ -0,0 +1,9 @@
+"ID","FIRST_NAME","LAST_NAME"
+"1","""John","Cooke"""
+"2","Jam
+e
+son""","McCollum"
+"3","""Jean","Conn"
+"4","Jenny","Traer"
+"5","Bo""bbie","Faga"
+"6","Mica","Copeland"

data/spec/fixtures/quoted2.csv

@@ -0,0 +1,4 @@
+"REVIEW DATE","AUTHOR","""ISBN""","DISCOUNTED ""PRICE"""
+"1985/01/21","Douglas Adams",0345391802,5.95
+"1998/07/15","Timothy ""The Parser"" Campbell",0968411304,18.99
+"1999/12/03","Richard Friedman",0060630353,5.95

data/spec/fixtures/trading.csv

@@ -0,0 +1,3 @@
+Account_ID,options_trader,Stock_Symbol,Shares Issued,Purchase Date
+0002310234,Mike Smith,TSLA,2300,2011-08-19
+0024923423,John Doe,AAPL,1300,2013-03-21

data/spec/smarter_csv/ignore_comments_spec.rb

@@ -0,0 +1,30 @@
+require 'spec_helper'
+
+fixture_path = 'spec/fixtures'
+
+describe 'be_able_to' do
+  it 'ignore comments in CSV files' do
+    options = {}
+    data = SmarterCSV.process("#{fixture_path}/ignore_comments.csv", options)
+
+    data.size.should eq 5
+
+    # all the keys should be symbols
+    data.each{|item| item.keys.each{|x| x.is_a?(Symbol).should be_truthy}}
+    data.each do |h|
+      h.keys.each do |key|
+        [:"not_a_comment#first_name", :last_name, :dogs, :cats, :birds, :fish].should include( key )
+      end
+    end
+  end
+
+  it 'ignore comments in CSV files with CRLF' do
+    options = {row_sep: "\r\n"}
+    data = SmarterCSV.process("#{fixture_path}/ignore_comments2.csv", options)
+
+    # all the keys should be symbols
+    data.size.should eq 1
+    data.first[:h1].should eq 'a'
+    data.first[:h2].should eq "b\r\n#c"
+  end
+end

data/spec/smarter_csv/problematic.rb

@@ -0,0 +1,34 @@
+require 'spec_helper'
+
+fixture_path = 'spec/fixtures'
+
+describe 'loading file with UTF-8 characters in the header' do
+
+  # file which caused issues because of UTF-8 characters in the header
+  it 'loads the file with force_utf8 flag set' do
+    options = {col_sep:  ";", force_utf8: true}
+    data = SmarterCSV.process("#{fixture_path}/problematic.csv", options)
+
+    data.length.should eq 7
+  end
+
+  it 'loads the file with strings as keys' do
+    options = {
+      file_encoding: 'iso-8859-1:UTF-8', # important!
+      col_sep:  ";", strings_as_keys: true,
+    }
+    data = SmarterCSV.process("#{fixture_path}/problematic.csv", options)
+
+    data.length.should eq 7
+    data.first.keys.sort.should eq [
+      "compte",
+      "date_de_comptabilisation",
+      "date_opération",
+      "date_valeur",
+      "libellé",
+      "montant",
+      "référence"
+    ]
+  end
+
+end

data/spec/smarter_csv/quoted_spec.rb

@@ -2,9 +2,9 @@ require 'spec_helper'
 
 fixture_path = 'spec/fixtures'
 
-describe 'be_able_to' do
+describe 'loading file with quoted fields' do
 
-  it 'loads_file_with_quoted_fields' do
+  it 'leaving the quotes in the data' do
     options = {}
     data = SmarterCSV.process("#{fixture_path}/quoted.csv", options)
     data.flatten.size.should == 4
@@ -20,4 +20,29 @@ describe 'be_able_to' do
     end
   end
 
+
+  it 'removes quotes around quoted fields, but not inside data' do
+    options = {}
+    data = SmarterCSV.process("#{fixture_path}/quote_char.csv", options)
+
+    data.length.should eq 6
+    data[1][:first_name].should eq "Jam\ne\nson\""
+    data[2][:first_name].should eq "\"Jean"
+  end
+
+
+  # NOTE: quotes inside headers need to be escaped by doubling them
+  #       e.g. 'correct ""EXAMPLE""'
+  #       this escaping is illegal: 'incorrect \"EXAMPLE\"' <-- this caused CSV parsing error
+  #  in case of CSV parsing errirs, use :user_provided_headers, or key_mapping
+  #
+  it 'removes quotes around headers and extra quotes inside headers' do
+    options = {}
+    data = SmarterCSV.process("#{fixture_path}/quoted2.csv", options)
+
+    data.length.should eq 3
+    data.first.keys[2].should eq :isbn
+    data.first.keys[3].should eq :discounted_price
+  end
+
 end

data/spec/smarter_csv/trading_spec.rb

@@ -0,0 +1,25 @@
+require 'spec_helper'
+
+fixture_path = 'spec/fixtures'
+
+# somebody reported that a column called 'options_trader' would be truncated to 'trader'
+
+describe 'loads simple file format' do
+
+  it 'with symbols as keys when using defaults' do
+    options = {}
+    data = SmarterCSV.process("#{fixture_path}/trading.csv", options)
+
+    data.flatten.size.should eq 2
+    data.each do |item|
+      # all keys should be symbols when using v1.x backwards compatible mode
+      item.keys.each{|x| x.class.should eq Symbol}
+      item[:account_id].class.should eq Fixnum
+      item[:options_trader].class.should eq String
+      item[:stock_symbol].class.should eq String
+      item[:shares_issued].class.should eq Fixnum
+      item[:purchase_date].class.should eq String
+    end
+  end
+
+end

metadata

@@ -1,16 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: smarter_csv
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.2.7
 platform: ruby
 authors:
 - 'Tilo Sloboda
 
-'
-autorequire: 
+  '
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-01-20 00:00:00.000000000 Z
+date: 2021-02-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -32,7 +32,7 @@ description: Ruby Gem for smarter importing of CSV Files as Array(s) of Hashes,
 email:
 - 'tilo.sloboda@gmail.com
 
-'
+  '
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -58,6 +58,8 @@ files:
 - spec/fixtures/chunk_cornercase.csv
 - spec/fixtures/duplicate_headers.csv
 - spec/fixtures/empty.csv
+- spec/fixtures/ignore_comments.csv
+- spec/fixtures/ignore_comments2.csv
 - spec/fixtures/line_endings_n.csv
 - spec/fixtures/line_endings_r.csv
 - spec/fixtures/line_endings_rn.csv
@@ -68,9 +70,13 @@ files:
 - spec/fixtures/no_header.csv
 - spec/fixtures/numeric.csv
 - spec/fixtures/pets.csv
+- spec/fixtures/problematic.csv
+- spec/fixtures/quote_char.csv
 - spec/fixtures/quoted.csv
+- spec/fixtures/quoted2.csv
 - spec/fixtures/separator.csv
 - spec/fixtures/skip_lines.csv
+- spec/fixtures/trading.csv
 - spec/fixtures/user_import.csv
 - spec/fixtures/valid_unicode.csv
 - spec/fixtures/with_dashes.csv
@@ -84,6 +90,7 @@ files:
 - spec/smarter_csv/convert_values_to_numeric_spec.rb
 - spec/smarter_csv/extenstions_spec.rb
 - spec/smarter_csv/header_transformation_spec.rb
+- spec/smarter_csv/ignore_comments_spec.rb
 - spec/smarter_csv/invalid_headers_spec.rb
 - spec/smarter_csv/keep_headers_spec.rb
 - spec/smarter_csv/key_mapping_spec.rb
@@ -92,6 +99,7 @@ files:
 - spec/smarter_csv/malformed_spec.rb
 - spec/smarter_csv/no_header_spec.rb
 - spec/smarter_csv/not_downcase_header_spec.rb
+- spec/smarter_csv/problematic.rb
 - spec/smarter_csv/quoted_spec.rb
 - spec/smarter_csv/remove_empty_values_spec.rb
 - spec/smarter_csv/remove_keys_from_hashes_spec.rb
@@ -101,6 +109,7 @@ files:
 - spec/smarter_csv/skip_lines_spec.rb
 - spec/smarter_csv/strings_as_keys_spec.rb
 - spec/smarter_csv/strip_chars_from_headers_spec.rb
+- spec/smarter_csv/trading_spec.rb
 - spec/smarter_csv/valid_unicode_spec.rb
 - spec/smarter_csv/value_converters_spec.rb
 - spec/spec.opts
@@ -111,7 +120,7 @@ licenses:
 - MIT
 - GPL-2
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -127,9 +136,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements:
 - csv
-rubyforge_project: 
-rubygems_version: 2.6.13
-signing_key: 
+rubygems_version: 3.0.6
+signing_key:
 specification_version: 4
 summary: Ruby Gem for smarter importing of CSV Files (and CSV-like files), with lots
   of optional features, e.g. chunked processing for huge CSV files
@@ -143,6 +151,8 @@ test_files:
 - spec/fixtures/chunk_cornercase.csv
 - spec/fixtures/duplicate_headers.csv
 - spec/fixtures/empty.csv
+- spec/fixtures/ignore_comments.csv
+- spec/fixtures/ignore_comments2.csv
 - spec/fixtures/line_endings_n.csv
 - spec/fixtures/line_endings_r.csv
 - spec/fixtures/line_endings_rn.csv
@@ -153,9 +163,13 @@ test_files:
 - spec/fixtures/no_header.csv
 - spec/fixtures/numeric.csv
 - spec/fixtures/pets.csv
+- spec/fixtures/problematic.csv
+- spec/fixtures/quote_char.csv
 - spec/fixtures/quoted.csv
+- spec/fixtures/quoted2.csv
 - spec/fixtures/separator.csv
 - spec/fixtures/skip_lines.csv
+- spec/fixtures/trading.csv
 - spec/fixtures/user_import.csv
 - spec/fixtures/valid_unicode.csv
 - spec/fixtures/with_dashes.csv
@@ -169,6 +183,7 @@ test_files:
 - spec/smarter_csv/convert_values_to_numeric_spec.rb
 - spec/smarter_csv/extenstions_spec.rb
 - spec/smarter_csv/header_transformation_spec.rb
+- spec/smarter_csv/ignore_comments_spec.rb
 - spec/smarter_csv/invalid_headers_spec.rb
 - spec/smarter_csv/keep_headers_spec.rb
 - spec/smarter_csv/key_mapping_spec.rb
@@ -177,6 +192,7 @@ test_files:
 - spec/smarter_csv/malformed_spec.rb
 - spec/smarter_csv/no_header_spec.rb
 - spec/smarter_csv/not_downcase_header_spec.rb
+- spec/smarter_csv/problematic.rb
 - spec/smarter_csv/quoted_spec.rb
 - spec/smarter_csv/remove_empty_values_spec.rb
 - spec/smarter_csv/remove_keys_from_hashes_spec.rb
@@ -186,6 +202,7 @@ test_files:
 - spec/smarter_csv/skip_lines_spec.rb
 - spec/smarter_csv/strings_as_keys_spec.rb
 - spec/smarter_csv/strip_chars_from_headers_spec.rb
+- spec/smarter_csv/trading_spec.rb
 - spec/smarter_csv/valid_unicode_spec.rb
 - spec/smarter_csv/value_converters_spec.rb
 - spec/spec.opts