smarter_csv 1.2.0 → 1.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 251eb1eff306211d163c6be6880ba45e3eb29985
-  data.tar.gz: 9e16d1d4aaab86df7a65d73e6a23ace8f481ba4e
+SHA256:
+  metadata.gz: c0efcd4dbad2546469ba99c1a133bc7548382e147fca66207d11065607fdb5ee
+  data.tar.gz: 98024633925ed73251dd00606f23f73ca6cbda5517c24882b95935138ab4d7e3
 SHA512:
-  metadata.gz: e2ad758ddb7d644777df1b9f38b73c7c4c0753d76507834bf68e86394ae017b1f5caa16867efd82f4c0443d4a7c1b4c1d7f704d9c3796e71083bde689ee7aeba
-  data.tar.gz: 8f7126500628a17fda587a05678aca9140d0ebb7ca168fc3144b1d9c00d2be6ec901a43900bd3d558aceb0a8dcfc139aba7347e6dbad01b2ff83c19dc3816763
+  metadata.gz: ac64f3a688f9b5b4bce09e26d8ee140994bf585578ea6eed1d998cec434535f3afb4aa8c2ea222b873cb5ee113fba748412a96e1b3b23a8b5425d13aa7a6436c
+  data.tar.gz: f8c3132f1f2cf60f8f67d2278d0b3660d7aab456581256e1c2508d1397c1cc31d455035c68656777c34e90ad6238c239699b1c59cdd918cdcd63c9f5a264aaba

data/README.md

@@ -2,6 +2,19 @@
 
 [![Build Status](https://secure.travis-ci.org/tilo/smarter_csv.svg?branch=master)](http://travis-ci.org/tilo/smarter_csv) [![Gem Version](https://badge.fury.io/rb/smarter_csv.svg)](http://badge.fury.io/rb/smarter_csv)
 
+---------------
+#### Service Announcement
+
+Work towards SmarterCSV 2.0 is on it's way, with much improved features, and more streamlined options.
+
+Please check the 2.0-develop branch, and open issues marked v2.0 and leave your comments.
+
+New versions on the 1.2 branch will soon print a deprecation warning if you set :verbose to true
+See below for list of deprecated options.
+
+---------------
+#### SmarterCSV
+
 `smarter_csv` is a Ruby Gem for smarter importing of CSV Files as Array(s) of Hashes, suitable for direct processing with Mongoid or ActiveRecord,
 and parallel processing with Resque or Sidekiq.
 
@@ -182,18 +195,48 @@ The options and the block are optional.
 
 `SmarterCSV.process` supports the following options:
 
+#### Options:
+
      | Option                      | Default  |  Explanation                                                                         |
      ---------------------------------------------------------------------------------------------------------------------------------
+     | :chunk_size                 |   nil    | if set, determines the desired chunk-size (defaults to nil, no chunk processing)     |
+     |                             |          |                                                                                      |
+     | :file_encoding              |   utf-8  | Set the file encoding eg.: 'windows-1252' or 'iso-8859-1'                            |
+     | :invalid_byte_sequence      |   ''     | what to replace invalid byte sequences with                                          |
+     | :force_utf8                 |   false  | force UTF-8 encoding of all lines (including headers) in the CSV file                |
+     | :skip_lines                 |   nil    | how many lines to skip before the first line or header line is processed             |
+     | :comment_regexp             |   /^#/   | regular expression which matches comment lines (see NOTE about the CSV header)       |
+     ---------------------------------------------------------------------------------------------------------------------------------
      | :col_sep                    |   ','    | column separator                                                                     |
+     | :force_simple_split         |   false  | force simple splitting on :col_sep character for non-standard CSV-files.             |
+     |                             |          | e.g. when :quote_char is not properly escaped                                        |
      | :row_sep                    | $/ ,"\n" | row separator or record separator , defaults to system's $/ , which defaults to "\n" |
      |                             |          | This can also be set to :auto, but will process the whole cvs file first  (slow!)    |
      | :auto_row_sep_chars         |   500    | How many characters to analyze when using `:row_sep => :auto`. nil or 0 means whole file. |
      | :quote_char                 |   '"'    | quotation character                                                                  |
-     | :comment_regexp             |   /^#/   | regular expression which matches comment lines (see NOTE about the CSV header)       |
-     | :chunk_size                 |   nil    | if set, determines the desired chunk-size (defaults to nil, no chunk processing)     |
+     ---------------------------------------------------------------------------------------------------------------------------------
+     | :headers_in_file            |   true   | Whether or not the file contains headers as the first line.                          |
+     |                             |          | Important if the file does not contain headers,                                      |
+     |                             |          | otherwise you would lose the first line of data.                                     |
+     | :user_provided_headers      |   nil    | *careful with that axe!*                                                             |
+     |                             |          | user provided Array of header strings or symbols, to define                          |
+     |                             |          | what headers should be used, overriding any in-file headers.                         |
+     |                             |          | You can not combine the :user_provided_headers and :key_mapping options              |
+     | :remove_empty_hashes        |   true   | remove / ignore any hashes which don't have any key/value pairs                      |
+     | :verbose                    |   false  | print out line number while processing (to track down problems in input files)       |
+     ---------------------------------------------------------------------------------------------------------------------------------
+
+#### Deprecated 1.x Options: to be replaced in 2.0
+
+There have been a lot of 1-offs and feature creep around these options, and going forward we'll have a simpler, but more flexible way to address these features.
+
+Instead of these options, there will be a new and more flexible way to process the header fields, as well as the fields in each line of the CSV.
+And header and data validations will also be supported in 2.x
+
+     | Option                      | Default  |  Explanation                                                                         |
      ---------------------------------------------------------------------------------------------------------------------------------
      | :key_mapping                |   nil    | a hash which maps headers from the CSV file to keys in the result hash               |
-     | :required_headers           |   nil    | An array. Eacn of the given headers must be present in the CSV  file,                |
+     | :required_headers           |   nil    | An array. Eacn of the given headers must be present after header manipulation,       |
      |                             |          | or an exception is raised   No validation if nil is given.                           |
      | :remove_unmapped_keys       |   false  | when using :key_mapping option, should non-mapped keys / columns be removed?         |
      | :downcase_header            |   true   | downcase all column headers                                                          |
@@ -201,17 +244,7 @@ The options and the block are optional.
      | :strip_whitespace           |   true   | remove whitespace before/after values and headers                                    |
      | :keep_original_headers      |   false  | keep the original headers from the CSV-file as-is.                                   |
      |                             |          | Disables other flags manipulating the header fields.                                 |
-     | :user_provided_headers      |   nil    | *careful with that axe!*                                                             |
-     |                             |          | user provided Array of header strings or symbols, to define                          |
-     |                             |          | what headers should be used, overriding any in-file headers.                         |
-     |                             |          | You can not combine the :user_provided_headers and :key_mapping options              |
      | :strip_chars_from_headers   |   nil    | RegExp to remove extraneous characters from the header line (e.g. if headers are quoted) |
-     | :headers_in_file            |   true   | Whether or not the file contains headers as the first line.                          |
-     |                             |          | Important if the file does not contain headers,                                      |
-     |                             |          | otherwise you would lose the first line of data.                                     |
-     | :skip_lines                 |   nil    | how many lines to skip before the first line or header line is processed             |
-     | :force_utf8                 |   false  | force UTF-8 encoding of all lines (including headers) in the CSV file                |
-     | :invalid_byte_sequence      |   ''     | how to replace invalid byte sequences with                                           |
      ---------------------------------------------------------------------------------------------------------------------------------
      | :value_converters           |   nil    | supply a hash of :header => KlassName; the class needs to implement self.convert(val)|
      | :remove_empty_values        |   true   | remove values which have nil or empty strings as values                              |
@@ -220,11 +253,7 @@ The options and the block are optional.
      |                             |          | /^\$0\.0+$/ to match $0.00 , or /^#VALUE!$/ to match errors in Excel spreadsheets    |
      | :convert_values_to_numeric  |   true   | converts strings containing Integers or Floats to the appropriate class              |
      |                             |          |      also accepts either {:except => [:key1,:key2]} or {:only => :key3}              |
-     | :remove_empty_hashes        |   true   | remove / ignore any hashes which don't have any key/value pairs                      |
-     | :file_encoding              |   utf-8  | Set the file encoding eg.: 'windows-1252' or 'iso-8859-1'                            |
-     | :force_simple_split         |   false  | force simple splitting on :col_sep character for non-standard CSV-files.            |
-     |                             |          | e.g. when :quote_char is not properly escaped                                        |
-     | :verbose                    |   false  | print out line number while processing (to track down problems in input files)       |
+     ---------------------------------------------------------------------------------------------------------------------------------
 
 
 #### NOTES about File Encodings:
@@ -295,6 +324,13 @@ Planned in the next releases:
 
 ## Changes
 
+#### 1.2.3 (2018-01-27)
+ * fixed regression / test
+ * fuxed quote_char interpolation for headers, but not data (thanks to Colin Petruno)
+
+#### 1.2.1 (2018-01-25)    ### YANKED!
+ * bugfix (thanks to Joshua Smith for reporting)
+
 #### 1.2.0 (2018-01-20)
  * add default validation that a header can only appear once
  * add option `required_headers`
@@ -465,6 +501,8 @@ And a special thanks to those who contributed pull requests:
  * [Ivan Ushakov](https://github.com/IvanUshakov)
  * [Matthieu Paret](https://github.com/mtparet)
  * [Rohit Amarnath](https://github.com/ramarnat)
+ * [Joshua Smith](https://github.com/enviable)
+ * [Colin Petruno](https://github.com/colinpetruno)
 
 
 ## Contributing

data/lib/smarter_csv/smarter_csv.rb

@@ -59,7 +59,7 @@ module SmarterCSV
         else
           file_headerA =  header.split(options[:col_sep])
         end
-        file_headerA.map!{|x| x.gsub(%r/options[:quote_char]/,'') }
+        file_headerA.map!{|x| x.gsub(%r/#{options[:quote_char]}/,'') }
         file_headerA.map!{|x| x.strip}  if options[:strip_whitespace]
         unless options[:keep_original_headers]
           file_headerA.map!{|x| x.gsub(/\s+|-+/,'_')}
@@ -68,14 +68,14 @@ module SmarterCSV
 
         file_header_size = file_headerA.size
       else
-        raise SmarterCSV::IncorrectOption , "ERROR [smarter_csv]: If :headers_in_file is set to false, you have to provide :user_provided_headers" if options[:user_provided_headers].nil?
+        raise SmarterCSV::IncorrectOption , "ERROR: If :headers_in_file is set to false, you have to provide :user_provided_headers" if options[:user_provided_headers].nil?
       end
       if options[:user_provided_headers] && options[:user_provided_headers].class == Array && ! options[:user_provided_headers].empty?
         # use user-provided headers
         headerA = options[:user_provided_headers]
         if defined?(file_header_size) && ! file_header_size.nil?
           if headerA.size != file_header_size
-            raise SmarterCSV::HeaderSizeMismatch , "ERROR [smarter_csv]: :user_provided_headers defines #{headerA.size} headers !=  CSV-file #{input} has #{file_header_size} headers"
+            raise SmarterCSV::HeaderSizeMismatch , "ERROR: :user_provided_headers defines #{headerA.size} headers !=  CSV-file #{input} has #{file_header_size} headers"
           else
             # we could print out the mapping of file_headerA to headerA here
           end
@@ -100,14 +100,14 @@ module SmarterCSV
       headerA.compact.each do |k|
         duplicate_headers << k if headerA.select{|x| x == k}.size > 1
       end
-      raise SmarterCSV::DuplicateHeaders , "ERORR [smarter_csv]: duplicate headers: #{duplicate_headers.join(',')}" unless duplicate_headers.empty?
+      raise SmarterCSV::DuplicateHeaders , "ERROR: duplicate headers: #{duplicate_headers.join(',')}" unless duplicate_headers.empty?
 
       if options[:required_headers] && options[:required_headers].is_a?(Array)
         missing_headers = []
         options[:required_headers].each do |k|
           missing_headers << k unless headerA.include?(k)
         end
-        raise SmarterCSV::MissingHeaders , "ERORR [smarter_csv]: missing headers: #{missing_headers.join(',')}" unless missing_headers.empty?
+        raise SmarterCSV::MissingHeaders , "ERROR: missing headers: #{missing_headers.join(',')}" unless missing_headers.empty?
       end
 
       # in case we use chunking.. we'll need to set it up..
@@ -155,7 +155,7 @@ module SmarterCSV
         else
           dataA =  line.split(options[:col_sep])
         end
-        dataA.map!{|x| x.gsub(%r/options[:quote_char]/,'') }
+####     dataA.map!{|x| x.gsub(%r/#{options[:quote_char]}/,'') }  # this is actually not a good idea as a default
         dataA.map!{|x| x.strip}  if options[:strip_whitespace]
         hash = Hash.zip(headerA,dataA)  # from Facets of Ruby library
         # make sure we delete any key/value pairs from the hash, which the user wanted to delete:

data/lib/smarter_csv/version.rb

@@ -1,3 +1,3 @@
 module SmarterCSV
-  VERSION = "1.2.0"
+  VERSION = "1.2.3"
 end

data/spec/fixtures/quote_char.csv

@@ -0,0 +1,9 @@
+"ID","FIRST_NAME","LAST_NAME"
+"1","""John","Cooke"""
+"2","Jam
+e
+son""","McCollum"
+"3","""Jean","Conn"
+"4","Jenny","Traer"
+"5","Bo""bbie","Faga"
+"6","Mica","Copeland"

data/spec/fixtures/quoted2.csv

@@ -0,0 +1,4 @@
+"REVIEW DATE","AUTHOR","""ISBN""","DISCOUNTED ""PRICE"""
+"1985/01/21","Douglas Adams",0345391802,5.95
+"1998/07/15","Timothy ""The Parser"" Campbell",0968411304,18.99
+"1999/12/03","Richard Friedman",0060630353,5.95

data/spec/fixtures/trading.csv

@@ -0,0 +1,3 @@
+Account_ID,options_trader,Stock_Symbol,Shares Issued,Purchase Date
+0002310234,Mike Smith,TSLA,2300,2011-08-19
+0024923423,John Doe,AAPL,1300,2013-03-21

data/spec/smarter_csv/quoted_spec.rb

@@ -2,9 +2,9 @@ require 'spec_helper'
 
 fixture_path = 'spec/fixtures'
 
-describe 'be_able_to' do
+describe 'loading file with quoted fields' do
 
-  it 'loads_file_with_quoted_fields' do
+  it 'leaving the quotes in the data' do
     options = {}
     data = SmarterCSV.process("#{fixture_path}/quoted.csv", options)
     data.flatten.size.should == 4
@@ -20,4 +20,29 @@ describe 'be_able_to' do
     end
   end
 
+
+  it 'removes quotes around quoted fields, but not inside data' do
+    options = {}
+    data = SmarterCSV.process("#{fixture_path}/quote_char.csv", options)
+
+    data.length.should eq 6
+    data[1][:first_name].should eq "Jam\ne\nson\""
+    data[2][:first_name].should eq "\"Jean"
+  end
+
+
+  # NOTE: quotes inside headers need to be escaped by doubling them
+  #       e.g. 'correct ""EXAMPLE""'
+  #       this escaping is illegal: 'incorrect \"EXAMPLE\"' <-- this caused CSV parsing error
+  #  in case of CSV parsing errirs, use :user_provided_headers, or key_mapping
+  #
+  it 'removes quotes around headers and extra quotes inside headers' do
+    options = {}
+    data = SmarterCSV.process("#{fixture_path}/quoted2.csv", options)
+
+    data.length.should eq 3
+    data.first.keys[2].should eq :isbn
+    data.first.keys[3].should eq :discounted_price
+  end
+
 end

data/spec/smarter_csv/trading_spec.rb

@@ -0,0 +1,25 @@
+require 'spec_helper'
+
+fixture_path = 'spec/fixtures'
+
+# somebody reported that a column called 'options_trader' would be truncated to 'trader'
+
+describe 'loads simple file format' do
+
+  it 'with symbols as keys when using defaults' do
+    options = {}
+    data = SmarterCSV.process("#{fixture_path}/trading.csv", options)
+
+    data.flatten.size.should eq 2
+    data.each do |item|
+      # all keys should be symbols when using v1.x backwards compatible mode
+      item.keys.each{|x| x.class.should eq Symbol}
+      item[:account_id].class.should eq Fixnum
+      item[:options_trader].class.should eq String
+      item[:stock_symbol].class.should eq String
+      item[:shares_issued].class.should eq Fixnum
+      item[:purchase_date].class.should eq String
+    end
+  end
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: smarter_csv
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.2.3
 platform: ruby
 authors:
 - 'Tilo Sloboda
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-01-20 00:00:00.000000000 Z
+date: 2018-01-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -68,9 +68,12 @@ files:
 - spec/fixtures/no_header.csv
 - spec/fixtures/numeric.csv
 - spec/fixtures/pets.csv
+- spec/fixtures/quote_char.csv
 - spec/fixtures/quoted.csv
+- spec/fixtures/quoted2.csv
 - spec/fixtures/separator.csv
 - spec/fixtures/skip_lines.csv
+- spec/fixtures/trading.csv
 - spec/fixtures/user_import.csv
 - spec/fixtures/valid_unicode.csv
 - spec/fixtures/with_dashes.csv
@@ -101,6 +104,7 @@ files:
 - spec/smarter_csv/skip_lines_spec.rb
 - spec/smarter_csv/strings_as_keys_spec.rb
 - spec/smarter_csv/strip_chars_from_headers_spec.rb
+- spec/smarter_csv/trading_spec.rb
 - spec/smarter_csv/valid_unicode_spec.rb
 - spec/smarter_csv/value_converters_spec.rb
 - spec/spec.opts
@@ -128,7 +132,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements:
 - csv
 rubyforge_project: 
-rubygems_version: 2.6.13
+rubygems_version: 2.7.4
 signing_key: 
 specification_version: 4
 summary: Ruby Gem for smarter importing of CSV Files (and CSV-like files), with lots
@@ -153,9 +157,12 @@ test_files:
 - spec/fixtures/no_header.csv
 - spec/fixtures/numeric.csv
 - spec/fixtures/pets.csv
+- spec/fixtures/quote_char.csv
 - spec/fixtures/quoted.csv
+- spec/fixtures/quoted2.csv
 - spec/fixtures/separator.csv
 - spec/fixtures/skip_lines.csv
+- spec/fixtures/trading.csv
 - spec/fixtures/user_import.csv
 - spec/fixtures/valid_unicode.csv
 - spec/fixtures/with_dashes.csv
@@ -186,6 +193,7 @@ test_files:
 - spec/smarter_csv/skip_lines_spec.rb
 - spec/smarter_csv/strings_as_keys_spec.rb
 - spec/smarter_csv/strip_chars_from_headers_spec.rb
+- spec/smarter_csv/trading_spec.rb
 - spec/smarter_csv/valid_unicode_spec.rb
 - spec/smarter_csv/value_converters_spec.rb
 - spec/spec.opts