smarter_csv 1.0.19 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9022f349dd8ee2590c73b198fb83114a8c96932d
-  data.tar.gz: 9c1a769c72e08e2e78d15ad444bf3f9642cd33e5
+  metadata.gz: ec1442dad9c0f71dc3264f6df10f5fe1116f9d23
+  data.tar.gz: 2025c6c7c81fc6c94fed0ed7d391eb122a464bd1
 SHA512:
-  metadata.gz: e3ccf944663244bc4b336d9980c26f1fda874d48586a131f3c761b6885a2753ac443c80a559046e2c6670f90ba192155e10aceb0e84798add22c9a20d78653a1
-  data.tar.gz: 69b3abf03488df9b79b796dd7efbc3612bd273fb1e4f6f156b238213ef377e22ff1852ae17fb7384722dfbba456d9ab36313e5d2c43d5a599a696d008194cd29
+  metadata.gz: 1dd00a098dba973b2f6e0303317bd11cdc527d05ca959dd21ae78f241a640e799b623f0b719da5d146c89856d0e50a557ffa8180725d29c6582e0e10231b091a
+  data.tar.gz: 7ab34fac0386ccef0ad13f1116b5ed0669d4ffe6270f8a4b9e39da8421132c4fac4f1c1764b78ca8ce5e702df29c9819ee1359bf0b18c98b735ea53a3935e4d8

data/.travis.yml

@@ -6,6 +6,7 @@ rvm:
   - 1.9.3
   - 2.0.0
   - 2.1.3
+  - 2.2.2
   - jruby
   - ruby-head
   - jruby-head

data/README.md

@@ -1,4 +1,6 @@
-# SmarterCSV  [![Build Status](https://secure.travis-ci.org/tilo/smarter_csv.png?branch=master)](http://travis-ci.org/tilo/smarter_csv)
+# SmarterCSV
+
+[![Build Status](https://secure.travis-ci.org/tilo/smarter_csv.png?branch=master)](http://travis-ci.org/tilo/smarter_csv) [![Gem Version](https://badge.fury.io/rb/smarter_csv.svg)](http://badge.fury.io/rb/smarter_csv)
 
 `smarter_csv` is a Ruby Gem for smarter importing of CSV Files as Array(s) of Hashes, suitable for direct processing with Mongoid or ActiveRecord,
 and parallel processing with Resque or Sidekiq.
@@ -32,7 +34,8 @@ The two main choices you have in terms of how to call `SmarterCSV.process` are:
  * calling `process` with or without a block
  * passing a `:chunk_size` to the `process` method, and processing the CSV-file in chunks, rather than in one piece.
 
-Tip: If you are uncertain about what line endings a CSV-file uses, try specifying `:row_sep => :auto` as part of the options. Checkout Example 5 for unusual `:row_sep` and `:col_sep`.
+Tip: If you are uncertain about what line endings a CSV-file uses, try specifying `:row_sep => :auto` as part of the options.
+But this could be slow, because it will try to analyze each CSV file first. If you want to speed things up, set the `:row_sep` manually! Checkout Example 5 for unusual `:row_sep` and `:col_sep`.
 
 #### Example 1a: How SmarterCSV processes CSV-files as array of hashes:
 Please note how each hash contains only the keys for columns with non-null values.
@@ -125,6 +128,40 @@ and how the `process` method returns the number of chunks when called with a blo
     end
     => returns number of chunks
 
+#### Example 6: Using Value Converters
+
+    $ cat spec/fixtures/with_dates.csv
+    first,last,date,price
+    Ben,Miller,10/30/1998,$44.50
+    Tom,Turner,2/1/2011,$15.99
+    Ken,Smith,01/09/2013,$199.99
+    $ irb
+    > require 'smarter_csv'
+    > require 'date'
+
+    # define a custom converter class, which implements self.convert(value)
+    class DateConverter
+      def self.convert(value)
+        Date.strptime( value, '%m/%d/%Y') # parses custom date format into Date instance
+      end
+    end
+
+    class DollarConverter
+      def self.convert(value)
+        value.sub('$','').to_f
+      end
+    end
+
+    options = {:value_converters => {:date => DateConverter, :price => DollarConverter}}
+    data = SmarterCSV.process("spec/fixtures/with_dates.csv", options)
+    data[0][:date]
+      => #<Date: 1998-10-30 ((2451117j,0s,0n),+0s,2299161j)>
+    data[0][:date].class
+      => Date
+    data[0][:price]
+      => 44.50
+    data[0][:price].class
+      => Float
 
 ## Documentation
 
@@ -141,7 +178,7 @@ The options and the block are optional.
      ---------------------------------------------------------------------------------------------------------------------------------
      | :col_sep                    |   ','    | column separator                                                                     |
      | :row_sep                    | $/ ,"\n" | row separator or record separator , defaults to system's $/ , which defaults to "\n" |
-     |                             |          |        this can also be set to :auto , but will process the whole cvs file first     |
+     |                             |          | This can also be set to :auto, but will process the whole cvs file first  (slow!)    |
      | :quote_char                 |   '"'    | quotation character                                                                  |
      | :comment_regexp             |   /^#/   | regular expression which matches comment lines (see NOTE about the CSV header)       |
      | :chunk_size                 |   nil    | if set, determines the desired chunk-size (defaults to nil, no chunk processing)     |
@@ -162,6 +199,7 @@ The options and the block are optional.
      |                             |          | Important if the file does not contain headers,                                      |
      |                             |          | otherwise you would lose the first line of data.                                     |
      ---------------------------------------------------------------------------------------------------------------------------------
+     | :value_converters           |   nil    | supply a hash of :header => KlassName; the class needs to implement self.convert(val)|
      | :remove_empty_values        |   true   | remove values which have nil or empty strings as values                              |
      | :remove_zero_values         |   true   | remove values which have a numeric value equal to zero / 0                           |
      | :remove_values_matching     |   nil    | removes key/value pairs if value matches given regular expressions. e.g.:            |
@@ -235,6 +273,12 @@ Or install it yourself as:
 
 ## Changes
 
+#### 1.1.0 (2015-07-26)
+ * added feature :value_converters, which allows parsing of dates, money, and other things (thanks to Raphaël Bleuse, Lucas Camargo de Almeida, Alejandro)
+ * added error if :headers_in_file is set to false, and no :user_provided_headers are given (thanks to innhyu)
+ * added support to convert dashes to underscore characters in headers (thanks to César Camacho)
+ * fixing automatic detection of \r\n line-endings (thanks to feens)
+
 #### 1.0.19 (2014-10-29)
  * added option :keep_original_headers to keep CSV-headers as-is (thanks to Benjamin Thouret)
 
@@ -339,6 +383,12 @@ Please [open an Issue on GitHub](https://github.com/tilo/smarter_csv/issues) if
 Many thanks to people who have filed issues and sent comments.
 And a special thanks to those who contributed pull requests:
 
+ * [Alejandro](https://github.com/agaviria)
+ * [Lucas Camargo de Almeida](https://github.com/lcalmeida)
+ * [Raphaël Bleuse](https://github.com/bleuse)
+ * [feens](https://github.com/feens)
+ * [César Camacho](https://github.com/chanko)
+ * [innhyu](https://github.com/innhyu)
  * [Benjamin Thouret](https://github.com/benichu)
  * [Chris Hilton](https://github.com/chrismhilton)
  * [Sean Duckett](http://github.com/sduckett)

data/lib/smarter_csv/smarter_csv.rb

@@ -9,7 +9,7 @@ module SmarterCSV
       :remove_empty_values => true, :remove_zero_values => false , :remove_values_matching => nil , :remove_empty_hashes => true , :strip_whitespace => true,
       :convert_values_to_numeric => true, :strip_chars_from_headers => nil , :user_provided_headers => nil , :headers_in_file => true,
       :comment_regexp => /^#/, :chunk_size => nil , :key_mapping_hash => nil , :downcase_header => true, :strings_as_keys => false, :file_encoding => 'utf-8',
-      :remove_unmapped_keys => false, :keep_original_headers => false,
+      :remove_unmapped_keys => false, :keep_original_headers => false, :value_converters => nil,
     }
     options = default_options.merge(options)
     csv_options = options.select{|k,v| [:col_sep, :row_sep, :quote_char].include?(k)} # options.slice(:col_sep, :row_sep, :quote_char)
@@ -40,13 +40,15 @@ module SmarterCSV
         file_headerA.map!{|x| x.gsub(%r/options[:quote_char]/,'') }
         file_headerA.map!{|x| x.strip}  if options[:strip_whitespace]
         unless options[:keep_original_headers]
-          file_headerA.map!{|x| x.gsub(/\s+/,'_')}
+          file_headerA.map!{|x| x.gsub(/\s+|-+/,'_')}
           file_headerA.map!{|x| x.downcase }   if options[:downcase_header]
         end
 
 #        puts "HeaderA: #{file_headerA.join(' , ')}" if options[:verbose]
 
         file_header_size = file_headerA.size
+      else
+        raise SmarterCSV::IncorrectOption , "ERROR [smarter_csv]: If :headers_in_file is set to false, you have to provide :user_provided_headers" if ! options.keys.include?(:user_provided_headers)
       end
       if options[:user_provided_headers] && options[:user_provided_headers].class == Array && ! options[:user_provided_headers].empty?
         # use user-provided headers
@@ -135,6 +137,15 @@ module SmarterCSV
             end
           end
         end
+
+        if options[:value_converters]
+          hash.each do |k,v|
+            converter = options[:value_converters][k]
+            next unless converter
+            hash[k] = converter.convert(v)
+          end
+        end
+
         next if hash.empty? if options[:remove_empty_hashes]
 
         if use_chunks
@@ -212,11 +223,23 @@ module SmarterCSV
 
     # count how many of the pre-defined line-endings we find
     # ignoring those contained within quote characters
+    last_char = nil
     filehandle.each_char do |c|
       quoted_char = !quoted_char if c == options[:quote_char]
-      next if quoted_char || c !~ /\r|\n|\r\n/
-      counts[c] += 1
+      next if quoted_char
+
+      if last_char == "\r"
+        if c == "\n"
+          counts["\r\n"] +=  1
+        else
+          counts["\r"] += 1  # \r are counted after they appeared, we might
+        end
+      elsif c == "\n"
+        counts["\n"] += 1
+      end
+      last_char = c
     end
+    counts["\r"] += 1 if last_char == "\r"
     # find the key/value pair with the largest counter:
     k,v = counts.max_by{|k,v| v}
     return k                    # the most frequent one is it

data/lib/smarter_csv/version.rb

@@ -1,3 +1,3 @@
 module SmarterCSV
-  VERSION = "1.0.19"
+  VERSION = "1.1.0"
 end

data/spec/fixtures/money.csv

@@ -0,0 +1,3 @@
+item,price
+Book,$9.99
+Mug,$14.99

data/spec/fixtures/with_dashes.csv

@@ -0,0 +1,8 @@
+First-Name,Last-Name,Dogs,Cats,Birds,Fish
+Dan,McAllister,2,0,,
+Lucy,Laweless,,5,0,
+,,,,,
+Miles,O'Brian,0,0,0,21
+Nancy,Homes,2,0,1,
+Hernán,Curaçon,3,0,0,
+,,,,,

data/spec/fixtures/with_dates.csv

@@ -0,0 +1,4 @@
+first,last,date,price
+Ben,Miller,10/30/1998,$44.50
+Tom,Turner,2/1/2011,$15
+Ken,Smith,01/09/2013,$0.11

data/spec/smarter_csv/header_transformation_spec.rb

@@ -0,0 +1,21 @@
+require 'spec_helper'
+
+fixture_path = 'spec/fixtures'
+
+describe 'be_able_to' do
+  it 'loads_file_with_dashes_in_header_fields as strings' do
+    options = {:strings_as_keys => true}
+    data = SmarterCSV.process("#{fixture_path}/with_dashes.csv", options)
+    data.flatten.size.should == 5
+    data[0]['first_name'].should eq 'Dan'
+    data[0]['last_name'].should eq 'McAllister'
+  end
+
+  it 'loads_file_with_dashes_in_header_fields as symbols' do
+    options = {:strings_as_keys => false}
+    data = SmarterCSV.process("#{fixture_path}/with_dashes.csv", options)
+    data.flatten.size.should == 5
+    data[0][:first_name].should eq 'Dan'
+    data[0][:last_name].should eq 'McAllister'
+  end
+end

data/spec/smarter_csv/value_converters_spec.rb

@@ -0,0 +1,52 @@
+require 'spec_helper'
+
+fixture_path = 'spec/fixtures'
+
+require 'date'
+class DateConverter
+  def self.convert(value)
+    Date.strptime( value, '%m/%d/%Y')
+  end
+end
+
+class CurrencyConverter
+  def self.convert(value)
+    value.sub(/[$]/,'').to_f  # would be nice to add a computed column :currency => '€'
+  end
+end
+
+describe 'be_able_to' do
+  it 'convert date values into Date instances' do
+    options = {:value_converters => {:date => DateConverter}}
+    data = SmarterCSV.process("#{fixture_path}/with_dates.csv", options)
+    data.flatten.size.should == 3
+    data[0][:date].class.should eq Date
+    data[0][:date].to_s.should eq "1998-10-30"
+    data[1][:date].to_s.should eq "2011-02-01"
+    data[2][:date].to_s.should eq "2013-01-09"
+  end
+
+  it 'converts dollar prices into float values' do
+    options = {:value_converters => {:price => CurrencyConverter}}
+    data = SmarterCSV.process("#{fixture_path}/money.csv", options)
+    data.flatten.size.should == 2
+    data[0][:price].class.should eq Float
+    data[0][:price].should eq 9.99
+    data[1][:price].should eq 14.99
+  end
+
+  it 'convert can use multiple value converters' do
+    options = {:value_converters => {:date => DateConverter, :price => CurrencyConverter}}
+    data = SmarterCSV.process("#{fixture_path}/with_dates.csv", options)
+    data.flatten.size.should == 3
+    data[0][:date].class.should eq Date
+    data[0][:date].to_s.should eq "1998-10-30"
+    data[1][:date].to_s.should eq "2011-02-01"
+    data[2][:date].to_s.should eq "2013-01-09"
+
+    data[0][:price].class.should eq Float
+    data[0][:price].should eq 44.50
+    data[1][:price].should eq 15.0
+    data[2][:price].should eq 0.11
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: smarter_csv
 version: !ruby/object:Gem::Version
-  version: 1.0.19
+  version: 1.1.0
 platform: ruby
 authors:
 - |
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-29 00:00:00.000000000 Z
+date: 2015-07-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -59,17 +59,21 @@ files:
 - spec/fixtures/line_endings_r.csv
 - spec/fixtures/line_endings_rn.csv
 - spec/fixtures/lots_of_columns.csv
+- spec/fixtures/money.csv
 - spec/fixtures/no_header.csv
 - spec/fixtures/numeric.csv
 - spec/fixtures/pets.csv
 - spec/fixtures/quoted.csv
 - spec/fixtures/separator.csv
+- spec/fixtures/with_dashes.csv
+- spec/fixtures/with_dates.csv
 - spec/smarter_csv/binary_file2_spec.rb
 - spec/smarter_csv/binary_file_spec.rb
 - spec/smarter_csv/carriage_return_spec.rb
 - spec/smarter_csv/chunked_reading_spec.rb
 - spec/smarter_csv/column_separator_spec.rb
 - spec/smarter_csv/convert_values_to_numeric_spec.rb
+- spec/smarter_csv/header_transformation_spec.rb
 - spec/smarter_csv/keep_headers_spec.rb
 - spec/smarter_csv/key_mapping_spec.rb
 - spec/smarter_csv/line_ending_spec.rb
@@ -84,6 +88,7 @@ files:
 - spec/smarter_csv/remove_zero_values_spec.rb
 - spec/smarter_csv/strings_as_keys_spec.rb
 - spec/smarter_csv/strip_chars_from_headers_spec.rb
+- spec/smarter_csv/value_converters_spec.rb
 - spec/spec.opts
 - spec/spec/spec_helper.rb
 - spec/spec_helper.rb
@@ -109,7 +114,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements:
 - csv
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: Ruby Gem for smarter importing of CSV Files (and CSV-like files), with lots
@@ -127,17 +132,21 @@ test_files:
 - spec/fixtures/line_endings_r.csv
 - spec/fixtures/line_endings_rn.csv
 - spec/fixtures/lots_of_columns.csv
+- spec/fixtures/money.csv
 - spec/fixtures/no_header.csv
 - spec/fixtures/numeric.csv
 - spec/fixtures/pets.csv
 - spec/fixtures/quoted.csv
 - spec/fixtures/separator.csv
+- spec/fixtures/with_dashes.csv
+- spec/fixtures/with_dates.csv
 - spec/smarter_csv/binary_file2_spec.rb
 - spec/smarter_csv/binary_file_spec.rb
 - spec/smarter_csv/carriage_return_spec.rb
 - spec/smarter_csv/chunked_reading_spec.rb
 - spec/smarter_csv/column_separator_spec.rb
 - spec/smarter_csv/convert_values_to_numeric_spec.rb
+- spec/smarter_csv/header_transformation_spec.rb
 - spec/smarter_csv/keep_headers_spec.rb
 - spec/smarter_csv/key_mapping_spec.rb
 - spec/smarter_csv/line_ending_spec.rb
@@ -152,6 +161,7 @@ test_files:
 - spec/smarter_csv/remove_zero_values_spec.rb
 - spec/smarter_csv/strings_as_keys_spec.rb
 - spec/smarter_csv/strip_chars_from_headers_spec.rb
+- spec/smarter_csv/value_converters_spec.rb
 - spec/spec.opts
 - spec/spec/spec_helper.rb
 - spec/spec_helper.rb