smarter_csv 1.0.15 → 1.0.16

data/.travis.yml

@@ -7,8 +7,6 @@ rvm:
   - 2.0.0
   - jruby-18mode
   - jruby-19mode
-  - rbx-18mode
-  - rbx-19mode
   - ruby-head
   - jruby-head
   - ree
@@ -18,6 +16,8 @@ jdk:
 env: JRUBY_OPTS="--server -Xcompile.invokedynamic=false -J-XX:+TieredCompilation -J-XX:TieredStopAtLevel=1 -J-noverify -J-Xms512m -J-Xmx1024m"
 matrix:
   allow_failures:
+    - rbx-18mode
+    - rbx-19mode
     - rvm: jruby-head
     - rvm: ruby-head
     - rvm: ree

data/README.md

@@ -3,6 +3,11 @@
 `smarter_csv` is a Ruby Gem for smarter importing of CSV Files as Array(s) of Hashes, suitable for direct processing with Mongoid or ActiveRecord, 
 and parallel processing with Resque or Sidekiq.
 
+One `smarter_csv` user wrote:
+
+  *Best gem for CSV for us yet. [...] taking an import process from 7+ hours to about 3 minutes.
+   [...] Smarter CSV was a big part and helped clean up our code ALOT*
+
 `smarter_csv` has lots of features:
  * able to process large CSV-files
  * able to chunk the input from the CSV file to avoid loading the whole CSV file into memory
@@ -143,12 +148,13 @@ The options and the block are optional.
      | :remove_values_matching     |   nil    | removes key/value pairs if value matches given regular expressions. e.g.:            |
      |                             |          | /^\$0\.0+$/ to match $0.00 , or /^#VALUE!$/ to match errors in Excel spreadsheets    |
      | :convert_values_to_numeric  |   true   | converts strings containing Integers or Floats to the appropriate class              |
+     |                             |          |      also accepts either {:except => [:key1,:key2]} or {:only => :key3}              |
      | :remove_empty_hashes        |   true   | remove / ignore any hashes which don't have any key/value pairs                      |
      | :user_provided_headers      |   nil    | *careful with that axe!*                                                             |
      |                             |          | user provided Array of header strings or symbols, to define                          |
      |                             |          | what headers should be used, overriding any in-file headers.                         |
      |                             |          | You can not combine the :user_provided_headers and :key_mapping options              |
-     | :strip_chars_from_headers   |   nil    | remove extraneous characters from the header line (e.g. if the headers are quoted)   |
+     | :strip_chars_from_headers   |   nil    | RegExp to remove extraneous characters from the header line (e.g. if headers are quoted) |
      | :headers_in_file            |   true   | Whether or not the file contains headers as the first line.                          |
      |                             |          | Important if the file does not contain headers,                                      |
      |                             |          | otherwise you would lose the first line of data.                                     |
@@ -178,6 +184,10 @@ The options and the block are optional.
  * if the chunk_size is > 0 , then the array may contain up to chunk_size Hashes.
  * this can be very useful when passing chunked data to a post-processing step, e.g. through Resque
 
+#### NOTES on improper quotation and unwanted characters in headers:
+ * some CSV files use un-escaped quotation characters inside fields. This can cause the import to break. To get around this, use the `:force_simple_split => true` option in combination with `:strip_chars_from_headers => /[\-"]/` . This will also significantly speed up the import.
+   If you would force a different :quote_char instead (setting it to a non-used character), then the import would be up to 5-times slower than using `:force_simple_split`.
+
 #### Known Issues:
  * if you are using 1.8.7 versions of Ruby, JRuby, or Ruby Enterprise Edition, `smarter_csv` will have problems with double-quoted fields, because of a bug in an underlying library.
  * if your CSV data contains the :row_sep character, e.g. CR, smarter_csv will not be able to handle the data, but will report `CSV::MalformedCSVError: Unclosed quoted field`.
@@ -218,7 +228,11 @@ Or install it yourself as:
 
 ## Changes
 
-#### 1.0.15 (2013-11-01)
+#### 1.0.16 (2014-01-13)
+ * :convert_values_to_numeric option can now be qualified with :except or :only (thanks to Hugo Lepetit)
+ * removed deprecated `process_csv` method
+ 
+#### 1.0.15 (2013-12-07)
  * new option: 
    * :remove_unmapped_keys  to completely ignore columns which were not mapped with :key_mapping (thanks to Dave Sanders)
 
@@ -319,6 +333,7 @@ And a special thanks to those who contributed pull requests:
  * [Marcos G. Zimmermann](https://github.com/marcosgz)
  * [Jordan Running](https://github.com/jrunning)
  * [Dave Sanders](https://github.com/DaveSanders)
+ * [Hugo Lepetit](https://github.com/giglemad)
 
 
 ## Contributing

data/lib/smarter_csv/smarter_csv.rb

@@ -1,7 +1,8 @@
 module SmarterCSV
 
-  class HeaderSizeMismatch < Exception
-  end
+  class HeaderSizeMismatch < Exception; end
+
+  class IncorrectOption < Exception; end
 
   def SmarterCSV.process(input, options={}, &block)   # first parameter: filename or input object with readline method
     default_options = {:col_sep => ',' , :row_sep => $/ , :quote_char => '"', :force_simple_split => false , :verbose => false ,
@@ -103,6 +104,10 @@ module SmarterCSV
         hash.delete_if{|k,v| v =~ options[:remove_values_matching]} if options[:remove_values_matching]
         if options[:convert_values_to_numeric]
           hash.each do |k,v|
+            # deal with the :only / :except options to :convert_values_to_numeric
+            next if SmarterCSV.only_or_except_limit_execution( options, :convert_values_to_numeric , k )
+
+            # convert if it's a numeric value:
             case v
             when /^[+-]?\d+\.\d+$/
               hash[k] = v.to_f
@@ -128,11 +133,9 @@ module SmarterCSV
           else
 
             # the last chunk may contain partial data, which also needs to be returned (BUG / ISSUE-18)
-            
 
           end
 
-
           # while a chunk is being filled up we don't need to do anything else here
 
         else # no chunk handling
@@ -164,9 +167,23 @@ module SmarterCSV
     end
   end
 
-  def SmarterCSV.process_csv(*args)
-    warn "[DEPRECATION] `process_csv` is deprecated.  Please use `process` instead."
-    SmarterCSV.process(*args)
+#  def SmarterCSV.process_csv(*args)
+#    warn "[DEPRECATION] `process_csv` is deprecated.  Please use `process` instead."
+#    SmarterCSV.process(*args)
+#  end
+
+  private
+  # acts as a road-block to limit processing when iterating over all k/v pairs of a CSV-hash:
+
+  def self.only_or_except_limit_execution( options, option_name, key )
+    if options[option_name].is_a?(Hash)
+      if options[option_name].has_key?( :except )
+        return true if Array( options[ option_name ][:except] ).include?(key)
+      elsif options[ option_name ].has_key?(:only)
+        return true unless Array( options[ option_name ][:only] ).include?(key)
+      end
+    end
+    return false
   end
 end
 

data/lib/smarter_csv/version.rb

@@ -1,3 +1,3 @@
 module SmarterCSV
-  VERSION = "1.0.15"
+  VERSION = "1.0.16"
 end

data/spec/fixtures/numeric.csv

@@ -0,0 +1,5 @@
+First Name,Last Name,Reference, Wealth
+Dan,McAllister,0123,3.5
+,,,,,
+Miles,O'Brian,2345,3
+Nancy,Homes,2345,01

data/spec/smarter_csv/convert_values_to_numeric_spec.rb

@@ -0,0 +1,48 @@
+require 'spec_helper'
+
+fixture_path = 'spec/fixtures'
+
+describe 'numeric conversion of values' do
+  it 'occurs by default' do
+    options = {}
+    data = SmarterCSV.process("#{fixture_path}/numeric.csv", options)
+    data.size.should == 3
+    
+    # all the keys should be symbols
+    data.each do |hash|
+      hash[:wealth].should be_a_kind_of(Numeric) unless hash[:wealth].nil?
+      hash[:reference].should be_a_kind_of(Numeric) unless hash[:reference].nil?
+    end
+  end
+
+  it 'can be prevented for all values' do
+    options = { convert_values_to_numeric: false }
+    data = SmarterCSV.process("#{fixture_path}/numeric.csv", options)
+    
+    data.each do |hash|
+      hash[:wealth].should be_a_kind_of(String) unless hash[:wealth].nil?
+      hash[:reference].should be_a_kind_of(String) unless hash[:reference].nil?
+    end
+  end
+
+  it 'can be prevented for some keys' do
+    options = { convert_values_to_numeric: { except: :reference }}
+    data = SmarterCSV.process("#{fixture_path}/numeric.csv", options)
+
+    data.each do |hash|
+      hash[:wealth].should be_a_kind_of(Numeric) unless hash[:wealth].nil?
+      hash[:reference].should be_a_kind_of(String) unless hash[:reference].nil?
+    end
+  end
+  
+  it 'can occur only for some keys' do
+    options = { convert_values_to_numeric: { only: :wealth }}
+    data = SmarterCSV.process("#{fixture_path}/numeric.csv", options)
+
+    data.each do |hash|
+      hash[:wealth].should be_a_kind_of(Numeric) unless hash[:wealth].nil?
+      hash[:reference].should be_a_kind_of(String) unless hash[:reference].nil?
+    end
+  end
+end
+

metadata

@@ -1,41 +1,45 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: smarter_csv
-version: !ruby/object:Gem::Version 
-  version: 1.0.15
+version: !ruby/object:Gem::Version
+  version: 1.0.16
+  prerelease: 
 platform: ruby
-authors: 
-- |
-  Tilo Sloboda
+authors:
+- ! 'Tilo Sloboda
 
+'
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2013-12-07 00:00:00 Z
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2014-01-13 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: rspec
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
-    requirements: 
-    - &id002 
-      - ">="
-      - !ruby/object:Gem::Version 
-        version: "0"
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-  version_requirements: *id001
-description: Ruby Gem for smarter importing of CSV Files as Array(s) of Hashes, with optional features for processing large files in parallel, embedded comments, unusual field- and record-separators, flexible mapping of CSV-headers to Hash-keys
-email: 
-- |
-  tilo.sloboda@gmail.com
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Ruby Gem for smarter importing of CSV Files as Array(s) of Hashes, with
+  optional features for processing large files in parallel, embedded comments, unusual
+  field- and record-separators, flexible mapping of CSV-headers to Hash-keys
+email:
+- ! 'tilo.sloboda@gmail.com
 
+'
 executables: []
-
 extensions: []
-
 extra_rdoc_files: []
-
-files: 
+files:
 - .gitignore
 - .rspec
 - .rvmrc
@@ -53,6 +57,7 @@ files:
 - spec/fixtures/chunk_cornercase.csv
 - spec/fixtures/lots_of_columns.csv
 - spec/fixtures/no_header.csv
+- spec/fixtures/numeric.csv
 - spec/fixtures/pets.csv
 - spec/fixtures/quoted.csv
 - spec/fixtures/separator.csv
@@ -60,6 +65,7 @@ files:
 - spec/smarter_csv/binary_file_spec.rb
 - spec/smarter_csv/chunked_reading_spec.rb
 - spec/smarter_csv/column_separator_spec.rb
+- spec/smarter_csv/convert_values_to_numeric_spec.rb
 - spec/smarter_csv/key_mapping_spec.rb
 - spec/smarter_csv/load_basic_spec.rb
 - spec/smarter_csv/no_header_spec.rb
@@ -75,35 +81,40 @@ files:
 - spec/spec/spec_helper.rb
 - spec/spec_helper.rb
 homepage: https://github.com/tilo/smarter_csv
-licenses: 
+licenses:
 - MIT
 - GPL-2
-metadata: {}
-
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
-  requirements: 
-  - *id002
-required_rubygems_version: !ruby/object:Gem::Requirement 
-  requirements: 
-  - *id002
-requirements: 
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements:
 - csv
 rubyforge_project: 
-rubygems_version: 2.0.3
+rubygems_version: 1.8.23
 signing_key: 
-specification_version: 4
-summary: Ruby Gem for smarter importing of CSV Files (and CSV-like files), with lots of optional features, e.g. chunked processing for huge CSV files
-test_files: 
+specification_version: 3
+summary: Ruby Gem for smarter importing of CSV Files (and CSV-like files), with lots
+  of optional features, e.g. chunked processing for huge CSV files
+test_files:
 - spec/fixtures/basic.csv
 - spec/fixtures/binary.csv
 - spec/fixtures/chunk_cornercase.csv
 - spec/fixtures/lots_of_columns.csv
 - spec/fixtures/no_header.csv
+- spec/fixtures/numeric.csv
 - spec/fixtures/pets.csv
 - spec/fixtures/quoted.csv
 - spec/fixtures/separator.csv
@@ -111,6 +122,7 @@ test_files:
 - spec/smarter_csv/binary_file_spec.rb
 - spec/smarter_csv/chunked_reading_spec.rb
 - spec/smarter_csv/column_separator_spec.rb
+- spec/smarter_csv/convert_values_to_numeric_spec.rb
 - spec/smarter_csv/key_mapping_spec.rb
 - spec/smarter_csv/load_basic_spec.rb
 - spec/smarter_csv/no_header_spec.rb

checksums.yaml

@@ -1,7 +0,0 @@
---- 
-SHA1: 
-  metadata.gz: 7fe73934009f54b4447577c4190d8051e315544f
-  data.tar.gz: b32b1635d2cb25f5feac9e06522058c319d6291d
-SHA512: 
-  metadata.gz: 726b149bde30de57bbabd90078b92f20a4b59d6a41dfb076b0bdb8fae6d47d7701fbdd427f154e1c523fcce3e93ae0af71c4f9426ff48edca10241ab1b50ce5b
-  data.tar.gz: 4cc80396cb2a41fae08c20969b3c4ff05b13900289f81a32ef36cc830f363d710428d050b666bc0843b0641ea63486b0157ddf2c728b59a2a47e3eb9974e265d