smarter_csv 1.0.11 → 1.0.12

checksums.yaml

@@ -1,7 +1,7 @@
 --- 
 SHA1: 
-  metadata.gz: b8da8da62110a7342bf1607f6d971125192d79d4
-  data.tar.gz: f1e42ed0dba9a1cadb609044929bdea223874887
+  metadata.gz: 0d4a07a52c8c5b769270afadb3a0057c9cc5f632
+  data.tar.gz: 27ba30864fc20db120821a4f34bb28f2247e5c4a
 SHA512: 
-  metadata.gz: 2a15724a84cb1a1985b557236dcbbc732b1afe2fac044db9002f4c4df8e178fdd875aeaf5d0f69b76478aac77b7e96cc874ae069f90e0701b0d113823f3daf24
-  data.tar.gz: aced7129287273dbfea183445bf448303c0aa51af159d7880cab6e51fa79b672f8a8231cf2cfbb5a864075dae42d52e036bd5fab850298f15ca54ab27764b8f8
+  metadata.gz: acf2ab6699268f419fcde32239da0b97f200dff744cd9c920c1f2c8cfedf9d6be0aba76b4ab77e359c429e911e83f41c6f23d7629b40a19dae0b34c134023eaa
+  data.tar.gz: 53620d67dd6a6bb8cddf17807ed6a688fb98ccc136b367b44ce77a1ea9f22b420c87c196b8430b472947c42ea091a015b05866234620b2b9f140b8a3e2ca47b7

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format documentation

data/.travis.yml

@@ -0,0 +1,9 @@
+language: ruby
+bundler_args: --without development
+rvm:
+  - "1.9.2"
+  - "1.9.3"
+  - "2.0.0"
+branches:
+  only:
+    - master

data/Gemfile

@@ -2,3 +2,10 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in smarter_csv.gemspec
 gemspec
+
+
+gem "rake"
+
+group :test do
+  gem "rspec", "~> 2.14"
+end

data/README.md

@@ -1,4 +1,4 @@
-# SmarterCSV
+# SmarterCSV  [![Build Status](https://secure.travis-ci.org/tilo/smarter_csv.png?branch=master)](http://travis-ci.org/tilo/smarter_csv)
 
 `smarter_csv` is a Ruby Gem for smarter importing of CSV Files as Array(s) of Hashes, suitable for direct processing with Mongoid or ActiveRecord, 
 and parallel processing with Resque or Sidekiq.
@@ -176,6 +176,21 @@ The options and the block are optional.
  * if the chunk_size is > 0 , then the array may contain up to chunk_size Hashes.
  * this can be very useful when passing chunked data to a post-processing step, e.g. through Resque
 
+#### Known Issue:
+ * if your CSV data contains the :row_sep character, e.g. CR, smarter_csv will not be able to handle the data, but will report `CSV::MalformedCSVError: Unclosed quoted field`.
+
+
+Example of Invalid CSV:
+
+    id,name,comment
+    1,James,a simple comment
+    2,Paul,"a comment which contains
+    the :row_sep character CR"
+    3,Frank,"some other comment"
+
+The second row contains a comment with an embedded \n carriage return character.
+`smarter_csv` handles this special case as invalid CSV.
+
 
 ## See also:
 
@@ -200,6 +215,9 @@ Or install it yourself as:
 
 ## Changes
 
+#### 1.0.12 (2013-10-15)
+ * added RSpec tests
+
 #### 1.0.11 (2013-09-28)
  * bugfix : fixed issue #18 - fixing issue with last chunk not being properly returned (thanks to Jordan Running)
  * added RSpec tests

data/lib/smarter_csv/smarter_csv.rb

@@ -34,6 +34,9 @@ module SmarterCSV
         file_headerA.map!{|x| x.strip}  if options[:strip_whitespace]
         file_headerA.map!{|x| x.gsub(/\s+/,'_')}
         file_headerA.map!{|x| x.downcase }   if options[:downcase_header]
+
+#        puts "HeaderA: #{file_headerA.join(' , ')}" if options[:verbose]
+
         file_header_size = file_headerA.size
       end
       if options[:user_provided_headers] && options[:user_provided_headers].class == Array && ! options[:user_provided_headers].empty?

data/lib/smarter_csv/version.rb

@@ -1,3 +1,3 @@
 module SmarterCSV
-  VERSION = "1.0.11"
+  VERSION = "1.0.12"
 end

data/spec/fixtures/basic.csv

@@ -1,7 +1,8 @@
-first name,last name,dogs,cats,birds,fish
-Dan,McAllister,2,,,
-Lucy,Laweless,,5,,
+First Name,Last Name,Dogs,Cats,Birds,Fish
+Dan,McAllister,2,0,,
+Lucy,Laweless,,5,0,
+,,,,,
+Miles,O'Brian,0,0,0,21
+Nancy,Homes,2,0,1, 
+Hernán,Curaçon,3,0,0,
 ,,,,,
-Miles,O'Brian,,,,21
-Nancy,Homes,2,,1, 
-,,,,,

data/spec/fixtures/binary.csv

@@ -0,0 +1 @@
+#timestamp
item_id
parent_id
name#some comments here#these can be multiple comments in the header section#even more comments here1381388409
10
1
Thing 11381388409
11
1
Thing 21381388409
12
1
Thing 31381388409
1

Parent 11381388409
2

Parent 21381388409
20
2
Thing 41381388409
21
2
Thing 51381388409
22
2
Thing 6

data/spec/fixtures/no_header.csv

@@ -0,0 +1,7 @@
+Dan,McAllister,2,0,,
+Lucy,Laweless,,5,0,
+,,,,,
+Miles,O'Brian,0,0,0,21
+Nancy,Homes,2,0,1, 
+Hernán,Curaçon,3,0,0,
+,,,,,

data/spec/fixtures/quoted.csv

@@ -0,0 +1,5 @@
+Year,Make,Model,Description,Price
+1997,Ford,E350,"ac, abs, moon",3000.00
+1999,Chevy,"Venture ""Extended Edition""","",4900.00
+1999,Chevy,"Venture ""Extended Edition, Very Large""",,5000.00
+1996,Jeep,Grand Cherokee,"MUST SELL! air, moon roof, loaded",4799.00

data/spec/fixtures/separator.csv

@@ -0,0 +1,4 @@
+Year;Make;Model;Length
+1997;Ford;E350;2,34
+2000;Mercury;Cougar;2,38
+2013;Tesla;Model S;4,97

data/spec/smarter_csv/binary_file2_spec.rb

@@ -0,0 +1,24 @@
+require 'spec_helper'
+
+fixture_path = 'spec/fixtures'
+
+# this reads a binary database dump file, which is in structure like a CSV file
+# but contains control characters delimiting the rows and columns, and also 
+# contains a comment section which is commented our by a leading # character
+
+# same as binary_file_spec , but reading the file with strings as keys
+
+describe 'be_able_to' do
+  it 'loads_binary_file_with_strings_as_keys' do 
+    options = {:col_sep => "\cA", :row_sep => "\cB", :comment_regexp => /^#/, :strings_as_keys => true}
+    data = SmarterCSV.process("#{fixture_path}/binary.csv", options)
+    data.flatten.size.should == 8
+    data.each do |item|
+      # all keys should be strings
+      item.keys.each{|x| x.class.should be == String}
+      item['timestamp'].should == 1381388409
+      item['item_id'].class.should be == Fixnum
+      item['name'].size.should be > 0
+    end
+  end
+end

data/spec/smarter_csv/binary_file_spec.rb

@@ -0,0 +1,22 @@
+require 'spec_helper'
+
+fixture_path = 'spec/fixtures'
+
+# this reads a binary database dump file, which is in structure like a CSV file
+# but contains control characters delimiting the rows and columns, and also 
+# contains a comment section which is commented our by a leading # character
+
+describe 'be_able_to' do
+  it 'loads_binary_file_with_comments' do 
+    options = {:col_sep => "\cA", :row_sep => "\cB", :comment_regexp => /^#/}
+    data = SmarterCSV.process("#{fixture_path}/binary.csv", options)
+    data.flatten.size.should == 8
+    data.each do |item|
+      # all keys should be symbols
+      item.keys.each{|x| x.class.should be == Symbol}
+      item[:timestamp].should == 1381388409
+      item[:item_id].class.should be == Fixnum
+      item[:name].size.should be > 0
+    end
+  end
+end

data/spec/smarter_csv/chunked_reading_spec.rb

@@ -0,0 +1,14 @@
+require 'spec_helper'
+
+fixture_path = 'spec/fixtures'
+
+describe 'be_able_to' do
+  it 'loads_chunk_cornercase_csv_files' do 
+    (0..5).each do |chunk_size|    # test for all chunk-sizes
+      options = {:chunk_size => chunk_size, :remove_empty_hashes => true}
+      data = SmarterCSV.process("#{fixture_path}/chunk_cornercase.csv", options)
+      data.flatten.size.should == 5  # end-result must always be 5 rows
+    end
+  end
+
+end

data/spec/smarter_csv/column_separator_spec.rb

@@ -0,0 +1,11 @@
+require 'spec_helper'
+
+fixture_path = 'spec/fixtures'
+
+describe 'be_able_to' do
+  it 'loads_file_with_different_column_separator' do 
+    options = {:col_sep => ';'}
+    data = SmarterCSV.process("#{fixture_path}/separator.csv", options)
+    data.flatten.size.should == 3
+  end
+end

data/spec/smarter_csv/key_mapping_spec.rb

@@ -0,0 +1,25 @@
+require 'spec_helper'
+
+fixture_path = 'spec/fixtures'
+
+describe 'be_able_to' do
+  it 'remove_values_matching' do 
+    options = {:remove_zero_values => true, :key_mapping => {:first_name => :vorname, :last_name => :nachname} }
+    data = SmarterCSV.process("#{fixture_path}/basic.csv", options)
+    data.size.should == 5
+    # all the keys should be symbols
+    data.each{|item| item.keys.each{|x| x.class.should be == Symbol}}
+
+    data.each do |hash| 
+      hash.keys.each do |key|
+        [:vorname, :nachname, :dogs, :cats, :birds, :fish].should include( key )
+      end
+      hash.values.should_not include( 0 )
+    end
+
+    data.each do |h|
+      h.size.should <= 6
+    end
+  end
+
+end

data/spec/smarter_csv/load_basic_spec.rb

@@ -5,8 +5,14 @@ fixture_path = 'spec/fixtures'
 describe 'be_able_to' do
   it 'loads_basic_csv_file' do 
     data = SmarterCSV.process("#{fixture_path}/basic.csv")
-    data.size.should == 4
+    data.size.should == 5
+
+    # all the keys should be symbols
+    data.each{|item| item.keys.each{|x| x.class.should be == Symbol}}
     data.each do |h|
+      h.keys.each do |key|
+        [:first_name, :last_name, :dogs, :cats, :birds, :fish].should include( key )
+      end
       h.size.should <= 6
     end
   end

data/spec/smarter_csv/no_header_spec.rb

@@ -0,0 +1,24 @@
+require 'spec_helper'
+
+fixture_path = 'spec/fixtures'
+
+describe 'be_able_to' do
+  it 'loads_csv_file_without_header' do 
+    options = {:headers_in_file => false, :user_provided_headers => [:a,:b,:c,:d,:e,:f]}
+    data = SmarterCSV.process("#{fixture_path}/no_header.csv", options)
+    data.size.should == 5
+    # all the keys should be symbols
+    data.each{|item| item.keys.each{|x| x.class.should be == Symbol}}
+
+    data.each do |item| 
+      item.keys.each do |key|
+        [:a,:b,:c,:d,:e,:f].should include( key )
+      end
+    end
+
+    data.each do |h|
+      h.size.should <= 6
+    end
+  end
+
+end

data/spec/smarter_csv/not_downcase_header_spec.rb

@@ -0,0 +1,24 @@
+require 'spec_helper'
+
+fixture_path = 'spec/fixtures'
+
+describe 'be_able_to' do
+  it 'not_downcase_headers' do 
+    options = {:downcase_header => false}
+    data = SmarterCSV.process("#{fixture_path}/basic.csv", options)
+    data.size.should == 5
+    # all the keys should be symbols
+    data.each{|item| item.keys.each{|x| x.class.should be == Symbol}}
+
+    data.each do |item| 
+      item.keys.each do |key|
+        [:First_Name, :Last_Name, :Dogs, :Cats, :Birds, :Fish].should include( key )
+      end
+    end
+
+    data.each do |h|
+      h.size.should <= 6
+    end
+  end
+
+end

data/spec/smarter_csv/quoted_spec.rb

@@ -0,0 +1,13 @@
+require 'spec_helper'
+
+fixture_path = 'spec/fixtures'
+
+describe 'be_able_to' do
+  it 'loads_file_with_quoted_fields' do 
+    options = {}
+    data = SmarterCSV.process("#{fixture_path}/quoted.csv", options)
+    data.flatten.size.should == 4
+    data[1][:description].should be_nil
+    data[2][:description].should be_nil
+  end
+end

data/spec/smarter_csv/remove_keys_from_hashes_spec.rb

@@ -0,0 +1,25 @@
+require 'spec_helper'
+
+fixture_path = 'spec/fixtures'
+
+describe 'be_able_to' do
+  it 'remove_values_matching' do 
+    options = {:remove_zero_values => true, :key_mapping => {:first_name => :vorname, :last_name => :nachname, :fish => nil} }
+    data = SmarterCSV.process("#{fixture_path}/basic.csv", options)
+    data.size.should == 5
+    # all the keys should be symbols
+    data.each{|item| item.keys.each{|x| x.class.should be == Symbol}}
+
+    data.each do |hash| 
+      hash.keys.each do |key|
+        [:vorname, :nachname, :dogs, :cats, :birds].should include( key )
+      end
+      hash.values.should_not include( 0 )
+    end
+
+    data.each do |h|
+      h.size.should <= 6
+    end
+  end
+
+end

data/spec/smarter_csv/remove_values_matching_spec.rb

@@ -0,0 +1,26 @@
+require 'spec_helper'
+
+fixture_path = 'spec/fixtures'
+
+describe 'be_able_to' do
+  it 'remove_values_matching' do 
+    options = {:remove_zero_values => true, :remove_empty_values => true, :remove_values_matching => /^\d+$/}
+    data = SmarterCSV.process("#{fixture_path}/basic.csv", options)
+    data.size.should == 5
+    # all the keys should be symbols
+    data.each{|item| item.keys.each{|x| x.class.should be == Symbol}}
+
+    data.each do |hash| 
+      hash.keys.each do |key|
+        [:first_name, :last_name].should include( key )
+      end
+      hash.values.each{|x| x.class.should be == String}
+      hash.values.should_not include( 0 )
+    end
+
+    data.each do |h|
+      h.size.should <= 6
+    end
+  end
+
+end

data/spec/smarter_csv/remove_zero_values_spec.rb

@@ -0,0 +1,25 @@
+require 'spec_helper'
+
+fixture_path = 'spec/fixtures'
+
+describe 'be_able_to' do
+  it 'remove_zero_values' do 
+    options = {:remove_zero_values => true, :remove_empty_values => true}
+    data = SmarterCSV.process("#{fixture_path}/basic.csv", options)
+    data.size.should == 5
+    # all the keys should be symbols
+    data.each{|item| item.keys.each{|x| x.class.should be == Symbol}}
+
+    data.each do |hash| 
+      hash.keys.each do |key|
+        [:first_name, :last_name, :dogs, :cats, :birds, :fish].should include( key )
+      end
+      hash.values.should_not include( 0 )
+    end
+
+    data.each do |h|
+      h.size.should <= 6
+    end
+  end
+
+end

data/spec/smarter_csv/strings_as_keys_spec.rb

@@ -0,0 +1,24 @@
+require 'spec_helper'
+
+fixture_path = 'spec/fixtures'
+
+describe 'be_able_to' do
+  it 'use_strings_as_keys' do 
+    options = {:strings_as_keys => true}
+    data = SmarterCSV.process("#{fixture_path}/basic.csv", options)
+    data.size.should == 5
+    # all the keys should be symbols
+    data.each{|item| item.keys.each{|x| x.class.should be == String}}
+
+    data.each do |item| 
+      item.keys.each do |key|
+        ["first_name", "last_name", "dogs", "cats", "birds", "fish"].should include( key )
+      end
+    end
+
+    data.each do |h|
+      h.size.should <= 6
+    end
+  end
+
+end

data/spec/smarter_csv/strip_chars_from_headers_spec.rb

@@ -0,0 +1,24 @@
+require 'spec_helper'
+
+fixture_path = 'spec/fixtures'
+
+describe 'be_able_to' do
+  it 'strip_whitespace_from_headers' do 
+    options = {:strip_chars_from_headers => ' '}
+    data = SmarterCSV.process("#{fixture_path}/basic.csv", options)
+    data.size.should == 5
+    # all the keys should be symbols
+    data.each{|item| item.keys.each{|x| x.class.should be == Symbol}}
+
+    data.each do |item| 
+      item.keys.each do |key|
+        [:firstname, :lastname, :dogs, :cats, :birds, :fish].should include( key )
+      end
+    end
+
+    data.each do |h|
+      h.size.should <= 6
+    end
+  end
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: smarter_csv
 version: !ruby/object:Gem::Version 
-  version: 1.0.11
+  version: 1.0.12
 platform: ruby
 authors: 
 - |
@@ -11,7 +11,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2013-09-28 00:00:00 Z
+date: 2013-10-16 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: rspec
@@ -37,7 +37,9 @@ extra_rdoc_files: []
 
 files: 
 - .gitignore
+- .rspec
 - .rvmrc
+- .travis.yml
 - Gemfile
 - LICENSE
 - README.md
@@ -48,10 +50,26 @@ files:
 - lib/smarter_csv/version.rb
 - smarter_csv.gemspec
 - spec/fixtures/basic.csv
+- spec/fixtures/binary.csv
 - spec/fixtures/chunk_cornercase.csv
+- spec/fixtures/no_header.csv
 - spec/fixtures/pets.csv
-- spec/smarter_csv/chunked_reading.rb
+- spec/fixtures/quoted.csv
+- spec/fixtures/separator.csv
+- spec/smarter_csv/binary_file2_spec.rb
+- spec/smarter_csv/binary_file_spec.rb
+- spec/smarter_csv/chunked_reading_spec.rb
+- spec/smarter_csv/column_separator_spec.rb
+- spec/smarter_csv/key_mapping_spec.rb
 - spec/smarter_csv/load_basic_spec.rb
+- spec/smarter_csv/no_header_spec.rb
+- spec/smarter_csv/not_downcase_header_spec.rb
+- spec/smarter_csv/quoted_spec.rb
+- spec/smarter_csv/remove_keys_from_hashes_spec.rb
+- spec/smarter_csv/remove_values_matching_spec.rb
+- spec/smarter_csv/remove_zero_values_spec.rb
+- spec/smarter_csv/strings_as_keys_spec.rb
+- spec/smarter_csv/strip_chars_from_headers_spec.rb
 - spec/spec.opts
 - spec/spec/spec_helper.rb
 - spec/spec_helper.rb
@@ -80,10 +98,26 @@ specification_version: 4
 summary: Ruby Gem for smarter importing of CSV Files (and CSV-like files), with lots of optional features, e.g. chunked processing for huge CSV files
 test_files: 
 - spec/fixtures/basic.csv
+- spec/fixtures/binary.csv
 - spec/fixtures/chunk_cornercase.csv
+- spec/fixtures/no_header.csv
 - spec/fixtures/pets.csv
-- spec/smarter_csv/chunked_reading.rb
+- spec/fixtures/quoted.csv
+- spec/fixtures/separator.csv
+- spec/smarter_csv/binary_file2_spec.rb
+- spec/smarter_csv/binary_file_spec.rb
+- spec/smarter_csv/chunked_reading_spec.rb
+- spec/smarter_csv/column_separator_spec.rb
+- spec/smarter_csv/key_mapping_spec.rb
 - spec/smarter_csv/load_basic_spec.rb
+- spec/smarter_csv/no_header_spec.rb
+- spec/smarter_csv/not_downcase_header_spec.rb
+- spec/smarter_csv/quoted_spec.rb
+- spec/smarter_csv/remove_keys_from_hashes_spec.rb
+- spec/smarter_csv/remove_values_matching_spec.rb
+- spec/smarter_csv/remove_zero_values_spec.rb
+- spec/smarter_csv/strings_as_keys_spec.rb
+- spec/smarter_csv/strip_chars_from_headers_spec.rb
 - spec/spec.opts
 - spec/spec/spec_helper.rb
 - spec/spec_helper.rb

data/spec/smarter_csv/chunked_reading.rb

@@ -1,13 +0,0 @@
-require 'spec_helper'
-
-fixture_path = 'spec/fixtures'
-
-describe 'be_able_to' do
-  it 'loads_chunk_cornercase_csv_files' do 
-    (0..5).each do |chunk_size|    # test for all chunk-sizes
-    options = {:chunk_size => chunk_size, :remove_empty_hashes => true}
-    data = SmarterCSV.process("#{fixture_path}/chunk_cornercase.csv", options)
-    data.flatten.size.should == 5  # end-result must always be 5 rows
-  end
-
-end