iron-import 0.8.3 → 0.8.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2d9d43cccc1da84c981d29256f46c9c225241ba9
-  data.tar.gz: df64d0839daf658b286d0513c404271899303a40
+  metadata.gz: f53e828e0c419e7ad91abd7d4bc4879154941828
+  data.tar.gz: 239576c3acf292c759363da0e007acf82e4cbfc3
 SHA512:
-  metadata.gz: 23e16558e41b48c89d45c11e2fc38f1efebdd56dece3866a691c96c98d92371c7cae8e1ad0e874217d81ec649d906b71fac057208cd2c09e49cd47312a83b3ee
-  data.tar.gz: 9601f72f3b8c651b68c60440ee4ef5838c075daaa6a84146600ff876da2eeb2e1c5eb8bd4e9ce4e36aca689a8068afe486b941c0f807d7300e0d21e164ad62b0
+  metadata.gz: 858285de3786fb5e37e313fb48fc52e05fe39f81ef4ed972060bf87f2c906210e62a9c6e6a7200afb348250403162fcd89da66b7526def7b52aa5b0247a87c04
+  data.tar.gz: a9b29b5cdb47b3fc4885484b75b31e070667abf30955f77fda115d2878bbb2fa269f6402dd669aabbebec925edac4bc543a38ebf260f26dc5749e6669dce05f3

data/History.txt

@@ -1,3 +1,8 @@
+== 0.8.4 / 2018-01-24
+
+* Improve CSV reader to canonicalize newlines, converting \r + \r\n to \n before import, fixes Windows lameness
+* Fix nokogiri and roo gem detection to properly handle installed versions much greater than required
+
 == 0.8.3 / 2017-08-22
 
 * Add :bool column type (supports 'yes'/'no', 'Y'/'N', 0/1, 'true'/'false', 'T'/'F')

data/README.rdoc

@@ -41,7 +41,7 @@ any errors encountered... well, this is the library for you!
 - Import custom tabular data via passed block
 - Automatic column order and start row detection
 - Support for optional columns and dynamic column sets
-- Basic data coercion supporting string, int, float, date and cents types
+- Basic data coercion supporting string, int, float, date, bool and cents types
 - Custom data coercion via passed block
 - Custom data validation via passed block
 - Row filtering using custom block
@@ -65,11 +65,12 @@ any errors encountered... well, this is the library for you!
         end
         # And custom validation
         validate do |parsed_val|
-          add_error('Invalid description') unless parsed_val.length > 5
+          add_error('Description too short') unless parsed_val.length > 5
         end
       end
       column :price do
-        # Built in type conversion handles common cases
+        # Built in type conversion handles common cases - in this case
+        # will correctly turn 2.5, "$2.50" or "2.5" into 250
         type :cents
       end
       
@@ -99,7 +100,7 @@ any errors encountered... well, this is the library for you!
         end
         
       else
-        # General errors, dump report
+        # General errors, dump summary report
         puts "Error(s) on import: " + error_summary
       end
     end
@@ -118,17 +119,17 @@ any errors encountered... well, this is the library for you!
 
 It can be tricky to keep track of what happens in Importer#import, so here's a quick cheat-sheet:
 
-- Determine the **format** of stream/file to import
-- Determine **import scope** (sheet/table/whatever) using Importer#scope settings, if any
-- **Find column headers + start row**
-- Validate presence of **required columns**
-- **Validate column set** using Importer#validate_columns
+- Determine the *format* of stream/file to import
+- Determine *import scope* (sheet/table/whatever) using Importer#scope settings, if any
+- *Find column headers + start row*
+- Validate presence of *required columns*
+- *Validate column set* using Importer#validate_columns
 - Run each row:
-  - **Parse** each column's value using Column#parse or Column#type
-  - **Filter the row** using Importer#filter_rows on parsed values to reject unwanted rows
-  - **Calculate virtual columns** using Column#calculate
-  - **Validate each parsed value** using Column#validate
-  - **Validate entire row** using Importer#validate_rows
+  - *Parse* each column's value using Column#parse or Column#type
+  - *Filter the row* using Importer#filter_rows on parsed values to reject unwanted rows
+  - *Calculate virtual columns* using Column#calculate
+  - *Validate each parsed value* using Column#validate
+  - *Validate entire row* using Importer#validate_rows
 
 Generally, the import will stop when an error occurs, save on row processing, where each row will
 be run until an error for that row is found.  The goal is to accumulate actionable info for

data/Version.txt

@@ -1 +1 @@
-0.8.3
+0.8.4

data/lib/iron/import/csv_reader.rb

@@ -14,19 +14,32 @@ class Importer
       if mode == :stream
         # For streams, we just read 'em in and parse 'em
         text = source.read
-        encoding = @importer.encoding || 'UTF-8'
-        @raw_rows = CSV.parse(text, :encoding => "#{encoding}:UTF-8")
-        true
         
       elsif mode == :file
         # Files have a different path
-        encoding = @importer.encoding || 'UTF-8'
-        @raw_rows = CSV.read(source, :encoding => "#{encoding}:UTF-8")
-        true
+        text = File.read(source)
         
       else
-        @importer.add_error("Unsupported CSV mode: #{mode}")
-        false
+        # WTF?
+        @importer.add_error("Unsupported CSV mode: #{mode.inspect}")
+        return false
+      end
+
+      # Fix shitty Windows line-feeds so things are standardized
+      text.gsub!(/\r\n/, "\n")
+      text.gsub!(/\r/, "\n")
+
+      # Parse it out
+      encoding = @importer.encoding || 'UTF-8'
+      options = {
+        :encoding => "#{encoding}:UTF-8",
+        :skip_blanks => true
+      }
+      begin
+        @raw_rows = CSV.parse(text, options)
+      rescue Exception => e
+        @importer.add_error('Error encountered while parsing CSV')
+        @importer.add_exception(e)
       end
     end
    

data/lib/iron/import/data_reader.rb

@@ -9,7 +9,7 @@ class Importer
     attr_reader :format
 
     def self.verify_roo!
-      if Gem::Specification.find_all_by_name('roo', '~> 1.13.0').empty?
+      if Gem::Specification.find_all_by_name('roo', '>= 1.13.0').empty?
         raise "You are attempting to use the iron-import gem to import an Excel file.  Doing so requires installing the roo gem, version 1.13.0 or later."
       else
         require 'roo'
@@ -17,7 +17,7 @@ class Importer
     end
 
     def self.verify_nokogiri!
-      if Gem::Specification.find_all_by_name('nokogiri', '~> 1.6.0').empty?
+      if Gem::Specification.find_all_by_name('nokogiri', '>= 1.6.0').empty?
         raise "You are attempting to use the iron-import gem to import an HTML file.  Doing so requires installing the nokogiri gem, version 1.6.0 or later."
       else
         require 'nokogiri'

data/spec/importer/csv_reader_spec.rb

@@ -5,6 +5,24 @@ describe Importer::CsvReader do
     @reader = Importer::CsvReader.new(@importer)
   end
   
+  it 'should convert to standard newlines' do
+    importer = Importer.build do
+      headerless!
+      column :number do
+        type :integer
+      end
+      column :string do
+        type :string
+      end
+    end      
+    importer.import_string("1,\"foo\nbar\"\r\n2,hi\r3,yo").should be_true
+    importer.to_a.should == [
+      {:number => 1, :string => "foo\nbar"},
+      {:number => 2, :string => "hi"},
+      {:number => 3, :string => "yo"}
+    ]
+  end
+  
   it 'should load our simple CSV data' do
     importer = Importer.build do
       column :number do

data/spec/importer/importer_spec.rb

@@ -286,4 +286,16 @@ describe Importer do
     importer.found_columns.count.should == 2
   end
   
+  it 'should search multiple sheets to find header' do
+    importer = Importer.build do
+      column :date do
+        type :date
+      end
+      column :order
+    end
+    importer.import(SpecHelper.sample_path('2-sheets.xlsx')).should be_true
+    importer.errors.count.should == 0
+    importer.to_a.should == [{:order => '223300', :date => '1973-01-02'.to_date}]
+  end
+  
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: iron-import
 version: !ruby/object:Gem::Version
-  version: 0.8.3
+  version: 0.8.4
 platform: ruby
 authors:
 - Rob Morris
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-08-22 00:00:00.000000000 Z
+date: 2018-01-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: iron-extensions
@@ -123,6 +123,8 @@ files:
 - spec/importer/row_spec.rb
 - spec/importer/xls_reader_spec.rb
 - spec/importer/xlsx_reader_spec.rb
+- spec/samples/2-sheets.xls
+- spec/samples/2-sheets.xlsx
 - spec/samples/3-sheets.xls
 - spec/samples/col-span.html
 - spec/samples/html-th-td.html