iron-import 0.6.1 → 0.7.0

data/lib/iron/import/xls_reader.rb

@@ -1,46 +1,12 @@
 class Importer
   
-  class XlsReader < DataReader
+  # Uses the Roo gem to read in .xls files
+  class XlsReader < ExcelReader
     
     def initialize(importer)
-      super(importer, :xlsx)
+      super(importer, :xls)
     end
     
-    def init_source(mode, source)
-      if mode == :file
-        @spreadsheet = Roo::Excel.new(source, :file_warning => :ignore)
-        true
-      else
-        @importer.add_error("Unsupported XLS mode: #{mode}")
-        false
-      end
-    rescue Exception => e
-      @importer.add_error("Error reading file #{source}: #{e}")
-      false
-    end
-    
-    def load_raw_sheet(sheet)
-      @spreadsheet.sheets.each_with_index do |name, index|
-        # See if this sheet's name or index matches the requested sheet definition
-        if sheet.match_sheet?(name, index)
-          # Extract our raw data
-          raw_rows = []
-          @spreadsheet.sheet(name).each_with_index do |row, line|
-            raw_rows << row
-          end
-          return raw_rows
-        end
-      end
-      # This is not good.
-      @importer.add_error("Unable to find sheet #{sheet}")
-      return false
-
-    rescue Exception => e
-      # Not sure why we'd get here, but we strive for error-freedom here, yessir.
-      @importer.add_error("Error loading sheet #{sheet}: #{e}")
-      false
-    end
-  
   end
   
 end

data/lib/iron/import/xlsx_reader.rb

@@ -1,47 +1,12 @@
 class Importer
   
   # Uses the Roo gem to read in .xlsx files
-  class XlsxReader < DataReader
+  class XlsxReader < ExcelReader
     
     def initialize(importer)
       super(importer, :xlsx)
-      supports_file!
     end
-    
-    def init_source(mode, source)
-      if mode == :file
-        @spreadsheet = Roo::Excelx.new(source, :file_warning => :ignore)
-        true
-      else
-        @importer.add_error("Unsupported XLSX mode: #{mode}")
-        false
-      end
-    rescue Exception => e
-      @importer.add_error("Error reading file #{source}: #{e}")
-      false
-    end
-    
-    def load_raw_sheet(sheet)
-      @spreadsheet.sheets.each_with_index do |name, index|
-        # See if this sheet's name or index matches the requested sheet definition
-        if sheet.match_sheet?(name, index)
-          # Extract our raw data
-          raw_rows = []
-          @spreadsheet.sheet(name).each_with_index do |row, line|
-            raw_rows << row
-          end
-          return raw_rows
-        end
-      end
-      @importer.add_error("Unable to find sheet #{sheet}")
-      return false
-      
-    rescue Exception => e
-      # Not sure why we'd get here, but we strive for error-freedom here, yessir.
-      @importer.add_error("Error loading sheet #{sheet}: #{e}")
-      false
-    end
-    
+ 
   end
   
 end

data/lib/iron/import.rb

@@ -4,11 +4,12 @@ require 'iron/dsl'
 
 # Include required classes
 require_relative 'import/column'
-require_relative 'import/sheet'
 require_relative 'import/row'
 require_relative 'import/error'
 require_relative 'import/data_reader'
 require_relative 'import/csv_reader'
+require_relative 'import/html_reader'
+require_relative 'import/excel_reader'
 require_relative 'import/xls_reader'
 require_relative 'import/xlsx_reader'
 require_relative 'import/custom_reader'

data/spec/importer/column_spec.rb

@@ -2,17 +2,16 @@ describe Importer::Column do
 
   before do
     @importer = Importer.new
-    @sheet = @importer.default_sheet
-    @col = Importer::Column.new(@sheet, :test)
-    @row = Importer::Row.new(@sheet, 1)
+    @col = Importer::Column.new(@importer, :test)
+    @row = Importer::Row.new(@importer, 1)
   end
 
   it 'should respond to build' do
     @col.should respond_to(:build)
     @col.build do
-      required!
+      type :cents
     end
-    @col.required?.should be_true
+    @col.type.should == :cents
   end
   
   it 'should convert position strings to indexes' do

data/spec/importer/csv_reader_spec.rb

@@ -21,7 +21,7 @@ describe Importer::CsvReader do
       end
     end
     importer.import(SpecHelper.sample_path('simple.csv')).should be_true
-    importer.default_sheet.dump.should == [
+    importer.to_a.should == [
       {:number => 123, :string => 'Abc', :date => Date.new(1977,5,13), :cost => 899},
       {:number => nil, :string => nil, :date => nil, :cost => nil},
       {:number => 5, :string => 'String with end spaces', :date => Date.new(2004,2,1), :cost => 1000}

data/spec/importer/custom_reader_spec.rb

@@ -8,7 +8,7 @@ describe Importer::CustomReader do
     @importer.custom_reader.should be_nil
     @importer.build do
       headerless!
-      on_file do |source, sheet|
+      on_file do |source|
         []
       end
     end
@@ -20,14 +20,10 @@ describe Importer::CustomReader do
   it 'should load the ICD10 test document' do
     importer = Importer.build do
       headerless!
-      column :code do
-        required!
-      end
-      column :desc do
-        required!
-      end
+      column :code
+      column :desc
 
-      on_file do |source, sheet|
+      on_file do |source|
         File.readlines(source).collect do |line|
           line.extract(/([A-TV-Z][0-9][A-Z0-9]{1,5})\s+(.*)/)
         end
@@ -35,7 +31,7 @@ describe Importer::CustomReader do
     end
     importer.import(SpecHelper.sample_path('icd10-custom.txt'))
     importer.error_summary.should be_nil
-    importer.default_sheet.dump.should == [
+    importer.to_a.should == [
       {:code => 'A000', :desc => 'Cholera due to Vibrio cholerae 01, biovar cholerae'},
       {:code => 'A001', :desc => 'Cholera due to Vibrio cholerae 01, biovar eltor'},
       {:code => 'A009', :desc => 'Cholera, unspecified'},
@@ -49,7 +45,7 @@ describe Importer::CustomReader do
       column :code
       column :desc
 
-      on_file do |source, sheet|
+      on_file do |source|
         add_error('Unable to read cause no reader')
       end
     end

data/spec/importer/data_reader_spec.rb

@@ -15,7 +15,7 @@ describe Importer::DataReader do
       55 => 55,
       3.0 => 3
     }.each_pair do |val, res|
-      @reader.parse_value(val, :integer).should == res
+      @reader.parse_value(val, :integer).should === res
     end
   end
   
@@ -29,7 +29,7 @@ describe Importer::DataReader do
       55 => 55.0,
       '3' => 3.0
     }.each_pair do |val, res|
-      @reader.parse_value(val, :float).should == res
+      @reader.parse_value(val, :float).should === res
     end
   end
   
@@ -41,13 +41,14 @@ describe Importer::DataReader do
       255 => '255',
       -1.5 => '-1.5'
     }.each_pair do |val, res|
-      @reader.parse_value(val, :string).should == res
+      @reader.parse_value(val, :string).should === res
     end
   end
   
   it 'should parse cents' do
     {
       '$123.00' => 12300,
+      '9.95' => 995,
       '5' => 500,
       '0.5' => 50,
       '-95' => -9500,
@@ -55,7 +56,7 @@ describe Importer::DataReader do
       1.0 => 100,
       1.25 => 125
     }.each_pair do |val, res|
-      @reader.parse_value(val, :cents).should == res
+      @reader.parse_value(val, :cents).should === res
     end
   end
   
@@ -68,7 +69,7 @@ describe Importer::DataReader do
       '5/10/2014, 10:28:07 PM' => Date.new(2014,5,10),
       Date.new(2000,4,1) => Date.new(2000,4,1)
     }.each_pair do |val, res|
-      @reader.parse_value(val, :date).should == res
+      @reader.parse_value(val, :date).should === res
     end
   end
   

data/spec/importer/html_reader_spec.rb

@@ -0,0 +1,105 @@
+describe Importer::HtmlReader do
+
+  it 'should load a simple table' do
+    importer = Importer.build do
+      column :name
+      column :id do
+        type :int
+      end
+    end
+    res = importer.import(SpecHelper.sample_path('simple.html'))
+    importer.error_summary.should be_nil
+    res.should be_true
+    importer.to_a.should == [
+      {:name => 'John', :id => 888},
+      {:name => 'Anne', :id => 1234}
+    ]
+  end
+  
+  it 'should honor start row' do
+    txt = '<table><tr><th>Notes From Clark:</th><td class="notes">Can you please send us 5 more white hooks for your display. Please rush ship this order. Thank you!</td></tr></table>'
+    importer = Importer.build do
+      start_row 1
+      
+      column :note_header do
+        header /Notes From/i
+      end
+      column :note do
+        position 2
+      end
+    end
+    importer.import_string(txt).should be_true
+    importer.data.start_row.should == 1
+    importer.to_a.should == [{:note_header => 'Notes From Clark:', :note => 'Can you please send us 5 more white hooks for your display. Please rush ship this order. Thank you!'}]
+  end
+  
+  it 'should properly expand colspan cells' do
+    importer = Importer.build do
+      column :one
+      column :two
+      column :three
+    end
+    res = importer.import(SpecHelper.sample_path('col-span.html'))
+    importer.error_summary.should be_nil
+    res.should be_true
+    importer.to_a.each do |row|
+      row[:one].should == 'First' unless row[:one].nil?
+      row[:two].should == 'Second' unless row[:two].nil?
+      row[:three].should == 'Third' unless row[:three].nil?
+    end
+  end
+  
+  it 'should limit search by scope' do
+    importer = Importer.build do
+      column :alpha
+      column :beta
+      column :gamma
+    end
+    res = importer.import(SpecHelper.sample_path('multi-table.html'))
+    importer.error_summary.should be_nil
+    res.should be_true
+    importer.to_a.should == [
+      {:alpha => '1', :beta => '2', :gamma => '3'},
+      {:alpha => '4', :beta => '5', :gamma => '6'}
+    ]
+
+    res = importer.import(SpecHelper.sample_path('multi-table.html'), :scope => '.second table')
+    importer.error_summary.should be_nil
+    res.should be_true
+    importer.to_a.should == [
+      {:alpha => '7', :beta => '8', :gamma => '9'}
+    ]
+  end
+  
+  it 'should strip tags from cells' do
+    importer = Importer.build do
+      column :q1 do
+        header /^Q1$/
+      end
+    end
+    res = importer.import(SpecHelper.sample_path('scores.html'))
+    importer.error_summary.should be_nil
+    res.should be_true
+    importer.to_a.should == [
+      {:q1 => '16'},
+      {:q1 => '13'}
+    ]
+  end
+  
+  it 'should treat th and td cells impartially and return in order' do
+    importer = Importer.build do
+      column :a
+      column :b
+      column :c
+      column :d
+    end
+    res = importer.import(SpecHelper.sample_path('html-th-td.html'))
+    importer.error_summary.should be_nil
+    res.should be_true
+    importer.to_a.should == [
+      {:a => '1', :b => '2', :c => '3', :d => '4'},
+      {:a => '1', :b => '2', :c => '3', :d => '4'}
+    ]
+  end
+  
+end

data/spec/importer/importer_spec.rb

@@ -2,6 +2,86 @@ describe Importer do
 
   it 'should respond to build' do
     Importer.should respond_to(:build)
+    importer = Importer.build do
+      column :foo
+    end
+    importer.columns.count.should == 1
+  end
+  
+  it 'should set single search scopes' do
+    importer = Importer.build do
+      scope :xls, 'Sheet 2'
+    end
+    importer.scopes.should == { :xls => ['Sheet 2'] }
+  end
+  
+  it 'should set multiple search scopes' do
+    importer = Importer.build do
+      scopes :xls => [1, 'Sheet 2'],
+        :html => 'table.funny'
+    end
+    importer.scopes.should == { :xls => [1, 'Sheet 2'], :html => ['table.funny'] }
+  end
+  
+  it 'should find headers automatically' do
+    # Define a few sample columns
+    importer = Importer.new
+    importer.column(:alpha)
+    importer.column(:gamma)
+    # Some dummy data
+    rows = [
+      ['', '', '', ''],
+      ['Alpha', 'Beta', 'Gamma', 'Epsilon']
+    ]
+
+    # Parse it!
+    importer.find_header(rows).should be_true
+
+    importer.column(:alpha).data.index.should == 0
+    importer.column(:gamma).data.index.should == 2
+    importer.data.start_row.should == 3
+  end
+  
+  it 'should report missing columns' do
+    # Define a few sample columns
+    importer = Importer.new
+    importer.column(:alpha)
+    importer.column(:gamma)
+    # Some dummy data
+    rows = [
+      ['Bob', 'Beta', 'Gamma', 'Epsilon']
+    ]
+
+    # Parse it!
+    importer.find_header(rows).should be_false
+    importer.missing_headers.should == [:alpha]
+  end
+
+  it 'should capture errors' do
+    importer = Importer.build do
+      column :foo
+    end
+    importer.add_error('An error')
+    importer.has_errors?.should be_true
+    importer.errors.count.should == 1
+  end
+  
+  it 'should run conditional code when errors are present' do
+    importer = Importer.build do
+      column :foo
+    end
+    was_run = false
+
+    importer.on_error do
+      was_run = true
+    end
+    was_run.should be_false
+
+    importer.add_error('An error')
+    importer.on_error do
+      was_run = true
+    end
+    was_run.should be_true
   end
   
   it 'should import a test csv file' do
@@ -25,4 +105,31 @@ describe Importer do
     count.should == 3
   end
   
+  it 'should import a string' do
+    sum = 0
+    csv = "one,two\n1,2"
+    Importer.build do
+      column :one
+      column :two
+    end.import_string(csv, :format => :csv) do |rows|
+      rows[:one].should == '1'
+      rows[:two].should == '2'
+      sum = rows[:one].to_i + rows[:two].to_i
+    end
+    # Just make sure we ran correctly
+    sum.should == 3
+  end
+  
+  it 'should pick the proper format based on content' do
+    importer = Importer.build do
+      column :one
+      column :two
+    end
+    importer.format.should be_nil
+    importer.import_string("one,two\n1,2")
+    importer.format.should == :csv
+    importer.import_string("<div><table><tr><td>one</td></tr></table></div>")
+    importer.format.should == :html
+  end
+  
 end

data/spec/importer/row_spec.rb

@@ -2,8 +2,7 @@ describe Importer::Row do
 
   before do
     @importer = Importer.new
-    @sheet = @importer.default_sheet
-    @row = Importer::Row.new(@sheet, 5)
+    @row = Importer::Row.new(@importer, 5)
   end
 
   it 'should store and retrieve values' do
@@ -33,5 +32,13 @@ describe Importer::Row do
     @row.set_values(:a => nil, :b => nil)
     @row.should be_empty
   end
+  
+  it 'should not change when to_hash values are changed' do
+    @row.set_values(:a => 1, :b => 2)
+    hash = @row.to_hash
+    hash.should == {:a => 1, :b => 2}
+    hash.delete(:a)
+    @row[:a].should == 1
+  end
 
 end

data/spec/importer/xls_reader_spec.rb

@@ -0,0 +1,77 @@
+describe Importer::XlsReader do
+
+  it 'should read our products sample' do
+    importer = Importer.build do
+      column :part_num do
+        header /part/i
+      end
+      column :quantity do
+        type :int
+      end
+      column :desc do
+        header /description/i
+      end
+      column :unit_cost do
+        type :cents
+      end
+      column :total_cost do
+        type :cents
+      end
+    end
+    importer.import(SpecHelper.sample_path('test-products.xls'))
+    importer.error_summary.should be_nil
+    importer.to_a.should == [
+      {:part_num=>"00245",
+       :quantity=>2,
+       :desc=>"Washer",
+       :unit_cost=>899,
+       :total_cost=>1798},
+      {:part_num=>"10855",
+       :quantity=>4,
+       :desc=>"Misc Bits",
+       :unit_cost=>1000,
+       :total_cost=>4000},
+      {:part_num=>"19880-2",
+       :quantity=>3,
+       :desc=>"A duck!",
+       :unit_cost=>10731,
+       :total_cost=>32193},
+      {:part_num=>"18098-8",
+       :quantity=>1,
+       :desc=>"Tuesday",
+       :unit_cost=>5500,
+       :total_cost=>5500}
+    ]
+  end
+
+  it 'should search by scope' do
+    importer = Importer.build do
+      column :sheet do
+        type :int
+      end
+      column :val
+      
+      filter do |row|
+        row.all?
+      end
+    end
+
+    # Default case
+    res = importer.import(SpecHelper.sample_path('3-sheets.xls'))
+    importer.format.should == :xls
+    importer.error_summary.should be_nil
+    importer.to_a.should == [{:sheet => 1, :val => 'Monkey'}]
+
+    # Pass scope to import
+    res = importer.import(SpecHelper.sample_path('3-sheets.xls'), :scope => 2)
+    importer.error_summary.should be_nil
+    importer.to_a.should == [{:sheet => 2, :val => 'Rhino'}]
+
+    # Define scope on importer
+    importer.scope :xls, 'Sheet 3'
+    res = importer.import(SpecHelper.sample_path('3-sheets.xls'))
+    importer.error_summary.should be_nil
+    importer.to_a.should == [{:sheet => 3, :val => 'Ant'}]
+  end
+  
+end

data/spec/importer/xlsx_reader_spec.rb

@@ -1,9 +1,8 @@
 describe Importer::XlsxReader do
 
-  it 'should load our nanodrop data' do
+  it 'should load our nanodrop sample' do
     importer = Importer.build do
       column :sample_id do
-        required!
         validate do |val|
           raise 'Invalid ID' unless val.match(/[0-9]{3,}\.[0-9]\z/)
         end
@@ -26,7 +25,7 @@ describe Importer::XlsxReader do
     res = importer.import(SpecHelper.sample_path('nanodrop.xlsx'))
     importer.error_summary.should be_nil
     res.should be_true
-    importer.default_sheet.dump.should == [
+    importer.to_a.should == [
       {:sample_id => 'Windsor_buccal_500.1', :a260 => 2.574, :a280 => 1.277, :factor => 50},
       {:sample_id => 'Weston_fecal_206.2', :a260 => 0.746, :a280 => 0.351, :factor => 50}
     ]

data/spec/samples/col-span.html

@@ -0,0 +1,29 @@
+<table>
+  <tr>
+    <td>
+      One
+    </td>
+    <td>
+      Two
+    </td>
+    <td>
+      Three
+    </td>
+  </tr>
+  <tr>
+    <td colspan="2">First</td>
+    <td>Third</td>
+  </tr>
+  <tr>
+    <td>First</td>
+    <td colspan="2">Second</td>
+  </tr>
+  <tr>
+    <td colspan="3">First</td>
+  </tr>
+  <tr>
+    <td>First</td>
+    <td>Second</td>
+    <td>Third</td>
+  </tr>
+</table>

data/spec/samples/html-th-td.html

@@ -0,0 +1,11 @@
+<table>
+  <tr>
+    <th>A</th><th>B</th><th>C</th><td>D</td>
+  </tr>
+  <tr>
+    <th>1</th><td>2</td><td>3</td><th>4</th>
+  </tr>
+  <tr>
+    <td>1</td><td>2</td><th>3</th><td>4</td>
+  </tr>
+</table>

data/spec/samples/multi-table.html

@@ -0,0 +1,29 @@
+<!doctype html>
+<html>
+<body>
+<p>
+  Some text!  I like table.
+</p>
+<table>
+  <tr>
+    <td>Alpha</td><td>Beta</td><td>Gamma</td>
+  </tr>
+  <tr>
+    <td>1</td><td>2</td><td>3</td>
+  </tr>
+  <tr>
+    <td>4</td><td>5</td><td>6</td>
+  </tr>
+</table>
+<div class="second">
+  <table>
+    <tr>
+      <td>Alpha</td><td>Beta</td><td>Gamma</td>
+    </tr>
+    <tr>
+      <td>7</td><td>8</td><td>9</td>
+    </tr>
+  </table>
+</div>
+</body>
+</html>