iron-import 0.6.1 → 0.7.0

data/spec/samples/scores.html

@@ -0,0 +1,30 @@
+<table id="table_4ae4f429_6b24_496d_8dae_7c31a7644644" class="mx-grid boxscore d-b-s">
+	<thead>
+		<tr class="primary-header-row ">
+			<th class=" team string first" scope="col" data-cl="9"><span>1/31 @ 8p</span></th>
+      <th class=" quarter1 string score dw" scope="col" data-cl="2" title="Quarter 1"><span>Q1</span></th>
+      <th class=" quarter2 string score dw" scope="col" data-cl="2" title="Quarter 2"><span>Q2</span></th>
+      <th class=" quarter3 string score dw" scope="col" data-cl="2" title="Quarter 3"><span>Q3</span></th>
+      <th class=" quarter4 string score dw" scope="col" data-cl="2" title="Quarter 4"><span>Q4</span></th>
+      <th class=" score string last total score" scope="col" data-cl="5" title="Final"><span>Final</span></th>
+		</tr>
+	</thead>
+  <tbody>
+    <tr class="first">
+      <th class="team first" scope="row"><a  href="/high-schools/mcdowell-titans-(marion,nc)/basketball/home.htm" >McDowell</a></th>
+      <td class="quarter1 score dw">16</td>
+      <td class="quarter2 score dw">20</td>
+      <td class="quarter3 score dw">27</td>
+      <td class="quarter4 score dw">17</td>
+      <td class="score last total score">80</td>
+    </tr>
+    <tr class="last alternate">
+      <th class="team first" scope="row"><a  href="/high-schools/asheville-cougars-(asheville,nc)/basketball/home.htm" >Asheville</a></th>
+      <td class="quarter1 score dw">13</td>
+      <td class="quarter2 score dw">17</td>
+      <td class="quarter3 score dw">17</td>
+      <td class="quarter4 score dw">13</td>
+      <td class="score last total score">60</td>
+    </tr>
+  </tbody>
+</table>

data/spec/samples/simple.html

@@ -0,0 +1,14 @@
+<table>
+  <tr>
+    <td>Name</td>
+    <td>ID</td>
+  </tr>
+  <tr>
+    <td>John</td>
+    <td>888</td>
+  </tr>
+  <tr>
+    <td>Anne</td>
+    <td>1234</td>
+  </tr>
+</table>

data/spec/spec_helper.rb

@@ -1,5 +1,6 @@
 # Set up development requirements
 require 'roo'
+require 'nokogiri'
 
 # Require our library
 require File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib', 'iron', 'import'))

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: iron-import
 version: !ruby/object:Gem::Version
-  version: 0.6.1
+  version: 0.7.0
 platform: ruby
 authors:
 - Rob Morris
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-08-24 00:00:00.000000000 Z
+date: 2017-02-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: iron-extensions
@@ -72,9 +72,23 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.13'
-description: Simple yet powerful library for importing tabular data including support
-  for auto-detecting column order, parsing/validating cell data, aggregating errors,
-  etc.
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+description: Simple yet powerful library for importing tabular data from CSV, HTML,
+  XLS and XLSX files, including support for auto-detecting column order, parsing/validating
+  cell data, aggregating errors, etc.
 email:
 - rob@irongaze.com
 executables: []
@@ -93,22 +107,30 @@ files:
 - lib/iron/import/custom_reader.rb
 - lib/iron/import/data_reader.rb
 - lib/iron/import/error.rb
+- lib/iron/import/excel_reader.rb
+- lib/iron/import/html_reader.rb
 - lib/iron/import/importer.rb
 - lib/iron/import/row.rb
-- lib/iron/import/sheet.rb
 - lib/iron/import/xls_reader.rb
 - lib/iron/import/xlsx_reader.rb
 - spec/importer/column_spec.rb
 - spec/importer/csv_reader_spec.rb
 - spec/importer/custom_reader_spec.rb
 - spec/importer/data_reader_spec.rb
+- spec/importer/html_reader_spec.rb
 - spec/importer/importer_spec.rb
 - spec/importer/row_spec.rb
-- spec/importer/sheet_spec.rb
+- spec/importer/xls_reader_spec.rb
 - spec/importer/xlsx_reader_spec.rb
+- spec/samples/3-sheets.xls
+- spec/samples/col-span.html
+- spec/samples/html-th-td.html
 - spec/samples/icd10-custom.txt
+- spec/samples/multi-table.html
 - spec/samples/nanodrop.xlsx
+- spec/samples/scores.html
 - spec/samples/simple.csv
+- spec/samples/simple.html
 - spec/samples/test-products.xls
 - spec/spec_helper.rb
 homepage: http://irongaze.com
@@ -134,5 +156,5 @@ rubyforge_project:
 rubygems_version: 2.4.3
 signing_key: 
 specification_version: 4
-summary: CSV, XLS, and XLSX import automation support
+summary: CSV, HTML, XLS, and XLSX import automation support
 test_files: []

data/lib/iron/import/sheet.rb

@@ -1,263 +0,0 @@
-class Importer
-  
-  # The Sheet class handles building the sheet's column configuration and other
-  # setup, then holds all load-time row data.  In some file types (Excel mostly)
-  # there may be more than one sheet definition in a given importer.  In others,
-  # the default sheet is the only one (possibly implicitly) defined.
-  #
-  # The following builder options are available:
-  # 
-  #   Importer.build do
-  #     sheet('Some Sheet Name') do
-  #       # Don't try to look for a header using column definitions, there is no header
-  #       headerless!
-  #
-  #       # Manually set the start row for data in this sheet, defaults to nil
-  #       # indicating that the data rows start immediatly following the header.
-  #       start_row 4
-  #
-  #       # Define a filter that will skip unneeded rows.  The filter command takes
-  #       # a block that receives the parsed (but not validated!) row data as an 
-  #       # associative hash of :col_key => <parsed value>, and returns 
-  #       # true to keep the row or false to exclude it.
-  #       filter do |row|
-  #         row[:id].to_i > 5000
-  #       end
-  #
-  #       # Of course, the main thing to do in a sheet is define columns.  See the
-  #       # Column class' notes for options when defining a column.  Note that
-  #       # you can define columns using either hash-style:
-  #       column :id, :type => :integer
-  #       # or builder-style:
-  #       column :name do
-  #         header /company\s*name/
-  #         type :string
-  #       end
-  #    end
-  class Sheet
-    
-    # Inner class for holding load-time data that gets reset on each load call
-    class Data
-      attr_accessor :start_row, :rows
-      def initialize
-        @start_row = nil
-        @rows = []
-      end
-    end
-    
-    # Key data
-    attr_reader :importer
-    attr_reader :columns
-    attr_reader :data
-
-    # Settings
-    dsl_flag :headerless
-    dsl_accessor :id
-    dsl_accessor :start_row
-    dsl_accessor :filter
-    
-    def initialize(importer, id)
-      @importer = importer
-      @id = id
-
-      @headerless = false
-      @start_row = nil
-      @filter = nil
-      
-      @columns = []
-      
-      reset
-    end
-
-    # Define our columns etc. via builder-style method calling
-    def build(&block)
-      DslProxy.exec(self, &block)
-    end
-    
-    # Call with a block accepting a single Importer::Row with contents that
-    # look like :column_key => <parsed value>.  Any filtered rows
-    # will not be present.  If you want to register an error, simply 
-    # raise "some text" and it will be added to the importer's error
-    # list for display to the user, logging, or whatever.
-    def process
-      @data.rows.each do |row|
-        begin
-          yield row
-        rescue Exception => e
-          @importer.add_error(row, e.to_s)
-        end
-      end
-    end
-    
-    # Add a new column definition to our list, allows customizing the new
-    # column with a builder block.  See Importer::Column docs for 
-    # options.  In lieu of a builder mode, you can pass the same values
-    # as key => value pairs in the options hash to this method, so:
-    #
-    #   column(:foo) do
-    #     type :string
-    #     parse do |val|
-    #       val.to_s.upcase
-    #     end
-    #   end
-    # 
-    # Is equivalent to:
-    #
-    #   column(:foo, :type => :string, :parse => lambda {|val| val.to_s.upcase})
-    #
-    # Use whichever you prefer!
-    def column(key, options_hash = {}, &block)
-      # Find existing column with key to allow re-opening an existing definition
-      col = @columns.detect {|c| c.key == key }
-      unless col
-        # if none found, add a new one
-        col = Column.new(self, key, options_hash)
-        @columns << col
-      end
-      
-      # Customize if needed
-      DslProxy::exec(col, &block) if block
-      
-      col
-    end
-    
-    # Reset for load attempt
-    def reset
-      @data = Data.new
-    end
-    
-    def parse_raw_data(raw_rows)
-      # Find our column layout, start of data, etc
-      if parse_header(raw_rows)
-        # Now, run all the data and add it as a Row instance
-        raw_rows.each_with_index do |raw, index|
-          row_num = index + 1
-          if row_num >= @data.start_row
-            add_row(row_num, raw)
-          end
-        end
-      end
-    end
-    
-    # Add a new row to our stash
-    def add_row(line, raw_data)
-      # Gracefully handle custom parsers that return nil for a row's data
-      raw_data ||= []
-      # Add the row
-      row = Row.new(self, line)
-      
-      # Parse out the values
-      values = {}
-      @columns.each do |col|
-        index = col.data.index
-        raw_val = raw_data[index]
-        if col.parse
-          # Use custom parser if this row has one
-          val = col.parse_value(row, raw_val)
-        else
-          # Otherwise use our standard parser
-          val = @importer.data.parse_value(raw_val, col.type)
-        end
-        values[col.key] = val
-      end
-
-      # Set the values and filter if needed
-      row.set_values(values)
-      return nil unless !@filter || @filter.call(row)
-      
-      # Row is solid, now check for missing required vals
-      @columns.each do |col|
-        val = values[col.key]
-        if col.validate_value(row, val)
-          if col.required?
-            if values[col.key].nil?
-              @importer.add_error(row, "Missing required value for #{col}")
-            end
-          end
-        end
-      end
-      
-      # We is good
-      @data.rows << row
-      row
-    end
-    
-    # Process the raw values for the first rows in a sheet,
-    # and attempt to build a map of the column layout, and
-    # detect the first row of real data
-    def parse_header(raw_rows)
-      if headerless?
-        # Use implicit or explicit column position when told to not look for a header
-        next_index = 0
-        @columns.each do |col|
-          unless col.position.nil?
-            next_index = col.fixed_index
-          end
-          col.data.index = next_index
-          next_index += 1
-        end
-        @data.start_row = @start_row || 1
-        return true
-        
-      else
-        # Match by testing
-        raw_rows.each_with_index do |row, i|
-          # Um, have data?
-          next unless row
-          
-          # Set up for this iteration
-          remaining = @columns.dup
-      
-          # Step through this row's raw values, and look for a matching column for all columns
-          row.each_with_index do |val, i|
-            col = remaining.detect {|c| c.match_header?(val.to_s, i) }
-            if col
-              remaining -= [col]
-              col.data.index = i
-            end
-          end
-          
-          if remaining.empty?
-            # Found the cols, have a map, update our start row to be the next line and return!
-            @data.start_row = @start_row || i+2
-            return true
-          end
-        end
-        
-        # If we get here, we're hosed
-        @importer.add_error(self, "Unable to locate required column header(s) in sheet")
-        false
-      end
-    end
-    
-    # When true, the given sheet name or zero-based index
-    # is a match with our id.
-    def match_sheet?(name, index)
-      if @id.is_a?(Fixnum)
-        @id.to_i == index+1
-      else
-        @id.to_s.downcase == name.downcase
-      end
-    end
-
-    def add_error(msg)
-      @importer.add_error(self, msg)
-    end
-    
-    def add_warning(msg)
-      @importer.add_warning(self, msg)
-    end
-
-    def to_s
-      "Sheet #{@id}"
-    end
-    
-    # Return all parsed, filtered data in the sheet as an
-    # array of arrays.
-    def dump
-      @data.rows.collect(&:values)
-    end
-
-  end
-  
-end

data/spec/importer/sheet_spec.rb

@@ -1,65 +0,0 @@
-describe Importer::Sheet do
-
-  before do
-    @importer = Importer.new
-    @sheet = @importer.default_sheet
-  end
-
-  it 'should respond to build' do
-    @sheet.should respond_to(:build)
-    @sheet.build do
-      column :foo
-    end
-    @sheet.columns.count.should == 1
-  end
-  
-  it 'should define columns' do
-    @sheet.column(:foo)
-    @sheet.columns.count.should == 1
-  end
-  
-  it 'should find headers automatically' do
-    # Define a few sample columns
-    @sheet.column(:alpha)
-    @sheet.column(:gamma)
-    # Some dummy data
-    rows = [
-      ['', '', '', ''],
-      ['Alpha', 'Beta', 'Gamma', 'Epsilon']
-    ]
-    
-    # Parse it!
-    @sheet.parse_header(rows).should be_true
-    
-    @sheet.column(:alpha).data.index.should == 0
-    @sheet.column(:gamma).data.index.should == 2
-    @sheet.data.start_row.should == 3
-  end
-  
-  it 'should record an error if a column can\'t be found' do
-    # Define a few sample columns
-    @sheet.column(:alpha)
-    @sheet.column(:gamma)
-    # Some dummy data
-    rows = [
-      ['', '', '', ''],
-      ['Bob', 'Beta', 'Gamma', 'Epsilon']
-    ]
-    
-    # Parse it!
-    @sheet.parse_header(rows).should be_false
-    @importer.errors.count.should == 1
-    @importer.error_summary.should =~ /unable to locate required column header/i
-  end
-  
-  it 'should match by sheet name or number' do
-    @sheet.id = 5
-    @sheet.match_sheet?('foo', 3).should be_false
-    @sheet.match_sheet?('foo', 4).should be_true
-    
-    @sheet.id = 'Sheet 5'
-    @sheet.match_sheet?('Sheet', 4).should be_false
-    @sheet.match_sheet?('Sheet 5', 3).should be_true
-  end
-
-end