iron-import 0.6.1 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e989c98b9a8dd112ec7c5926f971934e162b2017
-  data.tar.gz: 7d0e72179fded329171398514a215e7176879bc0
+  metadata.gz: 6931785ff3b1cb03394e6a8be0943337c589ac90
+  data.tar.gz: 82a7b1f123134eea5388672e8aed578590cd2daa
 SHA512:
-  metadata.gz: 59449ccc93220e21c21b8bee5dfeea71ecf91cafb52fcff0832b991c0c408e82ef7c788da988c526ec0a471426f1b05a3c0fb28bdf7648af226976d88d3f83ed
-  data.tar.gz: 04c84036d445c7117a0dbedff629cf2197533d533243fdaa18a82021e6258e95aeea10153357eb95da625229bbdb844a9554f4457bdd453bc1c3463095f6b5b1
+  metadata.gz: d5d689572ef2680fce8273cc22c05ba76b2dc9a8a56f07c890d64ba6f1b4d4aba78f6bffea8f2b2b2eb9bdf057115bcf6911241f1febe567c7caee3a9eea6011
+  data.tar.gz: 4fafd479220b82ed7fd983b77b1c54d73ba993442d29c4b19a37c087e94feb35c25517549966fff2426be333491f71f59e23c20650c02a3ca5f6b34e4e6f4e91

data/History.txt

@@ -1,3 +1,18 @@
+== 0.7.0 / 2017-02-16
+
+* Breaking Change: Removed multi-sheet support - use multiple importers instead
+* Breaking Change: Removed warnings as they were not being used
+* Breaking Change: Removed Column#required! due to bugginess and overlap with Column#validate
+* Add Importer#scope to allow narrowing the search to one or more sheets/tables when importing
+* Add new HtmlReader support to handle parsing HTML <table> rows
+* Modify Importer#import to support block mode combining #import and #process
+* Add Importer#import_string for handling explicit CSV/HTML/Custom text
+* Add Importer#on_error(&block) to allow inline conditional error handling
+* Improve error message when headers can't be detected to list missing headers
+* Change Importer#error_summary to group identical errors into single summary line
+* Improve :cents type column rounding to better handle floating point ugliness
+* Much improved test coverage and documentation
+
 == 0.6.1 / 2015-08-24
 
 * Better handling for nil return value in custom format readers
@@ -8,7 +23,7 @@
 * Vastly improved internal and user-facing comments
 * Improved error logging, replaced some exceptions with errors
 
-== 0.5.0 / 2015-02-XX
+== 0.5.0 / 2015-03-19
 
 * Initial revision
 * Support for CSV, XLS and XLSX importing

data/README.rdoc

@@ -7,8 +7,8 @@ Written by Rob Morris @ Irongaze Consulting LLC (http://irongaze.com)
 Simple, reliable tabular data import.
 
 This gem provides a set of classes to support automating import of tabular data from 
-CSV, XLS and XLSX files, or custom formats via a simple block reader.  Provides help 
-in defining columns, auto-detecting column order, pre-parsing data, and error/warning tracking.
+CSV, HTML, XLS and XLSX files, or custom formats via a simple block reader.  Provides help 
+in defining columns, auto-detecting column order, pre-parsing data, and error tracking.
 
 The Roo/Spreadsheet gems do a great job of providing general purpose spreadsheet reading.
 However, using them with unreliable user submitted data requires a lot of error checking,
@@ -21,37 +21,64 @@ This is NOT a general-purpose tool for reading spreadsheets.  If you want access
 cell styling, reading underlying formulas, etc., you will be better served building
 a custom importer based on Roo.  But if you're looking to take an uploaded CSV file,
 validate and coerce values, then write each row to a database, all the while tracking
-any warnings and errors encountered... well, this is the library for you!
+any errors encountered... well, this is the library for you!
 
 IMPORTANT NOTE: this gem is in flux as we work to define the best possible abstraction
-for the task.  Breaking changes will be noted by increases in the second-level version,
+for the task.  Breaking changes will be noted by increases in the minor version,
 ie 0.5.0 and 0.5.1 will be compatible, but 0.6.0 will not (i.e. we follow semantic versioning).
 
 == SAMPLE USAGE
 
-    # Define our importer, with two columns.  The importer will look for a row containing
-    # "name" and "description" (case insensitively) and automatically determine column
+    # Define our importer, with three columns.  The importer will look for a row containing
+    # "name"/"product", "description" and "price" (case insensitively) and automatically determine column
     # order and starting row of the data.
     importer = Importer.build do
-      column :name
-      column :description
+      column :name do
+        # Column order and start row are auto-detected
+        header /(name|product)/i
+      end
+      column :description do
+        # Columns can do custom parsing
+        parse do |raw_val|
+          raw_val.to_s.strip
+        end
+        # And per-row validation
+        validate do |parsed_val|
+          raise "Invalid description" unless parsed_val.length > 5
+        end
+      end
+      column :price do
+        # Built in type conversion handles common cases
+        type :cents
+      end
+      
+      # Need to skip rows?  Use a filter!
+      filter do |row|
+        row[:price] != 0 && row[:name] != 'Sample'
+      end
     end
     
-    # Import the provided file row-by-row if importing succeeds, automatically
+    # Import the provided file row-by-row (if importing succeeds), automatically
     # using the proper library to read CSV data.  This same code would work
     # with XLS or XLSX files with no changes to the code.
-    if importer.import('/tmp/source.csv')
-      importer.process do |row|
-        puts row[:name] + ' = ' + row[:description]
-      end
+    importer.import('/tmp/source.csv') do |row|
+      puts row[:name] + ' = ' + row[:description]
     end    
 
+    # Check for errors and do the right thing:
+    importer.on_error do
+      puts "Error: " + error_summary
+    end
+
 == REQUIREMENTS
 
-Depends on the iron-extensions and iron-dsl gems, and optionally requires the roo gem to support XLS and 
-XLSX file import and parsing.  Without roo, all you get is CSV.
+Depends on the iron-extensions and iron-dsl gems for CSV and custom import formats.
+
+Optionally requires the roo gem to support XLS and XLSX import and parsing.  
+
+Optionally requires the nokogiri gem to support HTML import and parsing.  
 
-Requires RSpec and roo to build/test.
+Requires RSpec, nokogiri and roo to build/test.
 
 == INSTALLATION
 

data/Version.txt

@@ -1 +1 @@
-0.6.1
+0.7.0

data/lib/iron/import/column.rb

@@ -26,7 +26,11 @@ class Importer
   #       # seems like the "same" source value, for example an Excel source file
   #       # will give you a float value for all numeric types, even "integers".
   #       parse do |raw_value|
-  #         raw_value.to_i + 1000
+  #         val = raw_value.to_i + 1000
+  #         # NOTE: we're in a block, so don't do this:
+  #         return val
+  #         # Instead, use implied return:
+  #         val
   #       end
   #      
   #       # You can also add a custom validator to check the value and add
@@ -54,7 +58,6 @@ class Importer
     attr_reader :data
 
     # Configuration
-    dsl_flag :required
     dsl_accessor :header, :position, :type
     dsl_accessor :parse, :validate
    
@@ -85,21 +88,17 @@ class Importer
       str
     end     
 
-    # Create a new column definition, with the owning sheet, the key for the column,
+    # Create a new column definition with the key for the column,
     # and an optional set of options.  The options supported are the same as those supported
     # in block/builder mode.
-    def initialize(sheet, key, options_hash = {})
+    def initialize(importer, key, options_hash = {})
       # Save off our info
       @key = key
-      @sheet = sheet
-      @importer = @sheet.importer
+      @importer = importer
       
       # Return it as a string, by default
       @type = options_hash.delete(:type) { :string }
       
-      # By default, we allow empty values
-      @required = options_hash.delete(:required) { false }
-      
       # Position can be explicitly set
       @position = options_hash.delete(:position)
       
@@ -126,6 +125,19 @@ class Importer
     def reset
       @data = Data.new
     end
+
+    # DEPRECATED - duplicates functionality better provided by #validate, e.g.
+    #
+    #   validate do |val|
+    #     raise 'Missing required value for column foo' if val.nil?
+    #   end
+    def required!
+      Kernel.warn "[DEPRECATION] Importer::Column#required! is deprecated.  Please use #validate instead."
+      col = self.key
+      validate do |val|
+        raise "Missing required value for column :#{col}"
+      end
+    end
     
     # When true, our header definition or index match the passed text or column index.
     def match_header?(text, index)
@@ -151,7 +163,7 @@ class Importer
     
     # Applies any validation to a parsed value
     def validate_value(row, val)
-      return unless @validate
+      return true unless @validate
       begin 
         @validate.call(val)
         true
@@ -178,20 +190,21 @@ class Importer
       'Column ' + @data.pos
     end
     
-    # Extracts the sheet's values for this column and returns them in an array.
+    # Extracts the imported values for this column and returns them in an array.
     # Note that the array indices ARE NOT row indices, as the rows may have been
     # filtered and any header rows have been skipped.
     def to_a
-      @sheet.data.rows.collect {|r| r[@key] }
+      @importer.data.rows.collect {|r| r[@key] }
     end
     
-    # Extracts the sheet's values for this column and returns them in a hash of
+    # Extracts the values for this column and returns them in a hash of
     # row num => value for all non-filtered, non-header rows.
     def to_h
       res = {}
-      @sheet.data.rows.collect {|r| res[r.num] = r[@key] }
+      @importer.data.rows.collect {|r| res[r.num] = r[@key] }
       res
     end
+    def to_hash ; to_h ; end
   
   end
   

data/lib/iron/import/csv_reader.rb

@@ -30,10 +30,10 @@ class Importer
       end
     end
    
-    # Normally, we'd check the key and return the proper data, but for CSV files, 
-    # there's only one "sheet"
-    def load_raw_sheet(key)
-      @raw_rows
+    # Normally, we'd check the scopes and return the proper data, but for CSV files, 
+    # there's only one scope...
+    def load_raw(scopes, &block)
+      block.call(@raw_rows)
     end
     
   end

data/lib/iron/import/custom_reader.rb

@@ -24,19 +24,25 @@ class Importer
       @source = source
     end
     
-    def load_raw_sheet(sheet)
+    def load_raw(scopes, &block)
+      # Default to just running one scope passing nil
+      if scopes.nil? || scopes.empty?
+        scopes = [nil]
+      end
+      
+      # Get the proper reader
       reader = @readers[@mode]
-      res = DslProxy.exec(self, @source, sheet, &reader)
-      if !res.is_a?(Array) || @importer.has_errors?
-        false
-      else
-        res
+      scopes.each do |scope|
+        rows = DslProxy.exec(self, @source, scope, &reader)
+        if rows.is_a?(Array) && !@importer.has_errors?
+          found = block.call(rows)
+          break if found
+        end
       end
       
     rescue Exception => e
       # Catch any exceptions thrown and note them with helpful stacktrace info for debugging custom readers
-      @importer.add_error("Error in custom reader when loading #{sheet}: #{e} @ #{e.backtrace.first}")
-      false
+      add_error("Error in custom reader: #{e} @ #{e.backtrace.first}")
     end
     
   end

data/lib/iron/import/data_reader.rb

@@ -11,6 +11,16 @@ class Importer
     def self.verify_roo!
       if Gem::Specification.find_all_by_name('roo', '~> 1.13.0').empty?
         raise "You are attempting to use the iron-import gem to import an Excel file.  Doing so requires installing the roo gem, version 1.13.0 or later."
+      else
+        require 'roo'
+      end
+    end
+
+    def self.verify_nokogiri!
+      if Gem::Specification.find_all_by_name('nokogiri', '~> 1.6.0').empty?
+        raise "You are attempting to use the iron-import gem to import an HTML file.  Doing so requires installing the nokogiri gem, version 1.6.0 or later."
+      else
+        require 'nokogiri'
       end
     end
 
@@ -42,6 +52,9 @@ class Importer
       when :xlsx
         verify_roo!
         XlsxReader.new(importer)
+      when :html
+        verify_nokogiri!
+        HtmlReader.new(importer)
       else
         nil
       end
@@ -49,9 +62,11 @@ class Importer
     
     # Figure out which format to use for a given path based on file name
     def self.for_path(importer, path)
-      format = path.to_s.extract(/\.(csv|xlsx?)\z/i)
+      format = path.to_s.extract(/\.(csv|html?|xlsx?)\z/i)
       if format
-        format = format.downcase.to_sym
+        format = format.downcase
+        format = 'html' if format == 'htm'
+        format = format.to_sym
         for_format(importer, format)
       else
         nil
@@ -90,6 +105,10 @@ class Importer
       @supports = []
     end
 
+    def supports?(mode)
+      @supports.include?(mode)
+    end
+    
     def supports_stream!
       @supports << :stream
     end
@@ -98,10 +117,6 @@ class Importer
       @supports << :file
     end
     
-    def supports?(mode)
-      @supports.include?(mode)
-    end
-    
     def supports_file?
       supports?(:file)
     end
@@ -114,13 +129,22 @@ class Importer
     # a file path) and attempts to load it.  Returns true if successful, false
     # if not.  If false, there will be one or more errors explaining what went
     # wrong.
-    def load(path_or_stream)
+    #
+    # Passed scopes are interpreted by each derived class as makes sense, but
+    # generally are used to target seaching in multi-block formats such as
+    # Excel spreadsheets (sheet name/index) or HTML documents (css selectors,
+    # xpath selectors).  If scopes is nil, all possible blocks will be checked.
+    #
+    # Each block is read in as raw data from the source, and passed to the
+    # given block as an array of arrays.  If the block returns true, processing
+    # is stopped and no further blocks will be checked.
+    def load(path_or_stream, scopes = nil, &block)
       # Figure out what we've been passed, and handle it
       if self.class.is_stream?(path_or_stream)
         # We have a stream (open file, upload, whatever)
         if supports_stream?
           # Stream loader defined, run it
-          load_sheets(:stream, path_or_stream)
+          load_each(:stream, path_or_stream, scopes, &block)
         else
           # Write to temp file, as some of our readers only read physical files, annoyingly
           file = Tempfile.new(['importer', ".#{format}"])
@@ -128,7 +152,7 @@ class Importer
           begin
             file.write path_or_stream.read
             file.close
-            load_sheets(:file, file.path)
+            load_each(:file, file.path, scopes, &block)
           ensure
             file.close
             file.unlink
@@ -140,18 +164,18 @@ class Importer
         if File.exist?(path_or_stream)
           if supports_file?
             # We're all set, load up the given path
-            load_sheets(:file, path_or_stream)
+            load_each(:file, path_or_stream, scopes, &block)
           else
             # No file handler, so open the file and run the stream processor
             file = File.open(path_or_stream, 'rb')
-            load_sheets(:stream, file)
+            load_each(:stream, file, scopes, &block)
           end
         else
-          @importer.add_error("Unable to locate source file #{path_or_stream}")
+          add_error("Unable to locate source file #{path_or_stream}")
         end
         
       else
-        @importer.add_error("Unable to load data source - not a file path or stream: #{path_or_stream.inspect}")
+        add_error("Unable to load data source - not a file path or stream: #{path_or_stream.inspect}")
       end
       
       # Return our status
@@ -159,20 +183,12 @@ class Importer
     end
     
     # Load up the sheets in the correct mode
-    def load_sheets(mode, source)
+    def load_each(mode, source, scopes, &block)
       # Let our derived classes open the file, etc. as they need
       if init_source(mode, source)
         # Once the source is set, run through each defined sheet, pass it to
         # our sheet loader, and have the sheet parse it out.
-        @importer.sheets.values.each do |sheet|
-          res = load_raw_sheet(sheet)
-          if res === false
-            # D'oh.
-          else
-            # Tell the sheet to parse the data
-            sheet.parse_raw_data(res)
-          end
-        end
+        load_raw(scopes, &block)
       end
     end
     
@@ -185,8 +201,8 @@ class Importer
     # Override this method in derived classes to take the given sheet definition,
     # find that sheet in the input source, and read out the raw (unparsed) rows
     # as an array of arrays.  Return false if the sheet cannot be loaded.
-    def load_raw_sheet(sheet)
-      raise "Unimplemented method #load_raw_sheet in data reader #{self.class.name}"
+    def load_raw(scopes, &block)
+      raise "Unimplemented method #load_raw in data reader #{self.class.name}"
     end
     
     # Provides default value parsing/coersion for all derived data readers.  Attempts to be clever and
@@ -241,7 +257,7 @@ class Importer
         else
           floatval = parse_value(val, :float)
           if floatval
-            (floatval * 100).to_i
+            (floatval * 100).round
           else
             nil
           end
@@ -261,10 +277,6 @@ class Importer
       @importer.add_error(*args)
     end
     
-    def add_warning(*args)
-      @importer.add_warning(*args)
-    end
-    
   end
   
 end

data/lib/iron/import/error.rb

@@ -2,14 +2,11 @@ class Importer
   
   class Error
      
-    attr_reader :sheet, :row, :text
+    attr_reader :row, :text
     
     def initialize(context, text)
-      if context.is_a?(Importer::Sheet)
-        @sheet = context
-      elsif context.is_a?(Importer::Row)
+      if context.is_a?(Importer::Row)
         @row = context
-        @sheet = context.sheet
       end
       @text = text.to_s
     end
@@ -17,9 +14,7 @@ class Importer
     def summary
       summary = ''
       if @row
-        summary += "#{@sheet} #{@row}: "
-      elsif @sheet
-        summary += "#{@sheet}: "
+        summary += "#{@row}: "
       end
       summary + @text
     end
@@ -29,10 +24,9 @@ class Importer
     end
     
     # Returns the level at which this error occurred, one of
-    # :row, :sheet, :importer
+    # :row, :importer
     def level
       return :row if @row
-      return :sheet if @sheet
       return :importer
     end
     
@@ -40,10 +34,6 @@ class Importer
       level == :row
     end
     
-    def sheet_level?
-      level == :sheet
-    end
-    
     def importer_level?
       level == :importer
     end
@@ -54,8 +44,6 @@ class Importer
       case context
       when Row
         return @row == context
-      when Sheet
-        return @sheet == context
       else
         return true
       end  

data/lib/iron/import/excel_reader.rb

@@ -0,0 +1,69 @@
+class Importer
+  
+  # Uses the Roo gem to read in .xls files
+  class ExcelReader < DataReader
+    
+    def initialize(importer, format)
+      super(importer, format)
+      supports_file!
+    end
+    
+    def init_source(mode, source)
+      if mode == :file
+        if @format == :xls
+          @spreadsheet = Roo::Excel.new(source, :file_warning => :ignore)
+          true
+        elsif @format == :xlsx
+          @spreadsheet = Roo::Excelx.new(source, :file_warning => :ignore)
+          true
+        else
+          add_error("Unknown format for Excel file: :#{@format}")
+          false
+        end
+      else
+        add_error("Unsupported #{@format.to_s.upcase} mode: #{mode}")
+        false
+      end
+    rescue Exception => e
+      add_error("Error reading file #{source}: #{e}")
+      false
+    end
+    
+    def load_raw(scopes, &block)
+      @spreadsheet.sheets.each_with_index do |name, index|
+        # See if this sheet's name or index matches the requested sheet definition
+        if include_sheet?(scopes, name, index)
+          # Extract our raw data
+          raw_rows = []
+          @spreadsheet.sheet(name).each_with_index do |row, line|
+            raw_rows << row
+          end
+          # Yield our raw rows for this sheet
+          found = block.call(raw_rows)
+          # If we've found a working sheet, stop
+          return if found
+        end
+      end
+
+    rescue Exception => e
+      # Not sure why we'd get here, but we strive for error-freedom here, yessir.
+      @importer.add_error("Error loading Excel data: #{e}")
+    end
+  
+    # When true, the given sheet name or zero-based index
+    # is a match with our id.
+    def include_sheet?(scopes, name, index)
+      return true if scopes.nil? || scopes.empty?
+      scopes.each do |scope|
+        if scope.is_a?(Fixnum)
+          return true if scope.to_i == index+1
+        else
+          return true if scope.to_s.downcase == name.downcase
+        end
+      end
+      false
+    end
+
+  end
+  
+end

data/lib/iron/import/html_reader.rb

@@ -0,0 +1,78 @@
+class Importer
+  
+  class HtmlReader < DataReader
+    
+    def initialize(importer)
+      super(importer, :html)
+      supports_file!
+      supports_stream!
+      @tables = nil
+    end
+    
+    def init_source(mode, source)
+      if mode == :stream
+        @html = Nokogiri::HTML(source)
+      elsif mode == :file
+        if File.exist?(source)
+          @html = File.open(source) {|f| Nokogiri::HTML(f) }
+        else
+          add_error("File not found: #{source}")
+          return false
+        end
+      else
+        add_error("Unsupported HTML mode: #{mode}")
+        return false
+      end
+      
+      if @html
+        true
+      else
+        add_error("Failed parsing of HTML")
+        false
+      end
+      
+    rescue Exception => e
+      add_error("Error reading HTML source #{source}: #{e}")
+      false
+    end
+    
+    def load_raw(scopes, &block)
+      # Default to searching all tables in the document
+      if scopes.nil? || scopes.empty?
+        scopes = ['table']
+      end
+      
+      # Catch here lets us break out of the nested loop cleanly
+      catch(:found) do
+        # Run each scope, which should be a valid css selector
+        scopes.each do |scope|
+          @html.css(scope).each do |table_node|
+            rows = []
+            table_node.css('tr').each do |row_node|
+              row = []
+              row_node.children.each do |cell_node|
+                if ['th', 'td'].include?(cell_node.name)
+                  row << cell_node.text.strip
+                  # Handle col-span values appropriately
+                  span_count = cell_node.attr('colspan')
+                  (span_count.to_i - 1).times do 
+                    row << nil
+                  end
+                end
+              end
+              rows << row
+            end
+            found = block.call(rows)
+            throw(:found, true) if found
+          end
+        end
+      end
+
+    rescue Exception => e
+      # Not sure why we'd get here, but we strive for error-freedom here, yessir.
+      add_error("Error loading tables #{scopes.list_join(', ')}: #{e}")
+    end
+  
+  end
+  
+end