iron-import 0.5.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 70c4748d780e9854cbd60622563b74d3b7ce2b5c
+  data.tar.gz: d6503f0f7a08b4c88da5813b3114446baf1fff1a
+SHA512:
+  metadata.gz: 488a0e4b2d8ed83914bb2a6c907358ee584c0849f26bf9e64d6cc4bd8c2296997e4bc580f59b3bff4db6fa699a6abf94f5a85cd31c1585f03f728523025529a3
+  data.tar.gz: 00c6e27cf433423c9c1cc14828c11cd895459b0c12e86aa57ec65b35b049b0b7939dda98edd3359aa4dab8af945b0a17b9ef5b7fc486300edeb8b987b21d65dd

data/.rspec

@@ -0,0 +1 @@
+--require <%= File.join(File.expand_path(File.dirname(__FILE__)), 'spec', 'spec_helper.rb') %>

data/History.txt

@@ -0,0 +1,12 @@
+== 0.5.0 / 2015-02-XX
+
+* Initial revision
+* Support for CSV, XLS and XLSX importing
+* Multiple sheet support
+* Automatic header and start-of-data detection
+* Value coercion to :string, :integer, :float, :date, and :cents
+* Custom parsing of raw cell values
+* Custom validation of cell values
+* Conditional row filtering
+* Error and warning aggregation, by sheet/row as appropriate
+* Automatic stream-to-file conversion where needed by underlying libs

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2015 Irongaze Consulting LLC
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,70 @@
+= GEM: iron-import
+
+Written by Rob Morris @ Irongaze Consulting LLC (http://irongaze.com)
+
+== DESCRIPTION
+
+Simple, reliable tabular data import.
+
+This gem provides a set of classes to support automating import of tabular data from 
+CSV, XLS or XLSX files.  Provides help in defining columns, auto-detecting column 
+order, pre-parsing data, and error/warning tracking.
+
+The Roo/Spreadsheet gems do a great job of providing general purpose spreadsheet reading.
+However, using them with unreliable user submitted data requires a lot of error checking,
+monkeying with data coercion, etc.  At Irongaze, we do a lot of work with growing
+businesses, where Excel files are the lingua franca for all kinds of uses.  This gem 
+attempts to extract years of experience building one-off importers into a simple library 
+for rapid import coding.
+
+This is NOT a general-purpose tool for reading spreadsheets.  If you want access to 
+cell styling, reading underlying formulas, etc., you will be better served building
+a custom importer based on Roo.  But if you're looking to take an uploaded CSV file,
+validate and coerce values, then write each row to a database, all the while tracking
+any warnings and errors encountered... well, this is the library for you!
+
+IMPORTANT NOTE: this gem is in flux as we work to define the best possible abstraction
+for the task.  Breaking changes will be noted by increases in the second-level version,
+ie 0.5.0 and 0.5.1 will be compatible, but 0.6.0 will not.
+
+== SAMPLE USAGE
+
+    # Define our importer, with two columns.  The importer will look for a row containing
+    # "name" and "description" (case insensitively) and automatically determine column
+    # order and starting row of the data.
+    importer = Importer.build do
+      column :name
+      column :description
+    end
+    
+    # Import the provided file row-by-row if importing succeeds, automatically
+    # using the proper library to read CSV data.  This same code would work
+    # with XLS or XLSX files with no changes to the code.
+    if importer.import('/tmp/source.csv')
+      importer.process do |row|
+        puts row[:name] + ' = ' + row[:description]
+      end
+    end    
+
+== REQUIREMENTS
+
+Depends on the iron-extensions and iron-dsl gems, and optionally requires the roo gem to support XLS and 
+XLSX file import and parsing.  Without roo, all you get is CSV.
+
+Requires RSpec and roo to build/test.
+
+== INSTALLATION
+
+To install, simply run:
+
+    sudo gem install iron-import
+    
+RVM users can skip the sudo:
+  
+    gem install iron-import
+
+Then use
+
+    require 'iron/import'
+    
+to require the library code.

data/Version.txt

@@ -0,0 +1 @@
+0.5.0

data/lib/iron/import/column.rb

@@ -0,0 +1,177 @@
+class Importer
+
+  # Columns represent the settings for importing a given column within a Sheet.  They do not
+  # hold data, rather they capture the settings needed for identifying the column in the header,
+  # how to parse and validate each of their cell's data, and so forth.
+  #
+  # Here's the complete list of column configuration options:
+  #
+  #   Importer.build do
+  #     column :key do
+  #       # Set a fixed position - may be a column number or a letter-based
+  #       # column description, ie 'A' == 1.  In most cases, you can leave
+  #       # this defaulted to nil, which will mean "look for the proper header"
+  #       position 'C'
+  #
+  #       # Specify a regex to locate the header for this column, defaults to 
+  #       # finding a string containing the key.
+  #       header /(price|cost)/i
+  #
+  #       # Tells the data parser what type of data this column contains, one
+  #       # of :integer, :string, :date, :float, or :cents.  Defaults to :string.
+  #       type :cents
+  #
+  #       # Instead of a type, you can set an explicit parse block.  Be aware
+  #       # that different source types may give you different raw values for what
+  #       # seems like the "same" source value, for example an Excel source file
+  #       # will give you a float value for all numeric types, even "integers"
+  #       parse do |raw_value|
+  #         raw_value.to_i + 1000
+  #       end
+  #      
+  #       # You can also add a custom validator to check the value and add
+  #       # an error if it's not within a given range, or whatever:
+  #       validate do |parsed_value|
+  #         raise "Out of range" unless (parsed_value > 0 && parsed_value < 5000)
+  #       end
+  #     end
+  #   end
+  #
+  class Column
+
+    # Holds load-time data
+    class Data
+      attr_accessor :index
+      
+      def pos
+        @index ? Column::index_to_pos(@index) : 'Unknown'
+      end
+    end
+
+    # Core info
+    attr_reader :key
+    attr_reader :data
+
+    # Configuration
+    dsl_flag :required
+    dsl_accessor :header, :position, :type
+    dsl_accessor :parse, :validate
+   
+    def self.pos_to_index(pos)
+      raise 'Invalid column position: ' + pos.inspect unless pos.is_a?(String) && pos.match(/\A[a-z]{1,3}\z/i)
+      vals = pos.upcase.bytes.collect {|b| b - 64}
+      total = 0
+      multiplier = 1
+      vals.reverse.each do |val|
+        total += val * multiplier
+        multiplier *= 26
+      end
+      total - 1
+    end
+    
+    def self.index_to_pos(index)
+      val = index.to_i
+      raise 'Invalid column index: ' + index.inspect if (!index.is_a?(Fixnum) || index.to_i < 0)
+      
+      chars = ('A'..'Z').to_a
+      str = ''
+      while index > 25
+        str = chars[index % 26] + str
+        index /= 26
+        index -= 1
+      end
+      str = chars[index] + str
+      str
+    end     
+   
+    def initialize(sheet, key)
+      # Save off our info
+      @key = key
+      @sheet = sheet
+      @importer = @sheet.importer
+      
+      # Return it as a string, by default
+      @type = :string
+      
+      # By default, we allow empty values
+      @required = false
+      
+      # Position can be explicitly set
+      @position = nil
+      
+      # By default, don't parse incoming data, just pass it through
+      @parse = nil
+      
+      # Default matcher, looks for the presence of the column key as text anywhere
+      # in the header string, ignoring case and using underscores as spaces, ie
+      # :order_id => /\A\s*order id\s*\z/i
+      @header = Regexp.new('\A\s*' + key.to_s.gsub('_', ' ') + '\s*\z', Regexp::IGNORECASE)
+      
+      # Reset our state to pre-load status
+      reset
+    end
+    
+    def build(&block)
+      DslProxy.exec(self, &block)
+    end
+    
+    def reset
+      @data = Data.new
+    end
+    
+    # When true, matches either the passed value or the index (if position has been explicitly set)
+    def match_header?(text, index)
+      res = index == self.fixed_index || (@header && !@header.match(text).nil?)
+      # puts "#{@header.inspect} ~ #{text.inspect} => #{res.inspect}"
+      res
+    end
+    
+    # Use any custom parser defined to process the given value, capturing
+    # errors as needed
+    def parse_value(row, val)
+      return val if @parse.nil?
+      begin 
+        @parse.call(val)
+      rescue Exception => e
+        @importer.add_error(row, "Error parsing #{self}: #{e}")
+        nil
+      end
+    end
+    
+    def validate_value(row, val)
+      return unless @validate
+      begin 
+        @validate.call(val)
+        true
+      rescue Exception => e
+        @importer.add_error(row, "Validation error in #{self}: #{e}")
+        false
+      end
+    end
+    
+    def fixed_index
+      return nil unless @position
+      if @position.is_a?(Fixnum)
+        @position - 1
+      elsif @position.is_a?(String)
+        Column.pos_to_index(@position)
+      end
+    end
+    
+    def to_s
+      'Column ' + @data.pos
+    end
+    
+    def to_a
+      @sheet.data.rows.collect {|r| r[@key] }
+    end
+    
+    def to_h
+      res = {}
+      @sheet.data.rows.collect {|r| res[r.num] = r[@key] }
+      res
+    end
+  
+  end
+  
+end

data/lib/iron/import/csv_reader.rb

@@ -0,0 +1,26 @@
+require 'csv'
+
+class Importer
+  
+  class CsvReader < DataReader
+   
+    def initialize(importer)
+      super(importer, :csv)
+    end
+   
+    def load_stream(stream)
+      text = stream.read
+      encoding = @importer.encoding || 'UTF-8'
+      raw_rows = CSV.parse(text, :encoding => "#{encoding}:UTF-8")
+      @importer.default_sheet.parse_raw_data(raw_rows)
+    end
+    
+    def load_file(path)
+      encoding = @importer.encoding || 'UTF-8'
+      raw_rows = CSV.read(path, :encoding => "#{encoding}:UTF-8")
+      @importer.default_sheet.parse_raw_data(raw_rows)
+    end
+    
+  end
+  
+end

data/lib/iron/import/data_reader.rb

@@ -0,0 +1,176 @@
+class Importer
+
+  # Base class for our input reading - dealing with the raw file/stream,
+  # and extracting raw values.  In addition, we provide the base
+  # data coercion/parsing for our derived classes.
+  class DataReader
+
+    # Attributes
+    attr_reader :format
+
+    def self.verify_roo!
+      if Gem::Specification.find_all_by_name('roo', '~> 1.13.0').empty?
+        raise "You are attempting to use the iron-import gem to import an Excel file.  Doing so requires installing the roo gem, version 1.13.0 or later."
+      end
+    end
+
+    def self.for_format(importer, format)
+      case format
+      when :csv
+        CsvReader.new(importer)
+      when :xls
+        verify_roo!
+        XlsReader.new(importer)
+      when :xlsx
+        verify_roo!
+        XlsxReader.new(importer)
+      else
+        nil
+      end
+    end
+    
+    def self.for_path(importer, path)
+      format = path.to_s.extract(/\.(csv|xlsx?)\z/i)
+      if format
+        format = format.downcase.to_sym
+        for_format(importer, format)
+      else
+        nil
+      end
+    end
+    
+    def self.for_stream(importer, stream)
+      path = path_from_stream(stream)
+      for_path(importer, path)
+    end
+    
+    # Try to find the original file name for the given stream,
+    # as in the case where a file is uploaded to Rails and we're dealing with an
+    # ActionDispatch::Http::UploadedFile.
+    def self.path_from_stream(stream)
+      if stream.respond_to?(:original_filename)
+        stream.original_filename
+      elsif stream.respond_to?(:path)
+        stream.path
+      else
+        nil
+      end
+    end
+    
+    def initialize(importer, format)
+      @importer = importer
+      @format = format
+      @multisheet = true
+    end
+
+    def load(path_or_stream)
+      # Figure out what we've been passed, and handle it
+      if path_or_stream.respond_to?(:read)
+        # We have a stream (open file, upload, whatever)
+        if respond_to?(:load_stream)
+          # Stream loader defined, run it
+          load_stream(path_or_stream)
+        else
+          # Write to temp file, as some of our readers only read physical files, annoyingly
+          file = Tempfile.new(['importer', ".#{format}"])
+          file.binmode
+          begin
+            file.write path_or_stream.read
+            file.close
+            load_file(file.path)
+          ensure
+            file.close
+            file.unlink
+          end
+        end
+        
+      elsif path_or_stream.is_a?(String)
+        # Assume it's a path
+        if respond_to?(:load_file)
+          # We're all set, load up the given path
+          load_file(path_or_stream)
+        else
+          # No file handler, so open the file and run the stream processor
+          file = File.open(path_or_stream, 'rb')
+          load_stream(file)
+        end
+        
+      else
+        raise "Unable to load data: #{path_or_stream.inspect}"
+      end
+      
+      # Return our status
+      !@importer.has_errors?
+    end
+    
+    # Provides default value parsing/coersion for all derived data readers.  Attempts to be clever and
+    # handle edge cases like converting '5.00' to 5 when in integer mode, etc.  If you find your inputs aren't
+    # being parsed correctly, add a custom #parse block on your Column definition.
+    def parse_value(val, type)
+      return nil if val.nil? || val.to_s == ''
+      
+      case type
+      when :string then
+        val = val.to_s.strip
+        val.blank? ? nil : val
+        
+      when :integer, :int then 
+        if val.class < Numeric
+          # If numeric, verify that there's no decimal places to worry about
+          if (val.to_f % 1.0 == 0.0)
+            val.to_i
+          else
+            nil
+          end
+        else 
+          # Convert to string, strip off trailing decimal zeros
+          val = val.to_s.strip.gsub(/\.0*$/, '')
+          if val.integer?
+            val.to_i
+          else
+            nil
+          end
+        end
+        
+      when :float then
+        if val.class < Numeric
+          val.to_f
+        else 
+          # Convert to string, strip off trailing decimal zeros
+          val = val.to_s.strip
+          if val.match(/\A-?[0-9]+(?:\.[0-9]+)?\z/)
+            val.to_f
+          else
+            nil
+          end
+        end
+        
+      when :cents then
+        if val.is_a?(String)
+          val = val.gsub(/\s*\$\s*/, '')
+        end
+        intval = parse_value(val, :integer)
+        if !val.is_a?(Float) && intval
+          intval * 100
+        else
+          floatval = parse_value(val, :float)
+          if floatval
+            (floatval * 100).to_i
+          else
+            nil
+          end
+        end
+        
+      when :date then
+        # Pull out the date part of the string and convert
+        date_str = val.to_s.extract(/[0-9]+[\-\/][0-9]+[\-\/][0-9]+/)
+        date_str.to_date rescue nil
+        
+      else
+        raise "Unknown column type #{type.inspect} - unimplemented?"
+      end
+    end
+    
+  end
+  
+end

data/lib/iron/import/error.rb

@@ -0,0 +1,66 @@
+class Importer
+  
+  class Error
+     
+    attr_reader :sheet, :row, :text
+    
+    def initialize(context, text)
+      if context.is_a?(Importer::Sheet)
+        @sheet = context
+      elsif context.is_a?(Importer::Row)
+        @row = context
+        @sheet = context.sheet
+      end
+      @text = text.to_s
+    end
+    
+    def summary
+      summary = ''
+      if @row
+        summary += "#{@sheet} #{@row}: "
+      elsif @sheet
+        summary += "#{@sheet}: "
+      end
+      summary + @text
+    end
+
+    def to_s
+      summary
+    end
+    
+    # Returns the level at which this error occurred, one of
+    # :row, :sheet, :importer
+    def level
+      return :row if @row
+      return :sheet if @sheet
+      return :importer
+    end
+    
+    def row_level?
+      level == :row
+    end
+    
+    def sheet_level?
+      level == :sheet
+    end
+    
+    def importer_level?
+      level == :importer
+    end
+
+    # Returns true if this error is for the given context, where
+    # context can be a Row, Sheet or Importer instance.
+    def for_context?(context)
+      case context
+      when Row
+        return @row == context
+      when Sheet
+        return @sheet == context
+      else
+        return true
+      end  
+    end
+    
+  end
+  
+end