iron-import 0.8.1 → 0.8.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: af5ddfc35604cb25d28377798c3ea609ca7e575e
-  data.tar.gz: 76a6ed3cc099d49e6a5fa888b3a37dff26e2bee8
+  metadata.gz: ce84c9899f23abe7fd1f3ed06dc3f50e8986a585
+  data.tar.gz: e1cd18c73aa8663bd0b98305dd18b0ddaee9f75e
 SHA512:
-  metadata.gz: d8d0f3a7cd1a6a2503df394b94dd74630d1226e4c1b0ed43fe9d76d0f264df358641ce7be2d51f30b7c0d870118af109f7e183de37e7a8d3a0c4d3aea8cd68b7
-  data.tar.gz: 3d6d629a0c09ce84b526025d945b55d0f4c6b8e250b3053b43ddb5fdba91e62e7c13a623f66c65a2e39700dd632464471309a6c3357b04c5e7e45c4b757a69ba
+  metadata.gz: ca04f44c1434825045fb27b1d2aa0a2b16b70174dc43bf5f2aa89212331dc06e06ef41d1e4f0070971325f9014d0b6ed4e17a20cf47cb137c2312334255690ee
+  data.tar.gz: 824b6842a4a2c60c1d1ee8737b490fe96e8029e3f8042fa548b85927d8735d2c25c551b73fbc769e407294cdbcc663ec888477746833cb8296c9ffa5ffcaf928

data/History.txt

@@ -1,3 +1,9 @@
+== 0.8.2 / 2017-08-01
+* Add Importer#on_success to enable conditional processing once an import has been successfully completed
+* Pre-parse values with Column#type set when using Column#parse (instead of ignoring it)
+* Make importer scope available in Importer#process and the block form of Importer#import
+* Add backtrace info to error logging for exceptions to help during debugging
+
 == 0.8.1 / 2017-07-18
 * Do not include optional headers in #missing_headers
 * Improve string parsing to strip trailing '.0' from incoming float values

data/Version.txt

@@ -1 +1 @@
-0.8.1
+0.8.2

data/lib/iron/import/column.rb

@@ -29,7 +29,9 @@ class Importer
   #       # that different source types may give you different raw values for what
   #       # seems like the "same" source value, for example an Excel source file
   #       # will give you a float value for all numeric types, even "integers", while
-  #       # CSV and HTML values are always strings.
+  #       # CSV and HTML values are always strings.  By default, will take the raw
+  #       # value of the row, but if used with #type, you can have the pre-processing
+  #       # of the type as your input.
   #       parse do |raw_value|
   #         val = raw_value.to_i + 1000
   #         # NOTE: we're in a block, so don't do this:
@@ -125,7 +127,7 @@ class Importer
       @virtual = options_hash.delete(:virtual) { false }
       
       # Return it as a string, by default
-      @type = options_hash.delete(:type) { :string }
+      @type = options_hash.delete(:type)
       
       # Position can be explicitly set
       @position = options_hash.delete(:position)
@@ -184,6 +186,24 @@ class Importer
       end
     end
     
+    # Override normal dsl_accessor behavior to return our default type
+    # which will be :raw if a #parse handler has been set, else :string
+    alias_method :internal_type, :type
+    def type(*args)
+      if args.count > 0
+        internal_type(*args)
+      else
+        if @type
+          # Explicitly set type
+          @type
+        else
+          # Our default is generally :string, but if we have a parser,
+          # default to the :raw value
+          parses? ? :raw : :string
+        end
+      end
+    end
+    
     # Applies any custom parser defined to process the given value, capturing
     # errors as needed
     def parse_value(row, raw_val)

data/lib/iron/import/custom_reader.rb

@@ -42,7 +42,7 @@ class Importer
       
     rescue Exception => e
       # Catch any exceptions thrown and note them with helpful stacktrace info for debugging custom readers
-      add_error("Error in custom reader: #{e} @ #{e.backtrace.first}")
+      add_exception(e)
     end
     
   end

data/lib/iron/import/data_reader.rb

@@ -219,10 +219,16 @@ class Importer
       return nil if val.nil? || val.to_s.strip == ''
       
       case type
+      when :raw then
+        val
+        
       when :string then
         if val.is_a?(Float)
-          val.to_s.strip.gsub(/\.0+$/, '')
+          # Sometimes float values come in for "integer" columns from Excel,
+          # so if the user asks for a string, strip off that ".0" if present
+          val.to_s.gsub(/\.0+$/, '')
         else
+          # Strip whitespace and we're good to go
           val.to_s.strip
         end
         
@@ -248,7 +254,7 @@ class Importer
         if val.class < Numeric
           val.to_f
         else 
-          # Convert to string, strip off trailing decimal zeros
+          # Clean up then verify it matches a valid float format & convert
           val = val.to_s.strip
           if val.match(/\A-?[0-9]+(?:\.[0-9]+)?\z/)
             val.to_f
@@ -287,6 +293,10 @@ class Importer
       @importer.add_error(*args)
     end
     
+    def add_exception(*args)
+      @importer.add_exception(*args)
+    end
+    
   end
   
 end

data/lib/iron/import/error.rb

@@ -21,7 +21,7 @@ class Importer
         yield
       rescue RuntimeError => e
         # Old-style way of registering errors was to just raise 'foo'
-        importer.add_error(e.to_s)
+        importer.add_exception(e)
       end
       had_error = @error_occurred
       

data/lib/iron/import/excel_reader.rb

@@ -25,7 +25,7 @@ class Importer
         false
       end
     rescue Exception => e
-      add_error("Error reading #{source}: #{e}")
+      add_exception(e)
       false
     end
     
@@ -47,7 +47,7 @@ class Importer
 
     rescue Exception => e
       # Not sure why we'd get here, but we strive for error-freedom here, yessir.
-      @importer.add_error("Error loading Excel data: #{e}")
+      add_exception(e)
     end
   
     # When true, the given sheet name or zero-based index

data/lib/iron/import/html_reader.rb

@@ -27,7 +27,7 @@ class Importer
       end
       
     rescue Exception => e
-      add_error("Error reading HTML source #{source}: #{e}")
+      add_exception(e)
       false
     end
     
@@ -65,7 +65,7 @@ class Importer
 
     rescue Exception => e
       # Not sure why we'd get here, but we strive for error-freedom here, yessir.
-      add_error("Error loading tables #{scopes.list_join(', ')}: #{e}")
+      add_exception(e)
     end
   
   end

data/lib/iron/import/importer.rb

@@ -76,6 +76,10 @@
 #   end.on_error do
 #     # If we have any errors, do something
 #     raise error_summary
+#
+#   end.on_success do
+#     # No errors, do this
+#     puts "Imported successfully!"
 #   end
 #
 class Importer 
@@ -418,12 +422,12 @@ class Importer
   # will not be present.  If you want to register an error, simply 
   # raise "some text" or call #add_error and it will be added to the importer's 
   # error list for display to the user, logging, or whatever.
-  def process
+  def process(&block)
     @data.rows.each do |row|
       begin
-        yield row
+        DslProxy.exec(self, row, &block)
       rescue Exception => e
-        add_error(e.to_s, :row => row)
+        add_exception(e, :row => row)
       end
     end
   end
@@ -448,6 +452,24 @@ class Importer
     self
   end
   
+  # Call with a block to process any post-import tasks.  Block will only execute
+  # if an import has been run with no errors.
+  #
+  # Your block can access the #rows to do whatever
+  # logging, reporting etc. is desired.
+  def on_success(&block)
+    raise 'Invalid block passed to Importer#on_succes: block may accept 0 or 1arguments' if block.arity > 1
+    
+    if @data.rows.any? && !has_errors?
+      case block.arity
+      when 0 then DslProxy.exec(self, &block)
+      when 1 then DslProxy.exec(self, @data.rows, &block)
+      end
+    end
+    
+    self
+  end
+  
   # Call with a block accepting an array of Column objects and returning
   # true if the columns in the array should constitute a valid header row.  Intended
   # for use with optional columns to define multiple supported column sets, or
@@ -557,12 +579,11 @@ class Importer
       if col.present? && !col.virtual?
         index = col.data.index
         raw_val = raw_data[index]
+        # Otherwise use our standard parser
+        val = @reader.parse_value(raw_val, col.type)
         if col.parses?
           # Use custom parser if this row has one
-          val = col.parse_value(row, raw_val)
-        else
-          # Otherwise use our standard parser
-          val = @reader.parse_value(raw_val, col.type)
+          val = col.parse_value(row, val)
         end
         values[col.key] = val
       end
@@ -631,6 +652,12 @@ class Importer
     @data.errors << Error.new(msg, context)
   end
   
+  def add_exception(ex, context = {})
+    line = ex.backtrace.first.extract(/[a-z0-9_\-\.]+\:[0-9]+\:/i)
+    msg = [line, ex.to_s].list_join(' ')
+    @data.errors << Error.new(msg, context)
+  end
+  
   # Returns a human-readable summary of the errors present on the importer, or
   # nil if no errors are present
   def error_summary
@@ -647,10 +674,11 @@ class Importer
     
     # Build summary & return
     list.values.collect do |errs|
-      summary = errs.first.summary
+      err = errs.first
       if errs.count == 1
-        summary
+        err.summary
       else
+        summary = [err.column.to_s, err.text].list_join(': ')
         errs.count.to_s + ' x ' + summary
       end
     end.list_join(', ')

data/spec/importer/importer_spec.rb

@@ -23,21 +23,6 @@ describe Importer do
     importer.scopes.should == { :xls => [1, 'Sheet 2'], :html => ['table.funny'] }
   end
   
-  it 'should calculate virtual columns' do
-    importer = Importer.build do
-      column :num, :type => :int
-      virtual_column :summary do
-        calculate do |row|
-          "Value = #{row[:num]}"
-        end
-      end
-    end
-    
-    importer.import_string("num\n1\n2")
-    importer.error_summary.should be_nil
-    importer.column(:summary).to_a.should == ['Value = 1', 'Value = 2']
-  end
-  
   it 'should find headers automatically' do
     # Define a few sample columns
     importer = Importer.new
@@ -88,6 +73,37 @@ describe Importer do
     importer.find_header(rows).should be_true
     importer.missing_headers.should be_empty
   end
+
+  it 'should calculate virtual columns' do
+    importer = Importer.build do
+      column :num, :type => :int
+      virtual_column :summary do
+        calculate do |row|
+          "Value = #{row[:num]}"
+        end
+      end
+    end
+    
+    importer.import_string("num\n1\n2")
+    importer.error_summary.should be_nil
+    importer.column(:summary).to_a.should == ['Value = 1', 'Value = 2']
+  end
+  
+  it 'should honor type before applying custom parsers' do
+    importer = Importer.new
+    importer.column(:alpha) do
+      parse do |raw|
+        raw
+      end
+    end
+
+    importer.import_string("alpha\n1.0\n1.5")
+    importer.to_a.should == [{:alpha => '1.0'}, {:alpha => '1.5'}]
+
+    importer.column(:alpha).type :float
+    importer.import_string("alpha\n1.0\n1.5")
+    importer.to_a.should == [{:alpha => 1.0}, {:alpha => 1.5}]
+  end
   
   it 'should support row-based validation' do
     importer = Importer.build do
@@ -153,6 +169,25 @@ describe Importer do
     was_run.should be_true
   end
   
+  it 'should run conditional code when successful' do
+    importer = Importer.build do
+      column :foo
+    end
+
+    was_run = false
+    importer.on_success do
+      was_run = true
+    end
+    was_run.should be_false
+
+    importer.import_string("foo\n1")
+    importer.has_errors?.should be_false
+    importer.on_success do
+      was_run = true
+    end
+    was_run.should be_true
+  end
+  
   it 'should import a test csv file' do
     importer = Importer.build do
       column :number

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: iron-import
 version: !ruby/object:Gem::Version
-  version: 0.8.1
+  version: 0.8.2
 platform: ruby
 authors:
 - Rob Morris
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-07-18 00:00:00.000000000 Z
+date: 2017-08-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: iron-extensions