iron-import 0.8.2 → 0.8.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ce84c9899f23abe7fd1f3ed06dc3f50e8986a585
-  data.tar.gz: e1cd18c73aa8663bd0b98305dd18b0ddaee9f75e
+  metadata.gz: 2d9d43cccc1da84c981d29256f46c9c225241ba9
+  data.tar.gz: df64d0839daf658b286d0513c404271899303a40
 SHA512:
-  metadata.gz: ca04f44c1434825045fb27b1d2aa0a2b16b70174dc43bf5f2aa89212331dc06e06ef41d1e4f0070971325f9014d0b6ed4e17a20cf47cb137c2312334255690ee
-  data.tar.gz: 824b6842a4a2c60c1d1ee8737b490fe96e8029e3f8042fa548b85927d8735d2c25c551b73fbc769e407294cdbcc663ec888477746833cb8296c9ffa5ffcaf928
+  metadata.gz: 23e16558e41b48c89d45c11e2fc38f1efebdd56dece3866a691c96c98d92371c7cae8e1ad0e874217d81ec649d906b71fac057208cd2c09e49cd47312a83b3ee
+  data.tar.gz: 9601f72f3b8c651b68c60440ee4ef5838c075daaa6a84146600ff876da2eeb2e1c5eb8bd4e9ce4e36aca689a8068afe486b941c0f807d7300e0d21e164ad62b0

data/History.txt

@@ -1,16 +1,26 @@
+== 0.8.3 / 2017-08-22
+
+* Add :bool column type (supports 'yes'/'no', 'Y'/'N', 0/1, 'true'/'false', 'T'/'F')
+* Add error when failing to find valid header due to Importer#validate_columns failure
+* Utilize new Class#inspect_only feature to make #inspect worth a damn on Column & Row during debugging
+* Execute Importer#validate_columns in importer context to allow explicit #add_error calls
+
 == 0.8.2 / 2017-08-01
+
 * Add Importer#on_success to enable conditional processing once an import has been successfully completed
 * Pre-parse values with Column#type set when using Column#parse (instead of ignoring it)
 * Make importer scope available in Importer#process and the block form of Importer#import
 * Add backtrace info to error logging for exceptions to help during debugging
 
 == 0.8.1 / 2017-07-18
+
 * Do not include optional headers in #missing_headers
 * Improve string parsing to strip trailing '.0' from incoming float values
 * Add #to_h to Row for consistency with Column
 * Bugfix for calls to #add_error with invalid calling signature
 
 == 0.8.0 / 2017-06-29
+
 * Breaking Change: change signature of Importer#add_error to support new features
 * Breaking Change: Importer.missing_headers will be [] instead of nil on all headers found
 * Breaking Change: remove deprecated method Column#required!

data/Version.txt

@@ -1 +1 @@
-0.8.2
+0.8.3

data/lib/iron/import/column.rb

@@ -22,7 +22,7 @@ class Importer
   #       header /(price|cost)/i
   #
   #       # Tells the data parser what type of data this column contains, one
-  #       # of :integer, :string, :date, :float, or :cents.  Defaults to :string.
+  #       # of :integer, :string, :date, :float, :bool or :cents.  Defaults to :string.
   #       type :cents
   #
   #       # Instead of a type, you can set an explicit parse block.  Be aware
@@ -30,8 +30,8 @@ class Importer
   #       # seems like the "same" source value, for example an Excel source file
   #       # will give you a float value for all numeric types, even "integers", while
   #       # CSV and HTML values are always strings.  By default, will take the raw
-  #       # value of the row, but if used with #type, you can have the pre-processing
-  #       # of the type as your input.
+  #       # value of the row, but if used with #type, you can have the pre-processed
+  #       # output of that type as your input.
   #       parse do |raw_value|
   #         val = raw_value.to_i + 1000
   #         # NOTE: we're in a block, so don't do this:
@@ -53,9 +53,9 @@ class Importer
   #       virtual!
   #
   #       # When #virtual! is set, gets called to calculate each row's value for this
-  #       # column using the row's parsed values.
+  #       # column using the row's parsed values from other columns.
   #       calculate do |row|
-  #         row[:some_col] + 5
+  #         row[:other_col_key] + 5
   #       end
   #     end
   #   end
@@ -83,6 +83,9 @@ class Importer
     dsl_accessor :header, :position, :type
     dsl_accessor :parse, :validate, :calculate
     dsl_flag :optional, :virtual
+    
+    # Limit our inspect to avoid dumping whole importer
+    inspect_only :key, :type, :optional, :virtual, :position, :index
    
     def self.pos_to_index(pos)
       raise 'Invalid column position: ' + pos.inspect unless pos.is_a?(String) && pos.match(/\A[a-z]{1,3}\z/i)

data/lib/iron/import/data_reader.rb

@@ -283,7 +283,17 @@ class Importer
         # Pull out the date part of the string and convert
         date_str = val.to_s.extract(/[0-9]+[\-\/][0-9]+[\-\/][0-9]+/)
         date_str.to_date rescue nil
-        
+      
+      when :bool then
+        val_str = parse_value(val, :string).to_s.downcase
+        if ['true','yes','y','t','1'].include?(val_str)
+          return true
+        elsif ['false','no','n','f','0'].include?(val_str)
+          return false
+        else
+          nil
+        end
+      
       else
         raise "Unknown column type #{type.inspect} - unimplemented?"
       end

data/lib/iron/import/importer.rb

@@ -109,7 +109,7 @@ class Importer
   dsl_flag :headerless
   # Explicitly sets the row number (1-indexed) where data rows begin,
   # usually left defaulted to nil to automatically start after the header
-  # row.
+  # row, or on the first row if #headerless! is set.
   dsl_accessor :start_row
   # Set to a block/lambda taking a parsed but unvalidated row as a hash,
   # return true to keep, false to skip.
@@ -363,6 +363,7 @@ class Importer
     end
 
     # Read in the data!
+    loaded = false
     @reader.load(path_or_stream, scopes) do |raw_rows|
       # Find our column layout, start of data, etc
       if find_header(raw_rows)
@@ -374,6 +375,7 @@ class Importer
           end
         end
         # We've found a workable sheet/table/whatever, stop looking
+        loaded = true
         true
         
       else
@@ -383,9 +385,14 @@ class Importer
       end
     end
     
-    # If we have any missing headers, note that fact
-    if @missing_headers && @missing_headers.count > 0
-      add_error("Unable to locate required column header for column(s): " + @missing_headers.collect{|c| ":#{c}"}.list_join(', '))
+    # Verify that we found a working set of rows
+    unless loaded
+      # If we have any missing headers, note that fact
+      if @missing_headers && @missing_headers.count > 0
+        add_error("Unable to locate required column header for column(s): " + @missing_headers.collect{|c| ":#{c}"}.list_join(', '))
+      else 
+        add_error("Unable to locate required column headers!")
+      end
     end
     
     # If we're here with no errors, we rule!
@@ -533,15 +540,18 @@ class Importer
           col.data.index = nil
           col.data.header_text = nil
         end
-        
+
         # Have we found them all, or at least a valid sub-set?
         header_found = remaining.empty?
         unless header_found
-          if remaining.all?(&:optional?)
+          if found_columns.any? && remaining.all?(&:optional?)
             if @column_validator
               # Run custom column validator
-              cols = found_columns
-              header_found = @column_validator.call(cols)
+              valid = false
+              had_error = Error.with_context(@importer, nil, nil, nil) do
+                valid = DslProxy.exec(self, found_columns, &@column_validator)
+              end
+              header_found = !had_error && valid
             else
               # No validator... do we have any found columns at all???
               header_found = @columns.any?(&:present?)

data/lib/iron/import/row.rb

@@ -2,8 +2,12 @@ class Importer
   
   class Row
 
+    # Attributes
     attr_reader :line, :values, :errors
     
+    # Limit our inspect to avoid dumping whole importer
+    inspect_only :line, :values
+    
     def initialize(importer, line, value_hash = nil)
       @importer = importer
       @line = line

data/spec/importer/csv_reader_spec.rb

@@ -28,4 +28,75 @@ describe Importer::CsvReader do
     ]
   end
   
+  it 'should fail on WSM sample data' do
+    importer = Importer.build do
+      column :company_name do
+        optional!
+      end
+      virtual_column :company do
+        calculate do |row|
+          if column(:company_name).present?
+            row[:company_name]
+          else
+            [row[:store_code], row[:store_num]].list_join(', ')
+          end
+        end
+      end
+      column :store_code do
+        header /code$/i
+        optional!
+      end
+      column :store_num do
+        optional!
+        header /num(ber)?$/i
+      end
+      virtual_column :store do
+        calculate do |row|
+          Store.find_by_upc(row[:upc])
+        end
+      end
+      column :buyer_name do
+        optional!
+      end
+      column :buyer_email do
+        optional!
+        header /buyer\s*email/i
+        validate do |val|
+          val.match? /^\s*([a-z0-9_\-\+\.]+@[a-z0-9\.\-]+\.[a-z]+)(,\s*[a-z0-9_\-\+\.]+@[a-z0-9\.\-]+\.[a-z]+)*\s*$/i
+        end
+      end
+      column :rep_email do
+        optional!
+        header /^(sales\s*)?rep\s*email/i
+        validate do |val|
+          val.match? /^\s*([a-z0-9_\-\+\.]+@[a-z0-9\.\-]+\.[a-z]+)(,\s*[a-z0-9_\-\+\.]+@[a-z0-9\.\-]+\.[a-z]+)*\s*$/i
+        end
+      end
+      column :regional do
+        optional!
+        type :bool
+      end
+
+      # We need a company name column if none passed in
+      validate_columns do |cols|
+        keys = cols.collect(&:key)
+        has_company = keys.include?(:company_name)
+        has_company && (keys.include?(:store_num) || keys.include?(:store_code))
+      end
+
+      # Only pay attention to rows with a store num or code
+      # filter_rows do |row|
+      #   row[:store_num].present? || row[:store_code].present?
+      # end
+      
+      # Make sure rows are valid
+      validate_rows do |row|
+        add_error("Unable to locate specified company") unless row[:company].present?
+        add_error("Unable to locate specified store") unless row[:store].present?
+      end
+    end
+    importer.import(SpecHelper.sample_path('wsm-data.csv')).should be_false
+    importer.errors.first.to_s.should == "Unable to locate required column headers!"
+  end
+  
 end

data/spec/importer/data_reader_spec.rb

@@ -44,7 +44,8 @@ describe Importer::DataReader do
       '' => nil,
       255 => '255',
       -1.5 => '-1.5',
-      10.0 => '10'
+      10.0 => '10',
+      '10.0' => '10.0'
     }.each_pair do |val, res|
       @reader.parse_value(val, :string).should === res
     end
@@ -60,7 +61,9 @@ describe Importer::DataReader do
       '-95' => -9500,
       52 => 5200,
       1.0 => 100,
-      1.25 => 125
+      1.25 => 125,
+      -2.001 => -200,
+      'bob' => nil
     }.each_pair do |val, res|
       @reader.parse_value(val, :cents).should === res
     end
@@ -79,6 +82,29 @@ describe Importer::DataReader do
     end
   end
   
+  it 'should parse bools' do
+    {
+      'tRue' => true,
+      'yes ' => true,
+      'T' => true,
+      'y' => true,
+      '1' => true,
+      1.0 => true,
+      'FALSE' => false,
+      'no' => false,
+      'F' => false,
+      'n' => false,
+      '0' => false,
+      0 => false,
+      'tim' => nil,
+      nil => nil,
+      '' => nil,
+      'xyz' => nil
+    }.each_pair do |val, res|
+      @reader.parse_value(val, :bool).should === res
+    end
+  end
+  
   it 'should build an instance based on format' do
     Importer::DataReader.for_format(@importer, :csv).should be_a(Importer::CsvReader)
     Importer::DataReader.for_format(@importer, :xls).should be_a(Importer::XlsReader)

data/spec/samples/wsm-data.csv

@@ -0,0 +1,3 @@
+ID,Number,Code,Name,Region,,Buyer Name,Buyer Emails,Sales Rep Emails,,,,,,
+4293,10194,BHL,Bayhill,FL,BHL,,"example1@wholefoods.com, example3@wholefoods.com",,,,,,,
+4294,10096,BIS,Biscayne,FL,BIS,,"example2@wholefoods.com, example4@wholefoods.com",,,,,,,

data/spec/spec_helper.rb

@@ -10,6 +10,11 @@ RSpec.configure do |config|
   config.color = true
   config.add_formatter 'documentation'
   config.backtrace_exclusion_patterns = [/rspec/]
+  # Allow us to use: it '...', :focus do ... end 
+  # rather than needing: it '...', :focus => true do ... end
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+  # If everything is filtered, run everything - used if no :focus element is present
+  config.run_all_when_everything_filtered = true
 end
 
 module SpecHelper

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: iron-import
 version: !ruby/object:Gem::Version
-  version: 0.8.2
+  version: 0.8.3
 platform: ruby
 authors:
 - Rob Morris
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-08-01 00:00:00.000000000 Z
+date: 2017-08-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: iron-extensions
@@ -19,7 +19,7 @@ dependencies:
         version: '1.2'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.2.1
+        version: 1.2.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -29,7 +29,7 @@ dependencies:
         version: '1.2'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.2.1
+        version: 1.2.2
 - !ruby/object:Gem::Dependency
   name: iron-dsl
   requirement: !ruby/object:Gem::Requirement
@@ -133,6 +133,7 @@ files:
 - spec/samples/simple.csv
 - spec/samples/simple.html
 - spec/samples/test-products.xls
+- spec/samples/wsm-data.csv
 - spec/spec_helper.rb
 homepage: http://irongaze.com
 licenses: