abstract_importer 1.1.0 → 1.2.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 390699648f23aa027bf392f0f9095bf4b6801e3b
-  data.tar.gz: 5f1d0ccb8c4db21649913fa3914b7f6a4f0ba22c
+  metadata.gz: df7b7a0811459f30502cf09d03741312919b9f54
+  data.tar.gz: 116840924377d0562121169b715b9fd64f49fde1
 SHA512:
-  metadata.gz: 1081c7623c65fcbd0748385d6e30f139dcf0d33703ace0c172e9b3e2d9a07aac3cabff0de5f600dc40dc0de7d28af5de2d0865960252888bb5b101711c8ed252
-  data.tar.gz: b59ab4abc7f8c9634e00c24840a01dfe0974f5b7f57e8f34b57129ad10e8d7f0c226972c3323b67107b3aeec6741b2f9e7aaf072ccc8515dbae308c59ca8072b
+  metadata.gz: a9c6b45e74533ba5a1ef07432d677834057649b66656c07c61594ea34e91bb449e5346167fb69cad7d7c464a1c48899e420f8f73a051a22b731afeecfab1db49
+  data.tar.gz: b64e4b346bbc2f566c5686b95edde07f3f49f86be9a971916ac870a3d583038ef31da07cad2ed4e7d58f9f976c6f4de92343dfd93c7449841523cafd1488176d

data/README.md

@@ -1,7 +1,6 @@
 # AbstractImporter
 
 [![Build Status](https://travis-ci.org/concordia-publishing-house/abstract_importer.png?branch=master)](https://travis-ci.org/concordia-publishing-house/abstract_importer)
-
 [![Code Climate](https://codeclimate.com/github/concordia-publishing-house/abstract_importer.png)](https://codeclimate.com/github/concordia-publishing-house/abstract_importer)
 
 AbstractImporter provides services for importing complex data from an arbitrary data source. It:
@@ -54,7 +53,7 @@ end
 `MyImporter`'s initializer takes two arguments: `parent` and `data_source`:
 
  * `parent` is any object that will respond to the names of your collections with an `ActiveRecord::Relation`.
- * `data_source` is any object that will respond to the names of your collections by yielding a hash of attributes once for every record you should import.
+ * `data_source` is any object that will respond to the names of your collections with an `Enumerator`.
 
 Here are reasonable classes for `parent` and `data_source`:
 
@@ -70,15 +69,19 @@ end
 # data source
 class Database
   def students
-    yield id: 457, name: "Ron"
-    yield id: 458, name: "Ginny"
-    yield id: 459, name: "Fred"
-    yield id: 460, name: "George"
+    Enumerator.new do |e|
+      e.yield id: 457, name: "Ron"
+      e.yield id: 458, name: "Ginny"
+      e.yield id: 459, name: "Fred"
+      e.yield id: 460, name: "George"
+    end
   end
 
   def parents
-    yield id: 88, name: "Arthur"
-    yield id: 89, name: "Molly"
+    Enumerator.new do |e|
+      e.yield id: 88, name: "Arthur"
+      e.yield id: 89, name: "Molly"
+    end
   end
 end
 ```
@@ -118,7 +121,7 @@ class MyImporter < AbstractImporter::Base
     import.students do |options|
       options.finder :find_student
       options.before_build { |attrs| attrs.merge(name: attrs[:name].capitalize) }
-      options.on_complete :students_completed
+      options.after_all :students_completed
     end
     import.parents
   end
@@ -156,9 +159,13 @@ The complete list of callbacks is below.
 
 `after_create` is called with the original hash of attributes and the newly-saved record right after it is successfully saved.
 
-##### on_complete
+##### before_all
+
+`before_all` is called just before the records in a collection are been processed.
+
+##### after_all
 
-`on_complete` is called when all of the records in a collection have been processed.
+`after_all` is called when all of the records in a collection have been processed.
 
 
 

data/lib/abstract_importer/base.rb

@@ -15,7 +15,11 @@ module AbstractImporter
         yield @import_plan = ImportPlan.new
       end
       
-      attr_reader :import_plan
+      def depends_on(*dependencies)
+        @dependencies = dependencies
+      end
+      
+      attr_reader :import_plan, :dependencies
     end
     
     
@@ -31,11 +35,16 @@ module AbstractImporter
       @id_map       = IdMap.new
       @results      = {}
       @import_plan  = self.class.import_plan.to_h
+      @atomic       = options.fetch(:atomic, false)
       @collections  = []
     end
     
     attr_reader :source, :parent, :reporter, :id_map, :results
     
+    def atomic?
+      @atomic
+    end
+    
     def dry_run?
       @dry_run
     end
@@ -53,7 +62,9 @@ module AbstractImporter
       reporter.finish_setup(ms)
       
       ms = Benchmark.ms do
-        collections.each &method(:import_collection)
+        with_transaction do
+          collections.each &method(:import_collection)
+        end
       end
       
       teardown
@@ -142,8 +153,19 @@ module AbstractImporter
       end
     end
     
+    def dependencies
+      @dependencies ||= Array(self.class.dependencies).map do |name|
+        reflection = parent.class.reflect_on_association(name)
+        model = reflection.klass
+        table_name = model.table_name
+        scope = parent.public_send(name)
+        
+        Collection.new(name, model, table_name, scope, nil)
+      end
+    end
+    
     def prepopulate_id_map!
-      collections.each do |collection|
+      (collections + dependencies).each do |collection|
         query = collection.scope.where("#{collection.table_name}.legacy_id IS NOT NULL")
         map = values_of(query, :id, :legacy_id) \
           .each_with_object({}) { |(id, legacy_id), map| map[legacy_id] = id }
@@ -167,5 +189,15 @@ module AbstractImporter
       reporter.count_notice "#{plural}.#{foreign_key} will be nil: a #{depends_on.to_s.singularize} with the legacy id #{legacy_id} was not mapped."
     end
     
+    
+    
+    def with_transaction(&block)
+      if atomic?
+        ActiveRecord::Base.transaction(requires_new: true, &block)
+      else
+        block.call
+      end
+    end
+    
   end
 end

data/lib/abstract_importer/collection_importer.rb

@@ -30,11 +30,12 @@ module AbstractImporter
       reporter.start_collection(self)
       prepare!
       
+      invoke_callback(:before_all)
       summary.ms = Benchmark.ms do
         each_new_record &method(:process_record)
       end
+      invoke_callback(:after_all)
       
-      invoke_callback(:on_complete)
       reporter.finish_collection(self, summary)
       summary
     end
@@ -54,43 +55,56 @@ module AbstractImporter
         # has foreign keys that refer to another
         next unless association.macro == :belongs_to
         
-        # We don't at this time support polymorphic associations
-        # which would require extending id_map to take the foreign
-        # type fields into account.
-        #
-        # Rails can't return `association.table_name` so easily
-        # because `table_name` comes from `klass` and `klass`
-        # isn't predetermined.
-        next if association.options[:polymorphic]
-        
-        depends_on = association.table_name.to_sym
-        foreign_key = association.foreign_key.to_sym
-        
         # We support skipping some mappings entirely. I believe
         # this is largely to cut down on verbosity in the log
         # files and should be refactored to another place in time.
+        foreign_key = association.foreign_key.to_sym
         next unless remap_foreign_key?(name, foreign_key)
         
-        mappings << Proc.new do |hash|
-          if hash.key?(foreign_key)
-            hash[foreign_key] = map_foreign_key(hash[foreign_key], name, foreign_key, depends_on)
-          else
-            reporter.count_notice "#{name}.#{foreign_key} will not be mapped because it is not used"
-          end
+        if association.options[:polymorphic]
+          mappings << prepare_polymorphic_mapping!(association)
+        else
+          mappings << prepare_mapping!(association)
         end
       end
       mappings
     end
     
+    def prepare_mapping!(association)
+      depends_on = association.table_name.to_sym
+      foreign_key = association.foreign_key.to_sym
+      
+      Proc.new do |attrs|
+        if attrs.key?(foreign_key)
+          attrs[foreign_key] = map_foreign_key(attrs[foreign_key], name, foreign_key, depends_on)
+        else
+          reporter.count_notice "#{name}.#{foreign_key} will not be mapped because it is not used"
+        end
+      end
+    end
+    
+    def prepare_polymorphic_mapping!(association)
+      foreign_key = association.foreign_key.to_sym
+      foreign_type = association.foreign_key.gsub(/_id$/, "_type").to_sym
+      
+      Proc.new do |attrs|
+        if attrs.key?(foreign_key) && attrs.key?(foreign_type)
+          foreign_model = attrs[foreign_type]
+          depends_on = foreign_model.tableize.to_sym if foreign_model
+          attrs[foreign_key] = depends_on && map_foreign_key(attrs[foreign_key], name, foreign_key, depends_on)
+        else
+          reporter.count_notice "#{name}.#{foreign_key} will not be mapped because it is not used"
+        end
+      end
+    end
+    
     
     
     
     
     def each_new_record
-      source.public_send(name) do |hash_or_hashes|
-        Array.wrap(hash_or_hashes).each do |hash|
-          yield hash.dup
-        end
+      source.public_send(name).each do |attrs|
+        yield attrs.dup
       end
     end
     
@@ -114,6 +128,8 @@ module AbstractImporter
       else
         summary.invalid += 1
       end
+    rescue ::AbstractImporter::Skip
+      summary.skipped += 1
     end
     
     
@@ -154,7 +170,7 @@ module AbstractImporter
       # rescue_callback has one shot to fix things
       invoke_callback(:rescue, record) unless record.valid?
       
-      if record.save
+      if record.valid? && record.save
         invoke_callback(:after_create, hash, record)
         id_map << record
         
@@ -162,7 +178,7 @@ module AbstractImporter
         true
       else
         
-        reporter.record_failed(record)
+        reporter.record_failed(record, hash)
         false
       end
     end
@@ -186,4 +202,7 @@ module AbstractImporter
     end
     
   end
+  
+  class Skip < StandardError; end
+  
 end

data/lib/abstract_importer/import_options.rb

@@ -5,7 +5,8 @@ module AbstractImporter
                 :before_build_callback,
                 :before_create_callback,
                 :after_create_callback,
-                :on_complete_callback
+                :before_all_callback,
+                :after_all_callback
                 
     def finder(sym=nil, &block)
       @finder_callback = sym || block
@@ -27,8 +28,12 @@ module AbstractImporter
       @rescue_callback = sym || block
     end
     
-    def on_complete(sym=nil, &block)
-      @on_complete_callback = sym || block
+    def before_all(sym=nil, &block)
+      @before_all_callback = sym || block
+    end
+    
+    def after_all(sym=nil, &block)
+      @after_all_callback = sym || block
     end
     
   end

data/lib/abstract_importer/reporter.rb

@@ -54,7 +54,7 @@ module AbstractImporter
       io.print "." unless production?
     end
     
-    def record_failed(record)
+    def record_failed(record, hash)
       io.print "×" unless production?
       
       error_messages = invalid_params[record.class.name] ||= {}
@@ -111,8 +111,9 @@ module AbstractImporter
         stat "#{summary.already_imported} #{plural} were imported previously"
         stat "#{summary.redundant} #{plural} would create duplicates and will not be imported"
         stat "#{summary.invalid} #{plural} were invalid"
+        stat "#{summary.skipped} #{plural} were skipped"
         stat "#{summary.created} #{plural} were imported"
-        stat "#{distance_of_time(summary.ms)} elapsed (#{(summary.ms / summary.total).to_i}ms each)"
+        stat "#{distance_of_time(summary.ms)} elapsed (#{summary.average_ms.to_i}ms each)"
       else
         stat "#{distance_of_time(summary.ms)} elapsed"
       end

data/lib/abstract_importer/summary.rb

@@ -1,8 +1,13 @@
 module AbstractImporter
-  class Summary < Struct.new(:total, :redundant, :created, :already_imported, :invalid, :ms)
+  class Summary < Struct.new(:total, :redundant, :created, :already_imported, :invalid, :ms, :skipped)
     
     def initialize
-      super(0,0,0,0,0,0)
+      super(0,0,0,0,0,0,0)
+    end
+    
+    def average_ms
+      return nil if total == 0
+      ms / total
     end
     
   end

data/lib/abstract_importer/version.rb

@@ -1,3 +1,3 @@
 module AbstractImporter
-  VERSION = "1.1.0"
+  VERSION = "1.2.0.rc1"
 end

data/test/callback_test.rb

@@ -18,6 +18,20 @@ class CallbackTest < ActiveSupport::TestCase
       import!
       assert_equal ["Harry", "Ron", "Hermione"], account.students.pluck(:name)
     end
+    
+    should "allow you to skip certain records" do
+      plan do |import|
+        import.students do |options|
+          options.before_build do |attrs|
+            raise AbstractImporter::Skip if attrs[:name] == "Harry Potter"
+          end
+        end
+      end
+      
+      import!
+      assert_equal ["Ron Weasley", "Hermione Granger"], account.students.pluck(:name)
+      assert_equal 1, results[:students].skipped
+    end
   end
   
   
@@ -75,11 +89,28 @@ class CallbackTest < ActiveSupport::TestCase
   
   
   
-  context "on_complete" do
+  context "before_all" do
+    setup do
+      plan do |import|
+        import.students do |options|
+          options.before_all :callback
+        end
+      end
+    end
+    
+    should "should be invoked before the collection has been imported" do
+      mock(importer).callback.once
+      import!
+    end
+  end
+  
+  
+  
+  context "after_all" do
     setup do
       plan do |import|
         import.students do |options|
-          options.on_complete :callback
+          options.after_all :callback
         end
       end
     end

data/test/importer_test.rb

@@ -45,7 +45,44 @@ class ImporterTest < ActiveSupport::TestCase
       assert_equal ["James Potter", "Lily Potter"], harry.parents.pluck(:name)
     end
     
-    should "preserve mappings event when a record was previously imported" do
+    should "preserve mappings even when a record was previously imported" do
+      harry = account.students.create!(name: "Harry Potter", legacy_id: 456)
+      import!
+      assert_equal ["James Potter", "Lily Potter"], harry.parents.pluck(:name)
+    end
+    
+    context "when {atomic: true}" do
+      should "rollback the whole import if an part fails" do
+        mock(importer).atomic? { true }
+        mock.instance_of(Parent).save { raise "hell" }
+        import! rescue
+        assert_equal 0, account.parents.count, "No parents should have been imported with the exception"
+        assert_equal 0, account.students.count, "Expected students to have been rolled back"
+      end
+    end
+    
+    context "when {atomic: false}" do
+      should "not rollback the whole import if an part fails" do
+        mock(importer).atomic? { false }
+        mock.instance_of(Parent).save { raise "hell" }
+        import! rescue
+        assert_equal 0, account.parents.count, "No parents should have been imported with the exception"
+        assert_equal 3, account.students.count, "Expected students not to have been rolled back"
+      end
+    end
+  end
+  
+  
+  
+  context "with a dependency" do
+    setup do
+      depends_on :students
+      plan do |import|
+        import.parents
+      end
+    end
+    
+    should "preserve mappings when a dependency was imported by another importer" do
       harry = account.students.create!(name: "Harry Potter", legacy_id: 456)
       import!
       assert_equal ["James Potter", "Lily Potter"], harry.parents.pluck(:name)
@@ -103,4 +140,23 @@ class ImporterTest < ActiveSupport::TestCase
   
   
   
+  context "with polymorphic associations" do
+    setup do
+      plan do |import|
+        import.cats
+        import.owls
+        import.students
+      end
+    end
+    
+    should "preserve mappings" do
+      import!
+      assert_equal 2, account.students.map(&:pet).compact.count, "Expected two students to still be linked to their pets upon import"
+      assert_kind_of Owl, account.students.find_by_name("Harry Potter").pet, "Expected Harry's pet to be an Owl"
+      assert_kind_of Cat, account.students.find_by_name("Hermione Granger").pet, "Expected Harry's pet to be a Cat"
+    end
+  end
+  
+  
+  
 end

data/test/support/mock_data_source.rb

@@ -1,32 +1,56 @@
 class MockDataSource
   
+  
   def students
-    yield id: 456, name: "Harry Potter"
-    yield id: 457, name: "Ron Weasley"
-    yield id: 458, name: "Hermione Granger"
+    Enumerator.new do |e|
+      e.yield id: 456, name: "Harry Potter", pet_type: "Owl", pet_id: 901
+      e.yield id: 457, name: "Ron Weasley", pet_type: nil, pet_id: nil
+      e.yield id: 458, name: "Hermione Granger", pet_type: "Cat", pet_id: 901
+    end
   end
   
   def parents
-    yield id: 88, name: "James Potter", student_id: 456
-    yield id: 89, name: "Lily Potter", student_id: 456
+    Enumerator.new do |e|
+      e.yield id: 88, name: "James Potter", student_id: 456
+      e.yield id: 89, name: "Lily Potter", student_id: 456
+    end
   end
   
   def locations
-    yield id: 5, slug: "godric's-hollow" # <-- invalid
-    yield id: 6, slug: "azkaban"
+    Enumerator.new do |e|
+      e.yield id: 5, slug: "godric's-hollow" # <-- invalid
+      e.yield id: 6, slug: "azkaban"
+    end
   end
   
   def subjects
-    yield id: 49, name: "Care of Magical Creatures", student_ids: [456]
-    yield id: 50, name: "Advanced Potions", student_ids: [456, 457]
-    yield id: 51, name: "History of Magic", student_ids: [457]
-    yield id: 52, name: "Arithmancy", student_ids: [458]
-    yield id: 53, name: "Study of Ancient Runes", student_ids: [458]
+    Enumerator.new do |e|
+      e.yield id: 49, name: "Care of Magical Creatures", student_ids: [456]
+      e.yield id: 50, name: "Advanced Potions", student_ids: [456, 457]
+      e.yield id: 51, name: "History of Magic", student_ids: [457]
+      e.yield id: 52, name: "Arithmancy", student_ids: [458]
+      e.yield id: 53, name: "Study of Ancient Runes", student_ids: [458]
+    end
   end
   
   def grades
-    yield id: 500, subject_id: 50, student_id: 457, value: "Acceptable"
-    yield id: 501, subject_id: 51, student_id: 457, value: "Troll"
+    Enumerator.new do |e|
+      e.yield id: 500, subject_id: 50, student_id: 457, value: "Acceptable"
+      e.yield id: 501, subject_id: 51, student_id: 457, value: "Troll"
+    end
+  end
+  
+  def cats
+    Enumerator.new do |e|
+      e.yield id: 901, name: "Crookshanks"
+    end
   end
   
+  def owls
+    Enumerator.new do |e|
+      e.yield id: 901, name: "Hedwig"
+    end
+  end
+  
+  
 end

data/test/support/mock_objects.rb

@@ -2,6 +2,7 @@ class Student < ActiveRecord::Base
   has_and_belongs_to_many :subjects
   has_many :grades
   has_many :parents
+  belongs_to :pet, polymorphic: true
   
   def report_card
     subjects.map do |subject|
@@ -34,4 +35,14 @@ class Account < ActiveRecord::Base
   has_many :subjects
   has_many :grades
   has_many :locations
+  has_many :cats
+  has_many :owls
+end
+
+class Cat < ActiveRecord::Base
+  has_one :student, as: :pet
+end
+
+class Owl < ActiveRecord::Base
+  has_one :student, as: :pet
 end

data/test/support/schema.rb

@@ -8,6 +8,8 @@ ActiveRecord::Schema.define(:version => 1) do
     t.integer  "legacy_id"
     t.string   "name"
     t.string   "house"
+    t.string   "pet_type"
+    t.integer  "pet_id"
   end
   
   create_table "parents", :force => true do |t|
@@ -42,4 +44,16 @@ ActiveRecord::Schema.define(:version => 1) do
     t.string   "value"
   end
 
+  create_table "owls", :force => true do |t|
+    t.integer  "account_id"
+    t.integer  "legacy_id"
+    t.string   "name"
+  end
+
+  create_table "cats", :force => true do |t|
+    t.integer  "account_id"
+    t.integer  "legacy_id"
+    t.string   "name"
+  end
+
 end

data/test/test_helper.rb

@@ -50,12 +50,16 @@ class ActiveSupport::TestCase
   
 protected
   
-  attr_reader :account, :results
+  attr_reader :account, :results, :data_source
   
   def plan(&block)
     @klass.import(&block)
   end
   
+  def depends_on(*args)
+    @klass.depends_on(*args)
+  end
+  
   def import!
     @results = importer.perform!
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: abstract_importer
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.2.0.rc1
 platform: ruby
 authors:
 - Bob Lail
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-19 00:00:00.000000000 Z
+date: 2014-04-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -209,12 +209,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - '>'
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.3
+rubygems_version: 2.2.1
 signing_key: 
 specification_version: 4
 summary: Provides services for the mass-import of complex relational data