abstract_importer 1.2.0.rc1 → 1.2.1

data/lib/abstract_importer/reporters/null_reporter.rb

@@ -0,0 +1,19 @@
+module AbstractImporter
+  module Reporters
+    class NullReporter < BaseReporter
+      
+      def start_all(importer)
+      end
+      
+      def finish_all(importer, ms)
+      end
+      
+      def finish_setup(ms)
+      end
+      
+      def start_collection(collection)
+      end
+      
+    end
+  end
+end

data/lib/abstract_importer/reporters/performance_reporter.rb

@@ -0,0 +1,103 @@
+require "objspace"
+
+module AbstractImporter
+  module Reporters
+    class PerformanceReporter < BaseReporter
+      attr_reader :sample_size
+      
+      def initialize(io, options={})
+        super io
+        @sample_size = options.fetch(:sample_size, 50)
+        ObjectSpace.trace_object_allocations_start
+      end
+      
+      
+      def start_collection(collection)
+        super
+        @collection = collection
+        @major_gc_runs = GC.stat[:major_gc_count]
+        @i = 0
+      end
+      
+      def finish_collection(collection, summary)
+        @collection = nil
+        return if @i.zero?
+        find_objects_holding_onto_references_to_a collection.model
+      end
+      
+      def record_created(record)
+        print_stats if @i % sample_size == 0
+        @i += 1
+      end
+      
+      def record_failed(record, hash)
+        print_stats if @i % sample_size == 0
+        @i += 1
+      end
+      
+      
+      def print_stats
+        stats = GC.stat
+        objects = ObjectSpace.count_objects
+        puts "gc[minor]: #{stats[:minor_gc_count]}, gc[major]: #{stats[:major_gc_count]}, objects: #{objects[:TOTAL] - objects[:FREE]}, memsize: #{(ObjectSpace.memsize_of_all / 1048576.0).round(3)}MB, #{collection.name}: #{ObjectSpace.each_object(collection.model).count}"
+      end
+      
+    private
+      attr_reader :collection
+      
+      def find_objects_holding_onto_references_to_a(model)
+        GC.start
+        
+        # After GC.start, all models in this collection should be
+        # garbage-collected unless there is a memory leak. Find one
+        # of the uncollected objects and figure out what is holding
+        # onto a reference to it.
+        example = ObjectSpace.each_object(model).first
+        unless example
+          puts "\e[32mThere are no #{model.name.tableize.gsub("_", " ")} still in memory\e[0m"
+          return
+        end
+        puts "\e[33mThere are #{ObjectSpace.each_object(model).count} #{model.name.tableize.gsub("_", " ")} still in memory\e[0m"
+        
+        example_klass = example.class.name
+        example_id = example.object_id
+        example = nil
+        
+        # Search through all objects to find ones that hold a reference
+        # to the model that hasn't been garbage-collected.
+        print "\e[90m"
+        require "progressbar"
+        pbar = ProgressBar.new("scanning", ObjectSpace.each_object.count)
+        objects_of_holding = []
+        ObjectSpace.each_object do |o|
+          pbar.inc
+          next if ObjectSpace.reachable_objects_from(o).none? { |oo| oo.object_id == example_id }
+          
+          message = "#{o.class.name}"
+          case o
+          when Array
+            message << " (length: #{o.length})"
+          when ActiveRecord::Associations::Association
+            reflection = o.reflection
+            message << " (#{reflection.active_record.name}##{reflection.macro}" <<
+                       " :#{reflection.name})"
+          end
+          message << " [#{ObjectSpace.allocation_sourcefile(o)}" <<
+                     ":#{ObjectSpace.allocation_sourceline(o)}]"
+          
+          objects_of_holding.push(message)
+        end
+        pbar.finish
+        print "\e[0m"
+        
+        if objects_of_holding.none?
+          puts "\e[95mNo objects are holding a reference to the first one\e[0m"
+        else
+          puts "\e[95m#{objects_of_holding.length} objects hold a reference to the first one:",
+               "\e[35m#{objects_of_holding.join("\n")}\e[0m"
+        end
+      end
+      
+    end
+  end
+end

data/lib/abstract_importer/strategies.rb

@@ -0,0 +1,2 @@
+require "abstract_importer/strategies/default_strategy"
+require "abstract_importer/strategies/replace_strategy"

data/lib/abstract_importer/strategies/base.rb

@@ -0,0 +1,30 @@
+module AbstractImporter
+  module Strategies
+    class Base
+      attr_reader :collection
+
+      delegate :summary,
+               :remap_foreign_keys!,
+               :redundant_record?,
+               :invoke_callback,
+               :dry_run?,
+               :id_map,
+               :scope,
+               :reporter,
+               to: :collection
+
+      def initialize(collection)
+        @collection = collection
+      end
+
+      def process_record(hash)
+        raise NotImplementedError
+      end
+
+      def already_imported?(hash)
+        id_map.contains? collection.table_name, hash[:id]
+      end
+
+    end
+  end
+end

data/lib/abstract_importer/strategies/default_strategy.rb

@@ -0,0 +1,83 @@
+require "abstract_importer/strategies/base"
+
+module AbstractImporter
+  module Strategies
+    class DefaultStrategy < Base
+
+
+      def process_record(hash)
+        summary.total += 1
+
+        if already_imported?(hash)
+          summary.already_imported += 1
+          return
+        end
+
+        remap_foreign_keys!(hash)
+
+        if redundant_record?(hash)
+          summary.redundant += 1
+          return
+        end
+
+        if create_record(hash)
+          summary.created += 1
+        else
+          summary.invalid += 1
+        end
+      rescue ::AbstractImporter::Skip
+        summary.skipped += 1
+      end
+
+
+      def create_record(hash)
+        record = build_record(hash)
+
+        return true if dry_run?
+
+        invoke_callback(:before_create, record)
+        invoke_callback(:before_save, record)
+
+        # rescue_callback has one shot to fix things
+        invoke_callback(:rescue, record) unless record.valid?
+
+        if record.valid? && record.save
+          invoke_callback(:after_create, hash, record)
+          invoke_callback(:after_save, hash, record)
+          id_map << record
+
+          reporter.record_created(record)
+          clean_record(record)
+          true
+        else
+
+          reporter.record_failed(record, hash)
+          clean_record(record)
+          false
+        end
+      end
+
+      def build_record(hash)
+        hash = invoke_callback(:before_build, hash) || hash
+
+        legacy_id = hash.delete(:id)
+
+        collection.model.new(hash
+          .merge(legacy_id: legacy_id)
+          .merge(collection.association_attrs))
+      end
+
+      def clean_record(record)
+        # If this record isn't able to be garbage-collected,
+        # then we will print out all of the objects that are
+        # retaining a reference to this one. Ruby's garbage-
+        # collector is smart enough to clean up objects with
+        # circular references; but if we free these now, we
+        # will have fewer results to consider later.
+        record.remove_instance_variable :@association_cache
+        record.remove_instance_variable :@errors
+      end
+
+    end
+  end
+end

data/lib/abstract_importer/strategies/replace_strategy.rb

@@ -0,0 +1,67 @@
+require "abstract_importer/strategies/default_strategy"
+
+module AbstractImporter
+  module Strategies
+    class ReplaceStrategy < DefaultStrategy
+
+
+      def process_record(hash)
+        summary.total += 1
+
+        remap_foreign_keys!(hash)
+
+        if redundant_record?(hash)
+          summary.redundant += 1
+          return
+        end
+
+        if create_or_update_record(hash)
+          summary.created += 1
+        else
+          summary.invalid += 1
+        end
+      rescue ::AbstractImporter::Skip
+        summary.skipped += 1
+      end
+
+
+      def create_or_update_record(hash)
+        if already_imported?(hash)
+          update_record(hash)
+        else
+          create_record(hash)
+        end
+      end
+
+
+      def update_record(hash)
+        hash = invoke_callback(:before_build, hash) || hash
+
+        record = scope.find_by(legacy_id: hash.delete(:id))
+        record.attributes = hash
+
+        return true if dry_run?
+
+        invoke_callback(:before_update, record)
+        invoke_callback(:before_save, record)
+
+        # rescue_callback has one shot to fix things
+        invoke_callback(:rescue, record) unless record.valid?
+
+        if record.valid? && record.save
+          invoke_callback(:after_update, hash, record)
+          invoke_callback(:after_save, hash, record)
+
+          reporter.record_created(record)
+          true
+        else
+
+          reporter.record_failed(record, hash)
+          false
+        end
+      end
+
+
+    end
+  end
+end

data/lib/abstract_importer/version.rb

@@ -1,3 +1,3 @@
 module AbstractImporter
-  VERSION = "1.2.0.rc1"
+  VERSION = "1.2.1"
 end

data/test/importer_test.rb

@@ -21,12 +21,6 @@ class ImporterTest < ActiveSupport::TestCase
       import!
       assert_equal [456, 457, 458], account.students.pluck(:legacy_id)
     end
-    
-    should "not import existing records twice" do
-      account.students.create!(name: "Ron Weasley", legacy_id: 457)
-      import!
-      assert_equal 3, account.students.count
-    end
   end
   
   
@@ -159,4 +153,83 @@ class ImporterTest < ActiveSupport::TestCase
   
   
   
+  context "When we use the default strategy" do
+    setup do
+      plan do |import|
+        import.students
+      end
+    end
+    
+    context "and records already exist" do
+      setup do
+        account.students.create!(name: "Ron Weasley", legacy_id: 457)
+      end
+      
+      should "not import existing records twice" do
+        import!
+        assert_equal 3, account.students.count
+      end
+    end
+  end
+  
+  
+  
+  context "When we specify collections to skip" do
+    setup do
+      plan do |import|
+        import.students
+        import.parents
+      end
+    end
+    
+    context "using :skip" do
+      setup do
+        options.merge!(skip: :parents)
+      end
+      
+      should "not import the named collections" do
+        import!
+        assert_equal 3, account.students.length
+        assert_equal 0, account.parents.length
+      end
+    end
+    
+    context "using :only" do
+      setup do
+        options.merge!(only: [:students])
+      end
+      
+      should "import only the named collections" do
+        import!
+        assert_equal 3, account.students.length
+        assert_equal 0, account.parents.length
+      end
+    end
+  end
+  
+  
+  
+  context "When we use the :replace strategy" do
+    setup do
+      options.merge!(strategy: {students: :replace})
+      plan do |import|
+        import.students
+      end
+    end
+    
+    context "and records already exist" do
+      setup do
+        account.students.create!(name: "Ron Weasley", legacy_id: 457)
+      end
+      
+      should "reimport the existing records" do
+        import!
+        assert_equal "Gryffindor", account.students.find_by_name("Ron Weasley").house,
+          "Expected Ron's record to have been replaced with one that has a house"
+      end
+    end
+  end
+  
+  
+  
 end

data/test/support/mock_data_source.rb

@@ -3,9 +3,9 @@ class MockDataSource
   
   def students
     Enumerator.new do |e|
-      e.yield id: 456, name: "Harry Potter", pet_type: "Owl", pet_id: 901
-      e.yield id: 457, name: "Ron Weasley", pet_type: nil, pet_id: nil
-      e.yield id: 458, name: "Hermione Granger", pet_type: "Cat", pet_id: 901
+      e.yield id: 456, name: "Harry Potter", pet_type: "Owl", pet_id: 901, house: "Gryffindor"
+      e.yield id: 457, name: "Ron Weasley", pet_type: nil, pet_id: nil, house: "Gryffindor"
+      e.yield id: 458, name: "Hermione Granger", pet_type: "Cat", pet_id: 901, house: "Gryffindor"
     end
   end
   

data/test/test_helper.rb

@@ -41,6 +41,7 @@ class ActiveSupport::TestCase
     @data_source = MockDataSource.new
     @klass = Class.new(AbstractImporter::Base)
     @account = Account.create!
+    @options = {}
   end
   
   teardown do
@@ -50,7 +51,7 @@ class ActiveSupport::TestCase
   
 protected
   
-  attr_reader :account, :results, :data_source
+  attr_reader :account, :results, :data_source, :options
   
   def plan(&block)
     @klass.import(&block)
@@ -65,7 +66,7 @@ protected
   end
   
   def importer
-    @importer ||= @klass.new(@account, @data_source, io: $io)
+    @importer ||= @klass.new(@account, @data_source, options.merge(io: $io))
   end
   
 end