abstract_importer 1.0.0

data/lib/abstract_importer/import_options.rb

@@ -0,0 +1,35 @@
+module AbstractImporter
+  class ImportOptions
+    attr_reader :finder_callback,
+                :rescue_callback,
+                :before_build_callback,
+                :before_create_callback,
+                :after_create_callback,
+                :on_complete_callback
+                
+    def finder(sym=nil, &block)
+      @finder_callback = sym || block
+    end
+    
+    def before_build(sym=nil, &block)
+      @before_build_callback = sym || block
+    end
+    
+    def before_create(sym=nil, &block)
+      @before_create_callback = sym || block
+    end
+    
+    def after_create(sym=nil, &block)
+      @after_create_callback = sym || block
+    end
+    
+    def rescue(sym=nil, &block)
+      @rescue_callback = sym || block
+    end
+    
+    def on_complete(sym=nil, &block)
+      @on_complete_callback = sym || block
+    end
+    
+  end
+end

data/lib/abstract_importer/import_plan.rb

@@ -0,0 +1,17 @@
+module AbstractImporter
+  class ImportPlan
+    
+    def initialize
+      @plan = {} # <-- requires Ruby 1.9's ordered hash
+    end
+    
+    def to_h
+      @plan.dup
+    end
+    
+    def method_missing(plural, &block)
+      @plan[plural] = block
+    end
+    
+  end
+end

data/lib/abstract_importer/reporter.rb

@@ -0,0 +1,146 @@
+module AbstractImporter
+  class Reporter
+    
+    def initialize(io, production)
+      @io = io
+      @notices = {}
+      @errors  = {}
+      @production = production
+      @invalid_params = {}
+    end
+    
+    attr_reader :io, :invalid_params
+    
+    
+    
+    def production?
+      @production
+    end
+    
+    
+    
+    def start_all(importer)
+      status "Importing #{importer.describe_source} to #{importer.describe_destination}\n"
+    end
+    
+    def finish_all(importer, ms)
+      print_invalid_params
+      status "\n\nFinished in #{distance_of_time(ms)}"
+    end
+    
+    
+    
+    def finish_setup(ms)
+      status "Setup took #{distance_of_time(ms)}\n"
+    end
+    
+    
+    
+    def start_collection(collection)
+      status "\n#{("="*80)}\nImporting #{collection.name}\n#{("="*80)}\n"
+      @notices = {}
+      @errors  = {}
+    end
+    
+    def finish_collection(collection, summary)
+      ms = summary[5]
+      elapsed = distance_of_time(ms)
+      stat "\n  #{summary[0]} #{collection.name} were found"
+      if summary[0] > 0
+        stat     "#{summary[3]} #{collection.name} were imported previously"
+        stat     "#{summary[1]} #{collection.name} would create duplicates and will not be imported"
+        stat     "#{summary[4]} #{collection.name} were invalid"
+        stat     "#{summary[2]} #{collection.name} were imported"
+      end
+      stat     "#{elapsed} elapsed" << (summary[0] > 0 ? " (#{(ms / summary[0]).to_i}ms each)" : "")
+      
+      print_messages(@notices, "Notices")
+      print_messages(@errors,  "Errors")
+    end
+    
+    
+    
+    def record_created(record)
+      io.print "." unless production?
+    end
+    
+    def record_failed(record)
+      io.print "×" unless production?
+      
+      error_messages = invalid_params[record.class.name] ||= {}
+      record.errors.full_messages.each do |error_message|
+        error_messages[error_message] = hash unless error_messages.key?(error_message)
+        count_error(error_message)
+      end
+    end
+    
+    
+    
+    def status(s)
+      io.puts s
+    end
+    
+    def stat(s)
+      io.puts "  #{s}"
+    end
+    alias :info :stat
+    
+    def file(s)
+      io.puts s.inspect
+    end
+    
+    
+    
+    def count_notice(message)
+      return if production?
+      @notices[message] = (@notices[message] || 0) + 1
+    end
+    
+    def count_error(message)
+      @errors[message] = (@errors[message] || 0) + 1
+    end
+    
+    
+    
+  private
+    
+    def print_invalid_params
+      return if invalid_params.empty?
+      status "\n\n\n#{("="*80)}\nExamples of invalid hashes\n#{("="*80)}"
+      invalid_params.each do |model_name, errors|
+        status "\n\n--#{model_name}#{("-"*(78 - model_name.length))}"
+        errors.each do |error_message, hash|
+          status "\n  #{error_message}:\n    #{hash.inspect}"
+        end
+      end
+    end
+    
+    def print_messages(array, caption)
+      return if array.empty?
+      status "\n--#{caption}#{("-"*(78-caption.length))}\n\n"
+      array.each do |message, count|
+        stat "#{count} × #{message}"
+      end
+    end
+    
+    def distance_of_time(milliseconds)
+      milliseconds = milliseconds.to_i
+      seconds = milliseconds / 1000
+      milliseconds %= 1000
+      minutes = seconds / 60
+      seconds %= 60
+      hours = minutes / 60
+      minutes %= 60
+      days = hours / 24
+      hours %= 24
+      
+      time = []
+      time << "#{days} days" unless days.zero?
+      time << "#{hours} hours" unless hours.zero?
+      time << "#{minutes} minutes" unless minutes.zero?
+      time << "#{seconds}.#{milliseconds.to_s.rjust(3, "0")} seconds"
+      time.join(", ")
+    end
+    
+  end
+end

data/lib/abstract_importer/version.rb

@@ -0,0 +1,3 @@
+module AbstractImporter
+  VERSION = "1.0.0"
+end

data/lib/abstract_importer.rb

@@ -0,0 +1,2 @@
+require 'abstract_importer/base'
+require 'abstract_importer/version'

data/test/callback_test.rb

@@ -0,0 +1,95 @@
+require "test_helper"
+
+
+class CallbackTest < ActiveSupport::TestCase
+  
+  
+  
+  context "before_build" do
+    setup do
+      plan do |import|
+        import.students do |options|
+          options.before_build { |attrs| attrs.merge(name: attrs[:name][/(\S+)/, 1]) }
+        end
+      end
+    end
+    
+    should "should be invoked on the incoming attributes" do
+      import!
+      assert_equal ["Harry", "Ron", "Hermione"], account.students.pluck(:name)
+    end
+  end
+  
+  
+  
+  context "before_create" do
+    setup do
+      plan do |import|
+        import.students do |options|
+          options.before_create { |student| student.house = "Gryffindor" }
+        end
+      end
+    end
+    
+    should "should be invoked on imported records before they are saved" do
+      import!
+      assert_equal ["Gryffindor"], account.students.pluck(:house).uniq
+    end
+  end
+  
+  
+  
+  context "after_create" do
+    setup do
+      plan do |import|
+        import.students do |options|
+          options.after_create :callback
+        end
+      end
+    end
+    
+    should "should be invoked after the record is created" do
+      mock(importer).callback({name: "Harry Potter"}, satisfy(&:persisted?)).once
+      mock(importer).callback({name: "Ron Weasley"}, satisfy(&:persisted?)).once
+      mock(importer).callback({name: "Hermione Granger"}, satisfy(&:persisted?)).once
+      import!
+    end
+  end
+  
+  
+  
+  context "rescue" do
+    setup do
+      plan do |import|
+        import.locations do |options|
+          options.rescue { |location| location.slug = location.slug.gsub(/[^a-z0-9\-]/, "") }
+        end
+      end
+    end
+    
+    should "should be given a chance to amend an invalid record" do
+      import!
+      assert_equal ["godrics-hollow", "azkaban"], account.locations.pluck(:slug)
+    end
+  end
+  
+  
+  
+  context "on_complete" do
+    setup do
+      plan do |import|
+        import.students do |options|
+          options.on_complete :callback
+        end
+      end
+    end
+    
+    should "should be invoked after the collection has been imported" do
+      mock(importer).callback.once
+      import!
+    end
+  end
+  
+  
+  
+end

data/test/importer_test.rb

@@ -0,0 +1,106 @@
+require "test_helper"
+
+
+class ImporterTest < ActiveSupport::TestCase
+  
+  
+  
+  context "with a simple data source" do
+    setup do
+      plan do |import|
+        import.students
+      end
+    end
+    
+    should "import the given records" do
+      import!
+      assert_equal ["Harry Potter", "Ron Weasley", "Hermione Granger"], account.students.pluck(:name)
+    end
+    
+    should "record their legacy_id" do
+      import!
+      assert_equal [456, 457, 458], account.students.pluck(:legacy_id)
+    end
+    
+    should "not import existing records twice" do
+      account.students.create!(name: "Ron Weasley", legacy_id: 457)
+      import!
+      assert_equal 3, account.students.count
+    end
+  end
+  
+  
+  
+  context "with a complex data source" do
+    setup do
+      plan do |import|
+        import.students
+        import.parents
+      end
+    end
+    
+    should "preserve mappings" do
+      import!
+      harry = account.students.find_by_name("Harry Potter")
+      assert_equal ["James Potter", "Lily Potter"], harry.parents.pluck(:name)
+    end
+    
+    should "preserve mappings event when a record was previously imported" do
+      harry = account.students.create!(name: "Harry Potter", legacy_id: 456)
+      import!
+      assert_equal ["James Potter", "Lily Potter"], harry.parents.pluck(:name)
+    end
+  end
+  
+  
+  
+  context "when a finder is specified" do
+    setup do
+      plan do |import|
+        import.students do |options|
+          options.finder { |attrs| parent.students.find_by_name(attrs[:name]) }
+        end
+        import.parents
+      end
+    end
+    
+    should "not import redundant records" do
+      account.students.create!(name: "Ron Weasley", legacy_id: nil)
+      import!
+      assert_equal 3, account.students.count
+    end
+    
+    should "preserve mappings" do
+      harry = account.students.create!(name: "Harry Potter", legacy_id: nil)
+      import!
+      assert_equal ["James Potter", "Lily Potter"], harry.parents.pluck(:name)
+    end
+  end
+  
+  
+  
+  context "with a more complex data source" do
+    setup do
+      plan do |import|
+        import.students
+        import.subjects do |options|
+          options.before_build do |attributes|
+            attributes.merge(:student_ids => attributes[:student_ids].map do |student_id|
+              map_foreign_key(student_id, :subjects, :student_id, :students)
+            end)
+          end
+        end
+        import.grades
+      end
+    end
+    
+    should "preserve mappings" do
+      import!
+      ron = account.students.find_by_name "Ron Weasley"
+      assert_equal ["Advanced Potions: Acceptable", "History of Magic: Troll"], ron.report_card
+    end
+  end
+  
+  
+  
+end

data/test/support/mock_data_source.rb

@@ -0,0 +1,32 @@
+class MockDataSource
+  
+  def students
+    yield id: 456, name: "Harry Potter"
+    yield id: 457, name: "Ron Weasley"
+    yield id: 458, name: "Hermione Granger"
+  end
+  
+  def parents
+    yield id: 88, name: "James Potter", student_id: 456
+    yield id: 89, name: "Lily Potter", student_id: 456
+  end
+  
+  def locations
+    yield id: 5, slug: "godric's-hollow" # <-- invalid
+    yield id: 6, slug: "azkaban"
+  end
+  
+  def subjects
+    yield id: 49, name: "Care of Magical Creatures", student_ids: [456]
+    yield id: 50, name: "Advanced Potions", student_ids: [456, 457]
+    yield id: 51, name: "History of Magic", student_ids: [457]
+    yield id: 52, name: "Arithmancy", student_ids: [458]
+    yield id: 53, name: "Study of Ancient Runes", student_ids: [458]
+  end
+  
+  def grades
+    yield id: 500, subject_id: 50, student_id: 457, value: "Acceptable"
+    yield id: 501, subject_id: 51, student_id: 457, value: "Troll"
+  end
+  
+end

data/test/support/mock_objects.rb

@@ -0,0 +1,37 @@
+class Student < ActiveRecord::Base
+  has_and_belongs_to_many :subjects
+  has_many :grades
+  has_many :parents
+  
+  def report_card
+    subjects.map do |subject|
+      grade = grades.find_by_subject_id(subject.id)
+      "#{subject.name}: #{grade.value if grade}"
+    end
+  end
+end
+
+class Parent < ActiveRecord::Base
+  belongs_to :student
+end
+
+class Location < ActiveRecord::Base
+  validates :slug, format: {with: /\A[a-z0-9\-]+\z/}
+end
+
+class Subject < ActiveRecord::Base
+  has_and_belongs_to_many :students
+end
+
+class Grade < ActiveRecord::Base
+  belongs_to :student
+  belongs_to :subject
+end
+
+class Account < ActiveRecord::Base
+  has_many :parents
+  has_many :students
+  has_many :subjects
+  has_many :grades
+  has_many :locations
+end

data/test/support/schema.rb

@@ -0,0 +1,45 @@
+ActiveRecord::Schema.define(:version => 1) do
+  
+  create_table "accounts", :force => true do |t|
+  end
+  
+  create_table "students", :force => true do |t|
+    t.integer  "account_id"
+    t.integer  "legacy_id"
+    t.string   "name"
+    t.string   "house"
+  end
+  
+  create_table "parents", :force => true do |t|
+    t.integer  "account_id"
+    t.integer  "student_id"
+    t.integer  "legacy_id"
+    t.string   "name"
+  end
+  
+  create_table "locations", :force => true do |t|
+    t.integer  "account_id"
+    t.integer  "legacy_id"
+    t.string   "slug"
+  end
+  
+  create_table "students_subjects", :force => true do |t|
+    t.integer  "student_id"
+    t.integer  "subject_id"
+  end
+  
+  create_table "subjects", :force => true do |t|
+    t.integer  "account_id"
+    t.integer  "legacy_id"
+    t.string   "name"
+  end
+
+  create_table "grades", :force => true do |t|
+    t.integer  "account_id"
+    t.integer  "subject_id"
+    t.integer  "student_id"
+    t.integer  "legacy_id"
+    t.string   "value"
+  end
+
+end

data/test/test_helper.rb

@@ -0,0 +1,67 @@
+require "rubygems"
+
+require "simplecov"
+SimpleCov.start do
+  add_filter "test/"
+end
+
+require "rails"
+require "rails/test_help"
+require "turn"
+require "pry"
+require "rr"
+require "database_cleaner"
+require "abstract_importer"
+require "shoulda/context"
+require "active_record"
+require "support/mock_data_source"
+require "support/mock_objects"
+
+
+
+ActiveRecord::Base.establish_connection(
+  :adapter => "sqlite3",
+  :database => ":memory:",
+  :verbosity => "quiet")
+
+load File.join(File.dirname(__FILE__), "support", "schema.rb")
+
+
+
+DatabaseCleaner.strategy = :transaction
+$io = ENV['VERBOSE'] ? $stderr : File.open("/dev/null", "w")
+
+
+
+class ActiveSupport::TestCase
+  
+  setup do
+    DatabaseCleaner.start
+    
+    @data_source = MockDataSource.new
+    @klass = Class.new(AbstractImporter::Base)
+    @account = Account.create!
+  end
+  
+  teardown do
+    DatabaseCleaner.clean
+    @importer = nil
+  end
+  
+protected
+  
+  attr_reader :account, :results
+  
+  def plan(&block)
+    @klass.import(&block)
+  end
+  
+  def import!
+    @results = importer.perform!
+  end
+  
+  def importer
+    @importer ||= @klass.new(@account, @data_source, io: $io)
+  end
+  
+end