active_record_importer 0.4.0 → 1.0.0

data/lib/active_record_importer/instance_builder.rb

@@ -6,7 +6,13 @@ module ActiveRecordImporter
     attribute :insert_method, String, default: 'insert'
     attribute :instance_attrs, Hash, default: {}
     attribute :find_attributes, Hash, default: {}
+    attribute :lookup, Object, default: :default_lookup
 
+    ##
+    # Builds (or finds) the instance for a row, runs the before_save
+    # callback, assigns the attributes and validates it.
+    # The instance is NOT persisted here; see BatchPersister.
+    ##
     def build
       instance = initialize_instance
       process_data(instance)
@@ -14,23 +20,25 @@ module ActiveRecordImporter
 
     private
 
-    delegate :error_duplicate, :insert?, to: :insert_method_inquiry
+    delegate :error_duplicate?, :insert?, to: :insert_method_inquiry
 
     def initialize_instance
       return importable.new if insert?
 
       fail Errors::MissingFindByOption if find_attributes.blank?
-      importable.find_or_initialize_by(find_attributes)
+
+      existing = lookup.find(find_attributes)
+      fail Errors::DuplicateRecord if existing && error_duplicate?
+
+      existing || lookup.register(find_attributes, importable.new)
     end
 
     delegate :importer_options, to: :importable
     delegate :before_save, to: :importer_options
 
     def process_data(instance)
-      fail Errors::DuplicateRecord if error_duplicate?(instance)
-
       before_save_callback(instance)
-      assign_attrs_and_save!(instance)
+      assign_attrs_and_validate!(instance)
     end
 
     def before_save_callback(instance)
@@ -38,9 +46,9 @@ module ActiveRecordImporter
       ImportCallbacker.new(instance, before_save).call
     end
 
-    def assign_attrs_and_save!(instance)
+    def assign_attrs_and_validate!(instance)
       instance.attributes = instance_attrs
-      instance.save!
+      fail ActiveRecord::RecordInvalid.new(instance) unless instance.valid?
       instance
     end
 
@@ -48,8 +56,8 @@ module ActiveRecordImporter
       insert_method.inquiry
     end
 
-    def error_duplicate?(instance)
-      instance.persisted? && error_duplicate?
+    def default_lookup
+      RecordPreloader.new(importable)
     end
   end
 end

data/lib/active_record_importer/options_builder.rb

@@ -20,7 +20,7 @@ module ActiveRecordImporter
       attribute :remove_empty_values, Boolean, default: false
       attribute :comment_regexp, Regexp, default: Regexp.new(/^#=>/)
       attribute :force_utf8, Boolean, default: true
-      attribute :chunk_size, Integer, default: nil
+      attribute :chunk_size, Integer, default: 500
       attribute :col_sep, String, default: ','
     end
   end
@@ -42,5 +42,9 @@ module ActiveRecordImporter
     attribute :before_save
     attribute :after_save
     attribute :state_machine_attr, Array
+
+    # Set to false to save rows one by one (within a single transaction
+    # per chunk) so the model's own ActiveRecord callbacks keep running
+    attribute :bulk_insert, Boolean, default: true
   end
 end

data/lib/active_record_importer/record_preloader.rb

@@ -0,0 +1,72 @@
+module ActiveRecordImporter
+  ##
+  # Loads the existing records matched by the find options of a whole
+  # chunk with a single query per distinct set of find columns, so that
+  # upsert/error_duplicate do not have to query the database once per row.
+  #
+  # Instances built during the chunk can be registered too, so duplicates
+  # within the same chunk resolve to the same instance.
+  ##
+  class RecordPreloader
+    attr_reader :importable
+
+    def initialize(importable)
+      @importable = importable
+      @records = {}
+      @preloaded_column_sets = []
+    end
+
+    def preload(find_attributes_list)
+      find_attributes_list.compact.reject(&:blank?).group_by(&:keys).each do |columns, list|
+        load_records(columns, list)
+      end
+      self
+    end
+
+    def find(find_attributes)
+      return if find_attributes.blank?
+
+      key = key_for(find_attributes)
+      return @records[key] if @records.key?(key)
+
+      # Fallback for callers that did not preload this set of columns
+      @records[key] = importable.find_by(find_attributes)
+    end
+
+    def register(find_attributes, instance)
+      @records[key_for(find_attributes)] = instance
+    end
+
+    private
+
+    def load_records(columns, find_attributes_list)
+      return if @preloaded_column_sets.include?(columns)
+      @preloaded_column_sets << columns
+
+      conditions = columns.each_with_object({}) do |column, hash|
+        hash[column] = find_attributes_list.map { |attrs| attrs[column] }.uniq
+      end
+
+      importable.where(conditions).each do |record|
+        attrs = columns.each_with_object({}) { |c, h| h[c] = record.read_attribute(c) }
+        @records[key_for(attrs)] ||= record
+      end
+
+      find_attributes_list.each do |attrs|
+        key = key_for(attrs)
+        @records[key] = nil unless @records.key?(key)
+      end
+    end
+
+    def key_for(find_attributes)
+      find_attributes.sort_by { |column, _| column.to_s }.map do |column, value|
+        [column.to_sym, cast(column, value)]
+      end
+    end
+
+    def cast(column, value)
+      return value unless importable.respond_to?(:type_for_attribute)
+      importable.type_for_attribute(column.to_s).cast(value)
+    end
+  end
+end

data/lib/active_record_importer/version.rb

@@ -1,3 +1,3 @@
 module ActiveRecordImporter
-  VERSION = '0.4.0'
+  VERSION = '1.0.0'
 end

data/lib/active_record_importer.rb

@@ -1,5 +1,8 @@
+require 'csv'
 require 'active_record_importer/version'
 require 'active_support/core_ext/module'
+require 'active_record'
+require 'activerecord-import'
 require 'virtus'
 require 'enumerize'
 require 'smarter_csv'
@@ -11,7 +14,9 @@ module ActiveRecordImporter
   ::IMPORTABLES = []
 
   autoload :BatchImporter,        'active_record_importer/batch_importer'
+  autoload :BatchPersister,       'active_record_importer/batch_persister'
   autoload :DataProcessor,        'active_record_importer/data_processor'
+  autoload :RecordPreloader,      'active_record_importer/record_preloader'
   autoload :Dispatcher,           'active_record_importer/dispatcher'
   autoload :Errors,               'active_record_importer/errors'
   autoload :Importable,           'active_record_importer/importable'
@@ -19,7 +24,7 @@ module ActiveRecordImporter
   autoload :OptionsBuilder,       'active_record_importer/options_builder'
   autoload :TransitionProcessor,  'active_record_importer/transition_processor'
   autoload :ImportCallbacker,     'active_record_importer/import_callbacker'
-  autoload :FailedFileBuilder,     'active_record_importer/failed_file_builder'
+  autoload :FailedFileBuilder,    'active_record_importer/failed_file_builder'
 
   module Attribute
     autoload :AttributesBuilder,    'active_record_importer/attribute/attributes_builder'

data/spec/active_record_importer/attribute/builders_spec.rb

@@ -0,0 +1,51 @@
+require 'spec_helper'
+
+RSpec.describe ActiveRecordImporter::Attribute::AttributesBuilder do
+  before { User.acts_as_importable default_attributes: { role: 'member' } }
+
+  def build(row)
+    described_class.new(User, row).build
+  end
+
+  it 'keeps only importable columns with present values' do
+    attrs = build(first_name: 'Ann', last_name: '', email: nil, id: 5, bogus: 'x')
+
+    expect(attrs).to include(first_name: 'Ann', role: 'member', importing: true)
+    expect(attrs.keys).not_to include(:last_name, :email, :id, :bogus)
+  end
+
+  it 'adds created_at and updated_at' do
+    attrs = build(first_name: 'Ann')
+
+    expect(attrs[:created_at]).to be_a(Time)
+    expect(attrs[:updated_at]).to eq(attrs[:created_at])
+  end
+
+  it 'parses timestamps from the row' do
+    attrs = build(first_name: 'Ann', created_at: '2021-05-06 07:08:09')
+
+    expect(attrs[:created_at]).to eq(Time.parse('2021-05-06 07:08:09'))
+  end
+
+  it 'forces utf-8 encoding on strings' do
+    attrs = build(first_name: 'Ann'.dup.force_encoding('ASCII-8BIT'))
+
+    expect(attrs[:first_name].encoding).to eq(Encoding::UTF_8)
+  end
+end
+
+RSpec.describe ActiveRecordImporter::Attribute::FindOptionsBuilder do
+  let(:attrs) { { first_name: 'Ann', last_name: nil, email: 'ann@example.com', ref_email: 'r@example.com' } }
+
+  def build(options)
+    described_class.new({ importable: User, attrs: attrs }.merge(options)).build
+  end
+
+  it 'slices the find options, dropping nil values' do
+    expect(build(find_options: 'first_name, last_name,email')).to eq(first_name: 'Ann', email: 'ann@example.com')
+  end
+
+  it 'supports a prefix' do
+    expect(build(find_options: 'email', prefix: 'ref_')).to eq(email: 'r@example.com')
+  end
+end

data/spec/active_record_importer/batch_persister_spec.rb

@@ -0,0 +1,87 @@
+require 'spec_helper'
+
+RSpec.describe ActiveRecordImporter::BatchPersister do
+  before { User.acts_as_importable }
+
+  def processor_for(attrs)
+    ActiveRecordImporter::DataProcessor.new(importable: User, row_attrs: attrs).tap(&:process)
+  end
+
+  let(:rows) { user_rows(3) }
+  let(:processors) { rows.map { |row| processor_for(row) } }
+
+  subject(:persister) { described_class.new(importable: User, processors: processors) }
+
+  it 'does nothing without processors' do
+    expect { described_class.new(importable: User).persist! }.not_to change(User, :count)
+  end
+
+  it 'inserts new instances in bulk' do
+    counts = count_queries { persister.persist! }
+
+    expect(User.count).to eq(3)
+    expect(counts['INSERT']).to eq(1)
+    expect(processors.none?(&:failed?)).to be(true)
+  end
+
+  it 'marks the offending row as failed when a database error occurs' do
+    User.create!(first_name: 'Dup', email: 'dup@example.com', employee_no: 'EMP-2')
+
+    persister.persist!
+
+    expect(User.count).to eq(3)
+    expect(processors.map(&:failed?)).to eq([false, true, false])
+    expect(processors[1].row_errors).to be_present
+  end
+
+  it 'updates existing instances in bulk' do
+    User.acts_as_importable insert_method: 'upsert', find_options: %i(email)
+    first = User.create!(first_name: 'Old', email: 'user1@example.com', employee_no: 'X1')
+    second = User.create!(first_name: 'Old', email: 'user2@example.com', employee_no: 'X2')
+
+    counts = count_queries { persister.persist! }
+
+    expect(counts['INSERT']).to eq(2) # one for the new row, one upsert for the existing rows
+    expect(first.reload.first_name).to eq('User1')
+    expect(second.reload.first_name).to eq('User2')
+    expect(User.count).to eq(3)
+  end
+
+  it 'skips the update statement when nothing changed' do
+    User.acts_as_importable insert_method: 'upsert', find_options: %i(email)
+    User.create!(first_name: 'Same', email: 'same@example.com')
+    processor = ActiveRecordImporter::DataProcessor.new(
+      importable: User, row_attrs: { email: 'same@example.com' }
+    )
+    processor.process
+    processor.instance.updated_at = processor.instance.updated_at_was
+    processor.instance.created_at = processor.instance.created_at_was
+
+    counts = count_queries { described_class.new(importable: User, processors: [processor]).persist! }
+
+    expect(counts['INSERT'].to_i).to eq(0)
+    expect(processor.failed?).to be(false)
+  end
+
+  context 'when importer callbacks need a persisted instance' do
+    before { User.acts_as_importable after_save: proc { |u| User.after_save_log << u.id } }
+
+    it 'saves rows one by one and runs the callbacks' do
+      counts = count_queries { persister.persist! }
+
+      expect(counts['INSERT']).to eq(3)
+      expect(User.after_save_log.size).to eq(3)
+    end
+  end
+
+  context 'when bulk_insert is disabled' do
+    before { User.acts_as_importable bulk_insert: false }
+
+    it 'saves rows one by one' do
+      counts = count_queries { persister.persist! }
+
+      expect(counts['INSERT']).to eq(3)
+      expect(User.pluck(:imports_count).uniq).to eq([1])
+    end
+  end
+end

data/spec/active_record_importer/data_processor_spec.rb

@@ -0,0 +1,89 @@
+require 'spec_helper'
+
+RSpec.describe ActiveRecordImporter::DataProcessor do
+  before { User.acts_as_importable }
+
+  def processor(row_attrs, options = {})
+    described_class.new({ importable: User, row_attrs: row_attrs }.merge(options))
+  end
+
+  describe '#process' do
+    it 'builds a valid, unsaved instance' do
+      subject = processor(first_name: 'Ann', email: 'ann@example.com')
+
+      expect(subject.process).to be(true)
+      expect(subject.instance).to be_new_record
+      expect(subject.instance.first_name).to eq('Ann')
+      expect(subject.instance.importing?).to be(true)
+      expect(subject).to be_ready
+    end
+
+    it 'records validation errors without raising' do
+      subject = processor(first_name: '', email: 'ann@example.com')
+
+      expect(subject.process).to be(false)
+      expect(subject).to be_failed
+      expect(subject.row_errors).to eq("Validation failed: First name can't be blank")
+    end
+
+    it 'records errors raised while building attributes' do
+      User.acts_as_importable default_attributes: { role: proc { fail 'boom' } }
+      subject = processor(first_name: 'Ann', email: 'ann@example.com')
+
+      expect(subject.process).to be(false)
+      expect(subject.row_errors).to eq('boom')
+      expect(subject.instance).to be_nil
+    end
+  end
+
+  describe 'insert method and find options' do
+    it 'defaults to insert' do
+      expect(processor({}).insert_method).to eq('insert')
+      expect(processor({}).find_attributes_for_lookup).to be_nil
+    end
+
+    it 'prefers the Import object settings over the importer options' do
+      User.acts_as_importable insert_method: 'upsert', find_options: %i(email)
+      import = Import.new(resource: 'User', insert_method: 'error_duplicate', find_options: 'first_name')
+      subject = processor({ first_name: 'Ann', email: 'ann@example.com' }, import: import)
+      subject.prepare
+
+      expect(subject.insert_method).to eq('error_duplicate')
+      expect(subject.find_attributes_for_lookup).to eq(first_name: 'Ann')
+    end
+
+    it 'uses the importer options when there is no Import object' do
+      User.acts_as_importable insert_method: 'upsert', find_options: %i(email first_name)
+      subject = processor(first_name: 'Ann', email: 'ann@example.com')
+      subject.prepare
+
+      expect(subject.insert_method).to eq('upsert')
+      expect(subject.find_attributes).to eq(email: 'ann@example.com', first_name: 'Ann')
+    end
+  end
+
+  describe '#after_persist' do
+    it 'runs state transitions and after_save hooks on persisted instances' do
+      User.acts_as_importable insert_method: 'upsert', find_options: %i(email),
+                              state_machine_attr: %i(state),
+                              after_save: proc { |u| User.after_save_log << u.id }
+      user = User.create!(first_name: 'Ann', email: 'ann@example.com', state: 'pending')
+      subject = processor(email: 'ann@example.com', state: 'active')
+      subject.process
+      subject.instance.save!
+
+      subject.after_persist
+
+      expect(user.reload.state).to eq('active')
+      expect(User.after_save_log).to eq([user.id])
+    end
+
+    it 'does not assign state machine attributes directly' do
+      User.acts_as_importable state_machine_attr: %i(state)
+      subject = processor(first_name: 'Ann', email: 'ann@example.com', state: 'active')
+      subject.process
+
+      expect(subject.instance.state).to be_nil
+    end
+  end
+end

data/spec/active_record_importer/failed_file_builder_spec.rb

@@ -0,0 +1,26 @@
+require 'spec_helper'
+
+RSpec.describe ActiveRecordImporter::FailedFileBuilder do
+  let(:import) { Import.create!(resource: 'User') }
+  let(:path) { "/tmp/failed_file_#{import.id}.csv" }
+
+  subject(:builder) { described_class.new(import) }
+
+  after { FileUtils.rm_f(path) }
+
+  it 'writes a header row followed by the failed rows' do
+    builder.write(first_name: 'Ann', email: 'ann@example.com', import_errors: 'boom')
+    builder.write(first_name: 'Bob', email: 'bob@example.com', import_errors: 'bang, "quoted"')
+
+    csv = CSV.read(path, headers: true)
+    expect(csv.headers).to eq(%w(first_name email import_errors))
+    expect(csv.size).to eq(2)
+    expect(csv[1]['import_errors']).to eq('bang, "quoted"')
+  end
+
+  it 'ignores blank rows' do
+    builder.write({})
+
+    expect(File.exist?(path)).to be(false)
+  end
+end

data/spec/active_record_importer/import_callbacker_spec.rb

@@ -0,0 +1,25 @@
+require 'spec_helper'
+
+RSpec.describe ActiveRecordImporter::ImportCallbacker do
+  let(:object) { User.new(first_name: 'Ann') }
+
+  it 'calls a symbol method on the object' do
+    expect(object).to receive(:activate)
+    described_class.new(object, :activate).call
+  end
+
+  it 'calls a proc with the object' do
+    received = nil
+    described_class.new(object, proc { |o| received = o }).call
+
+    expect(received).to be(object)
+  end
+
+  it 'runs every callback of an array' do
+    received = []
+    expect(object).to receive(:archive)
+    described_class.new(object, [:archive, proc { |o| received << o }]).call
+
+    expect(received).to eq([object])
+  end
+end

data/spec/active_record_importer/instance_builder_spec.rb

@@ -0,0 +1,77 @@
+require 'spec_helper'
+
+RSpec.describe ActiveRecordImporter::InstanceBuilder do
+  before { User.acts_as_importable }
+
+  let(:lookup) { ActiveRecordImporter::RecordPreloader.new(User) }
+
+  def build(options = {})
+    described_class.new({ importable: User, lookup: lookup }.merge(options)).build
+  end
+
+  it 'builds a new validated instance for insert' do
+    instance = build(instance_attrs: { first_name: 'Ann', email: 'ann@example.com' })
+
+    expect(instance).to be_new_record
+    expect(instance).to be_valid
+  end
+
+  it 'raises RecordInvalid for invalid instances' do
+    expect { build(instance_attrs: { first_name: 'Ann', email: 'nope' }) }
+      .to raise_error(ActiveRecord::RecordInvalid, 'Validation failed: Email is invalid')
+  end
+
+  it 'runs the before_save callback before assigning attributes' do
+    User.acts_as_importable before_save: proc { |user| user.last_name = 'Callback' }
+
+    instance = build(instance_attrs: { first_name: 'Ann', email: 'ann@example.com' })
+
+    expect(instance.last_name).to eq('Callback')
+  end
+
+  context 'upsert' do
+    let!(:ann) { User.create!(first_name: 'Ann', email: 'ann@example.com') }
+
+    it 'returns the existing record with the new attributes' do
+      instance = build(insert_method: 'upsert',
+                       find_attributes: { email: 'ann@example.com' },
+                       instance_attrs: { first_name: 'Annie' })
+
+      expect(instance).to eq(ann)
+      expect(instance.first_name).to eq('Annie')
+      expect(instance).to be_changed
+    end
+
+    it 'initializes and registers a new record when none matches' do
+      instance = build(insert_method: 'upsert',
+                       find_attributes: { email: 'new@example.com' },
+                       instance_attrs: { first_name: 'New', email: 'new@example.com' })
+
+      expect(instance).to be_new_record
+      expect(lookup.find(email: 'new@example.com')).to be(instance)
+    end
+
+    it 'requires find attributes' do
+      expect { build(insert_method: 'upsert', find_attributes: {}) }
+        .to raise_error(ActiveRecordImporter::Errors::MissingFindByOption)
+    end
+  end
+
+  context 'error_duplicate' do
+    it 'raises when the record exists' do
+      User.create!(first_name: 'Ann', email: 'ann@example.com')
+
+      expect do
+        build(insert_method: 'error_duplicate', find_attributes: { email: 'ann@example.com' })
+      end.to raise_error(ActiveRecordImporter::Errors::DuplicateRecord)
+    end
+
+    it 'raises when the record was registered earlier in the chunk' do
+      lookup.register({ email: 'ann@example.com' }, User.new)
+
+      expect do
+        build(insert_method: 'error_duplicate', find_attributes: { email: 'ann@example.com' })
+      end.to raise_error(ActiveRecordImporter::Errors::DuplicateRecord)
+    end
+  end
+end

data/spec/active_record_importer/options_builder_spec.rb

@@ -0,0 +1,19 @@
+require 'spec_helper'
+
+RSpec.describe ActiveRecordImporter::OptionsBuilder do
+  it 'builds importer options with defaults' do
+    options = described_class.build
+
+    expect(options.find_options).to eq([])
+    expect(options.bulk_insert).to be(true)
+    expect(options.csv_opts.chunk_size).to eq(500)
+    expect(options.csv_opts.col_sep).to eq(',')
+  end
+
+  it 'merges csv options' do
+    options = described_class.build(csv_opts: { chunk_size: 10, col_sep: '|' }, bulk_insert: false)
+
+    expect(options.csv_opts.to_hash).to include(chunk_size: 10, col_sep: '|', force_utf8: true)
+    expect(options.bulk_insert).to be(false)
+  end
+end

data/spec/active_record_importer/record_preloader_spec.rb

@@ -0,0 +1,59 @@
+require 'spec_helper'
+
+RSpec.describe ActiveRecordImporter::RecordPreloader do
+  subject(:preloader) { described_class.new(User) }
+
+  let!(:ann) { User.create!(first_name: 'Ann', last_name: 'Lee', email: 'ann@example.com', employee_no: 'A', age: 30) }
+  let!(:bob) { User.create!(first_name: 'Bob', last_name: 'Lee', email: 'bob@example.com', employee_no: 'B', age: 40) }
+
+  describe '#preload' do
+    it 'loads all matching records with a single query' do
+      counts = count_queries do
+        preloader.preload([{ email: 'ann@example.com' }, { email: 'bob@example.com' }, { email: 'nobody@example.com' }])
+      end
+
+      expect(counts['SELECT']).to eq(1)
+      expect(count_queries { preloader.find(email: 'ann@example.com') }['SELECT']).to eq(0)
+      expect(preloader.find(email: 'ann@example.com')).to eq(ann)
+      expect(preloader.find(email: 'bob@example.com')).to eq(bob)
+      expect(preloader.find(email: 'nobody@example.com')).to be_nil
+    end
+
+    it 'matches on every column for composite find options' do
+      preloader.preload([{ first_name: 'Ann', last_name: 'Lee' }, { first_name: 'Bob', last_name: 'Smith' }])
+
+      expect(preloader.find(first_name: 'Ann', last_name: 'Lee')).to eq(ann)
+      expect(preloader.find(first_name: 'Bob', last_name: 'Smith')).to be_nil
+    end
+
+    it 'casts values with the column type' do
+      preloader.preload([{ age: '30' }])
+
+      expect(preloader.find(age: '30')).to eq(ann)
+      expect(preloader.find(age: 30)).to eq(ann)
+    end
+
+    it 'ignores blank entries' do
+      expect { preloader.preload([nil, {}]) }.not_to raise_error
+    end
+  end
+
+  describe '#find' do
+    it 'queries the database for a key that was not preloaded' do
+      expect(preloader.find(email: 'bob@example.com')).to eq(bob)
+    end
+
+    it 'returns nil for blank attributes' do
+      expect(preloader.find({})).to be_nil
+    end
+  end
+
+  describe '#register' do
+    it 'returns registered instances for subsequent lookups' do
+      instance = User.new(email: 'new@example.com')
+
+      expect(preloader.register({ email: 'new@example.com' }, instance)).to be(instance)
+      expect(preloader.find(email: 'new@example.com')).to be(instance)
+    end
+  end
+end

data/spec/active_record_importer_spec.rb

@@ -1,11 +1,7 @@
 require 'spec_helper'
 
-describe ActiveRecordImporter do
+RSpec.describe ActiveRecordImporter do
   it 'has a version number' do
     expect(ActiveRecordImporter::VERSION).not_to be nil
   end
-
-  it 'does something useful' do
-    expect(false).to eq(true)
-  end
 end