chewy 0.9.0 → 5.2.0

data/spec/chewy/type/import/bulk_request_spec.rb

@@ -0,0 +1,102 @@
+require 'spec_helper'
+
+describe Chewy::Type::Import::BulkRequest do
+  before { Chewy.massacre }
+
+  subject { described_class.new(type, suffix: suffix, bulk_size: bulk_size, **bulk_options) }
+  let(:suffix) {}
+  let(:bulk_size) {}
+  let(:bulk_options) { {} }
+  let(:type) { PlacesIndex::City }
+
+  describe '#initialize' do
+    specify { expect { described_class.new(nil, bulk_size: 100) }.to raise_error(ArgumentError) }
+    specify { expect { described_class.new(nil, bulk_size: 100.kilobytes) }.not_to raise_error }
+  end
+
+  describe '#perform' do
+    before do
+      stub_model(:city)
+      stub_index(:places) do
+        define_type City do
+          field :name
+        end
+      end
+    end
+
+    specify do
+      expect(Chewy.client).not_to receive(:bulk)
+      subject.perform([])
+    end
+
+    specify do
+      expect(Chewy.client).to receive(:bulk).with(
+        index: 'places',
+        type: 'city',
+        body: [{index: {id: 42, data: {name: 'Name'}}}]
+      )
+      subject.perform([{index: {id: 42, data: {name: 'Name'}}}])
+    end
+
+    context ':suffix' do
+      let(:suffix) { 'suffix' }
+
+      specify do
+        expect(Chewy.client).to receive(:bulk).with(
+          index: 'places_suffix',
+          type: 'city',
+          body: [{index: {id: 42, data: {name: 'Name'}}}]
+        )
+        subject.perform([{index: {id: 42, data: {name: 'Name'}}}])
+      end
+    end
+
+    context ':bulk_size' do
+      let(:bulk_size) { 1.2.kilobyte }
+
+      specify do
+        expect(Chewy.client).to receive(:bulk).with(
+          index: 'places',
+          type: 'city',
+          body: "{\"index\":{\"id\":42}}\n{\"name\":\"#{'Name' * 10}\"}\n{\"index\":{\"id\":43}}\n{\"name\":\"#{'Shame' * 10}\"}\n"
+        )
+        subject.perform([
+          {index: {id: 42, data: {name: 'Name' * 10}}},
+          {index: {id: 43, data: {name: 'Shame' * 10}}}
+        ])
+      end
+
+      specify do
+        expect(Chewy.client).to receive(:bulk).with(
+          index: 'places',
+          type: 'city',
+          body: "{\"index\":{\"id\":42}}\n{\"name\":\"#{'Name' * 30}\"}\n"
+        )
+        expect(Chewy.client).to receive(:bulk).with(
+          index: 'places',
+          type: 'city',
+          body: "{\"index\":{\"id\":43}}\n{\"name\":\"#{'Shame' * 100}\"}\n"
+        )
+        expect(Chewy.client).to receive(:bulk).with(
+          index: 'places',
+          type: 'city',
+          body: "{\"index\":{\"id\":44}}\n{\"name\":\"#{'Blame' * 30}\"}\n"
+        )
+        subject.perform([
+          {index: {id: 42, data: {name: 'Name' * 30}}},
+          {index: {id: 43, data: {name: 'Shame' * 100}}},
+          {index: {id: 44, data: {name: 'Blame' * 30}}}
+        ])
+      end
+    end
+
+    context 'bulk_options' do
+      let(:bulk_options) { {refresh: true} }
+
+      specify do
+        expect(Chewy.client).to receive(:bulk).with(hash_including(bulk_options))
+        subject.perform([{index: {id: 42, data: {name: 'Name'}}}])
+      end
+    end
+  end
+end

data/spec/chewy/type/import/journal_builder_spec.rb

@@ -0,0 +1,95 @@
+require 'spec_helper'
+
+describe Chewy::Type::Import::JournalBuilder, :orm do
+  before do
+    stub_model(:country)
+    stub_index 'namespace/places' do
+      define_type :city
+      define_type Country
+    end
+    Timecop.freeze(time)
+  end
+  after { Timecop.return }
+
+  let(:time) { Time.parse('2017-07-14 12:00Z') }
+
+  let(:type) { Namespace::PlacesIndex::City }
+  let(:index) { [] }
+  let(:delete) { [] }
+  subject { described_class.new(type, index: index, delete: delete) }
+
+  describe '#bulk_body' do
+    specify { expect(subject.bulk_body).to eq([]) }
+
+    context do
+      let(:index) { [{id: 1, name: 'City'}] }
+      specify do
+        expect(subject.bulk_body).to eq([{
+          index: {
+            _index: 'chewy_journal',
+            _type: 'journal',
+            data: {
+              'index_name' => 'namespace/places',
+              'type_name' => 'city',
+              'action' => 'index',
+              'references' => [Base64.encode64('{"id":1,"name":"City"}')],
+              'created_at' => time.as_json
+            }
+          }
+        }])
+      end
+    end
+
+    context do
+      let(:delete) { [{id: 1, name: 'City'}] }
+      specify do
+        expect(subject.bulk_body).to eq([{
+          index: {
+            _index: 'chewy_journal',
+            _type: 'journal',
+            data: {
+              'index_name' => 'namespace/places',
+              'type_name' => 'city',
+              'action' => 'delete',
+              'references' => [Base64.encode64('{"id":1,"name":"City"}')],
+              'created_at' => time.as_json
+            }
+          }
+        }])
+      end
+    end
+
+    context do
+      let(:type) { Namespace::PlacesIndex::Country }
+      let(:index) { [Country.new(id: 1, name: 'City')] }
+      let(:delete) { [Country.new(id: 2, name: 'City')] }
+      specify do
+        expect(subject.bulk_body).to eq([{
+          index: {
+            _index: 'chewy_journal',
+            _type: 'journal',
+            data: {
+              'index_name' => 'namespace/places',
+              'type_name' => 'country',
+              'action' => 'index',
+              'references' => [Base64.encode64('1')],
+              'created_at' => time.as_json
+            }
+          }
+        }, {
+          index: {
+            _index: 'chewy_journal',
+            _type: 'journal',
+            data: {
+              'index_name' => 'namespace/places',
+              'type_name' => 'country',
+              'action' => 'delete',
+              'references' => [Base64.encode64('2')],
+              'created_at' => time.as_json
+            }
+          }
+        }])
+      end
+    end
+  end
+end

data/spec/chewy/type/import/routine_spec.rb

@@ -0,0 +1,110 @@
+require 'spec_helper'
+
+# TODO: add more specs here later
+describe Chewy::Type::Import::Routine do
+  before { Chewy.massacre }
+  before do
+    stub_index(:cities) do
+      define_type :city do
+        field :name
+        field :object, type: 'object'
+      end
+    end
+    CitiesIndex.create!
+  end
+
+  let(:index) { [double(id: 1, name: 'Name', object: {}), double(id: 2, name: 'Name', object: {})] }
+  let(:delete) { [double(id: 3, name: 'Name')] }
+
+  describe '#options' do
+    specify do
+      expect(described_class.new(CitiesIndex::City).options).to eq(
+        journal: nil,
+        refresh: true,
+        update_failover: true,
+        update_fields: [],
+        batch_size: 1000
+      )
+    end
+
+    specify do
+      expect(described_class.new(
+        CitiesIndex::City, batch_size: 100, bulk_size: 1.megabyte, refresh: false
+      ).options).to eq(
+        journal: nil,
+        refresh: false,
+        update_failover: true,
+        update_fields: [],
+        bulk_size: 1_048_576,
+        batch_size: 100
+      )
+    end
+
+    context do
+      before { allow(Chewy).to receive_messages(configuration: Chewy.configuration.merge(journal: true)) }
+      specify do
+        expect(described_class.new(CitiesIndex::City).options).to eq(
+          journal: true,
+          refresh: true,
+          update_failover: true,
+          update_fields: [],
+          batch_size: 1000
+        )
+      end
+    end
+
+    specify do
+      expect(CitiesIndex.client).to receive(:bulk).with(hash_including(refresh: true))
+      described_class.new(CitiesIndex::City).process(index: index)
+    end
+
+    specify do
+      expect(CitiesIndex.client).to receive(:bulk).with(hash_including(refresh: false))
+      described_class.new(CitiesIndex::City, refresh: false).process(index: index)
+    end
+  end
+
+  describe '#parallel_options' do
+    specify { expect(described_class.new(CitiesIndex::City).parallel_options).to be_nil }
+    specify { expect(described_class.new(CitiesIndex::City, parallel: true).parallel_options).to eq({}) }
+    specify { expect(described_class.new(CitiesIndex::City, parallel: 3).parallel_options).to eq(in_processes: 3) }
+    specify { expect(described_class.new(CitiesIndex::City, parallel: {in_threads: 2}).parallel_options).to eq(in_threads: 2) }
+  end
+
+  describe '#stats' do
+    subject { described_class.new(CitiesIndex::City) }
+
+    specify { expect(subject.stats).to eq({}) }
+    specify do
+      expect { subject.process(index: index) }
+        .to change { subject.stats }.to(index: 2)
+    end
+    specify do
+      expect { subject.process(index: index, delete: delete) }
+        .to change { subject.stats }.to(index: 2, delete: 1)
+    end
+    specify do
+      expect do
+        subject.process(index: index, delete: delete)
+        subject.process(index: index, delete: delete)
+      end.to change { subject.stats }.to(index: 4, delete: 2)
+    end
+  end
+
+  describe '#errors' do
+    subject { described_class.new(CitiesIndex::City) }
+    let(:index) { [double(id: 1, name: 'Name', object: ''), double(id: 2, name: 'Name', object: {})] }
+
+    specify { expect(subject.errors).to eq([]) }
+    specify do
+      expect { subject.process(index: index) }
+        .to change { subject.errors }.to(have(1).item)
+    end
+    specify do
+      expect do
+        subject.process(index: index)
+        subject.process(index: index)
+      end.to change { subject.errors }.to(have(2).items)
+    end
+  end
+end

data/spec/chewy/type/import_spec.rb

@@ -15,44 +15,89 @@ describe Chewy::Type::Import do
     end
   end
 
+  def imported_cities
+    CitiesIndex::City.all.map do |city|
+      city.attributes.except('_score', '_explanation')
+    end
+  end
+
+  def subscribe_notification
+    outer_payload = {}
+    ActiveSupport::Notifications.subscribe('import_objects.chewy') do |_name, _start, _finish, _id, payload|
+      outer_payload.merge!(payload)
+    end
+    outer_payload
+  end
+
   let!(:dummy_cities) { Array.new(3) { |i| City.create(id: i + 1, name: "name#{i}") } }
-  let(:city) { CitiesIndex::City }
 
-  describe '.import', :orm do
-    specify { expect(city.import).to eq(true) }
-    specify { expect(city.import([])).to eq(true) }
-    specify { expect(city.import(dummy_cities)).to eq(true) }
-    specify { expect(city.import(dummy_cities.map(&:id))).to eq(true) }
+  describe 'index creation on import' do
+    let(:dummy_city) { City.create }
+
+    specify 'lazy (default)' do
+      expect(CitiesIndex).to receive(:exists?).and_call_original
+      expect(CitiesIndex).to receive(:create!).and_call_original
+      CitiesIndex::City.import(dummy_city)
+    end
+
+    specify 'lazy without objects' do
+      expect(CitiesIndex).not_to receive(:exists?)
+      expect(CitiesIndex).not_to receive(:create!)
+      CitiesIndex::City.import([])
+    end
 
-    specify { expect { city.import([]) }.not_to update_index(city) }
-    specify { expect { city.import }.to update_index(city).and_reindex(dummy_cities) }
-    specify { expect { city.import dummy_cities }.to update_index(city).and_reindex(dummy_cities) }
-    specify { expect { city.import dummy_cities.map(&:id) }.to update_index(city).and_reindex(dummy_cities) }
+    context 'skip' do
+      before do
+        # To avoid flaky issues when previous specs were run
+        expect(Chewy::Index).to receive(:descendants).and_return([CitiesIndex])
+        Chewy.create_indices
+        Chewy.config.settings[:skip_index_creation_on_import] = true
+      end
+      after { Chewy.config.settings[:skip_index_creation_on_import] = nil }
+
+      specify do
+        expect(CitiesIndex).not_to receive(:exists?)
+        expect(CitiesIndex).not_to receive(:create!)
+        CitiesIndex::City.import(dummy_city)
+      end
+    end
+  end
+
+  shared_examples 'importing' do
+    specify { expect(import).to eq(true) }
+    specify { expect(import([])).to eq(true) }
+    specify { expect(import(dummy_cities)).to eq(true) }
+    specify { expect(import(dummy_cities.map(&:id))).to eq(true) }
+
+    specify { expect { import([]) }.not_to update_index(CitiesIndex::City) }
+    specify { expect { import }.to update_index(CitiesIndex::City).and_reindex(dummy_cities) }
+    specify { expect { import dummy_cities }.to update_index(CitiesIndex::City).and_reindex(dummy_cities) }
+    specify { expect { import dummy_cities.map(&:id) }.to update_index(CitiesIndex::City).and_reindex(dummy_cities) }
 
     describe 'criteria-driven importing' do
-      let(:names) { %w(name0 name1) }
+      let(:names) { %w[name0 name1] }
 
       context 'mongoid', :mongoid do
-        specify { expect { city.import(City.where(:name.in => names)) }.to update_index(city).and_reindex(dummy_cities.first(2)) }
-        specify { expect { city.import(City.where(:name.in => names).map(&:id)) }.to update_index(city).and_reindex(dummy_cities.first(2)) }
+        specify { expect { import(City.where(:name.in => names)) }.to update_index(CitiesIndex::City).and_reindex(dummy_cities.first(2)) }
+        specify { expect { import(City.where(:name.in => names).map(&:id)) }.to update_index(CitiesIndex::City).and_reindex(dummy_cities.first(2)) }
       end
 
       context 'active record', :active_record do
-        specify { expect { city.import(City.where(name: names)) }.to update_index(city).and_reindex(dummy_cities.first(2)) }
-        specify { expect { city.import(City.where(name: names).map(&:id)) }.to update_index(city).and_reindex(dummy_cities.first(2)) }
+        specify { expect { import(City.where(name: names)) }.to update_index(CitiesIndex::City).and_reindex(dummy_cities.first(2)) }
+        specify { expect { import(City.where(name: names).map(&:id)) }.to update_index(CitiesIndex::City).and_reindex(dummy_cities.first(2)) }
       end
     end
 
     specify do
       dummy_cities.first.destroy
-      expect { city.import dummy_cities }
-        .to update_index(city).and_reindex(dummy_cities.from(1)).and_delete(dummy_cities.first)
+      expect { import dummy_cities }
+        .to update_index(CitiesIndex::City).and_reindex(dummy_cities.from(1)).and_delete(dummy_cities.first)
     end
 
     specify do
       dummy_cities.first.destroy
-      expect { city.import dummy_cities.map(&:id) }
-        .to update_index(city).and_reindex(dummy_cities.from(1)).and_delete(dummy_cities.first)
+      expect { import dummy_cities.map(&:id) }
+        .to update_index(CitiesIndex::City).and_reindex(dummy_cities.from(1)).and_delete(dummy_cities.first)
     end
 
     specify do
@@ -64,80 +109,34 @@ describe Chewy::Type::Import do
         nil
       }
 
-      city.import dummy_cities.map(&:id), batch_size: 2
+      import dummy_cities.map(&:id), batch_size: 2
       expect(imported.flatten).to match_array([
-        { index: { _id: 2, data: { 'name' => 'name1' } } },
-        { index: { _id: 3, data: { 'name' => 'name2' } } },
-        { delete: { _id: dummy_cities.first.id } }
+        {index: {_id: 2, data: {'name' => 'name1'}}},
+        {index: {_id: 3, data: {'name' => 'name2'}}},
+        {delete: {_id: dummy_cities.first.id}}
       ])
     end
 
     context ':bulk_size' do
-      specify { expect(city.import(dummy_cities.first, bulk_size: 1.2.kilobyte)).to eq(true) }
-      specify { expect(city.import(dummy_cities, bulk_size: 1.2.kilobyte)).to eq(true) }
-      specify { expect { city.import(dummy_cities, bulk_size: 1.2.kilobyte) }.to update_index(city).and_reindex(dummy_cities) }
-
-      specify do
-        dummy_cities.first.destroy
-
-        imported = []
-        allow(CitiesIndex.client).to receive(:bulk) { |params|
-          imported << params[:body]
-          nil
-        }
+      let!(:dummy_cities) { Array.new(3) { |i| City.create(id: i + 1, name: "name#{i}" * 20) } }
 
-        city.import dummy_cities.map(&:id), bulk_size: 1.2.kilobyte
-        expect(imported.flatten).to match_array([
-          %({"delete":{"_id":1}}\n),
-          %({"index":{"_id":2}}\n{"name":"name1"}\n{"index":{"_id":3}}\n{"name":"name2"}\n)
-        ])
-      end
+      specify { expect { import(dummy_cities, bulk_size: 1.2.kilobyte) }.to update_index(CitiesIndex::City).and_reindex(dummy_cities) }
 
       context do
-        let!(:dummy_cities) { Array.new(3) { |i| City.create(id: i + 1, name: "name#{i}" * 20) } }
-
-        specify do
-          dummy_cities.first.destroy
-
-          imported = []
-          allow(CitiesIndex.client).to receive(:bulk) { |params|
-            imported << params[:body]
-            nil
-          }
-
-          city.import dummy_cities.map(&:id), bulk_size: 1.2.kilobyte
-          expect(imported.flatten).to match_array([
-            %({"delete":{"_id":1}}\n),
-            %({"index":{"_id":2}}\n{"name":"#{'name1' * 20}"}\n),
-            %({"index":{"_id":3}}\n{"name":"#{'name2' * 20}"}\n)
-          ])
-        end
-
-        specify do
-          expect { city.import dummy_cities.map(&:id), bulk_size: 1.1.kilobyte }.to raise_error ArgumentError
-        end
+        before { expect(Chewy.client).to receive(:bulk).exactly(3).times.and_call_original }
+        specify { expect(import(dummy_cities, bulk_size: 1.2.kilobyte)).to eq(true) }
       end
     end
 
-    specify do
-      expect(CitiesIndex.client).to receive(:bulk).with(hash_including(refresh: true))
-      city.import dummy_cities
-    end
-
-    specify do
-      expect(CitiesIndex.client).to receive(:bulk).with(hash_including(refresh: false))
-      city.import dummy_cities, refresh: false
-    end
-
     context 'scoped' do
       before do
-        names = %w(name0 name1)
+        names = %w[name0 name1]
 
         criteria = case adapter
         when :mongoid
-          { :name.in => names }
+          {:name.in => names}
         else
-          { name: names }
+          {name: names}
         end
 
         stub_index(:cities) do
@@ -147,52 +146,40 @@ describe Chewy::Type::Import do
         end
       end
 
-      specify { expect { city.import }.to update_index(city).and_reindex(dummy_cities.first(2)) }
+      specify { expect { import }.to update_index(CitiesIndex::City).and_reindex(dummy_cities.first(2)) }
 
       context 'mongoid', :mongoid do
         specify do
-          expect { city.import City.where(_id: dummy_cities.first.id) }.to update_index(city).and_reindex(dummy_cities.first).only
+          expect { import City.where(_id: dummy_cities.first.id) }.to update_index(CitiesIndex::City).and_reindex(dummy_cities.first).only
         end
       end
 
       context 'active record', :active_record do
         specify do
-          expect { city.import City.where(id: dummy_cities.first.id) }.to update_index(city).and_reindex(dummy_cities.first).only
+          expect { import City.where(id: dummy_cities.first.id) }.to update_index(CitiesIndex::City).and_reindex(dummy_cities.first).only
         end
       end
     end
 
     context 'instrumentation payload' do
       specify do
-        outer_payload = nil
-        ActiveSupport::Notifications.subscribe('import_objects.chewy') do |_name, _start, _finish, _id, payload|
-          outer_payload = payload
-        end
-
+        payload = subscribe_notification
         dummy_cities.first.destroy
-        city.import dummy_cities
-        expect(outer_payload).to eq(type: CitiesIndex::City, import: { delete: 1, index: 2 })
+        import dummy_cities
+        expect(payload).to eq(type: CitiesIndex::City, import: {delete: 1, index: 2})
       end
 
       specify do
-        outer_payload = nil
-        ActiveSupport::Notifications.subscribe('import_objects.chewy') do |_name, _start, _finish, _id, payload|
-          outer_payload = payload
-        end
-
+        payload = subscribe_notification
         dummy_cities.first.destroy
-        city.import dummy_cities, batch_size: 2
-        expect(outer_payload).to eq(type: CitiesIndex::City, import: { delete: 1, index: 2 })
+        import dummy_cities, batch_size: 2
+        expect(payload).to eq(type: CitiesIndex::City, import: {delete: 1, index: 2})
       end
 
       specify do
-        outer_payload = nil
-        ActiveSupport::Notifications.subscribe('import_objects.chewy') do |_name, _start, _finish, _id, payload|
-          outer_payload = payload
-        end
-
-        city.import dummy_cities, batch_size: 2
-        expect(outer_payload).to eq(type: CitiesIndex::City, import: { index: 3 })
+        payload = subscribe_notification
+        import dummy_cities, batch_size: 2
+        expect(payload).to eq(type: CitiesIndex::City, import: {index: 3})
       end
 
       context do
@@ -204,235 +191,296 @@ describe Chewy::Type::Import do
           end
         end
 
-        let(:write_failure_exception) do
-          'WriteFailureException; nested: MapperParsingException[object mapping for [city] ' \
-            'tried to parse field [name] as object, but got EOF, has a concrete value been provided to it?]; '
-        end
         let(:mapper_parsing_exception) do
-          'MapperParsingException[object mapping for [city] tried to parse field [name] ' \
-            'as object, but got EOF, has a concrete value been provided to it?]'
-        end
-
-        specify do
-          skip_on_version_gte('2.0', 'format of exception changed in 2.x')
-          outer_payload = nil
-          ActiveSupport::Notifications.subscribe('import_objects.chewy') do |_name, _start, _finish, _id, payload|
-            outer_payload = payload
-          end
-
-          city.import dummy_cities, batch_size: 2
-          expect(outer_payload).to eq(type: CitiesIndex::City,
-            errors: {
-              index: {
-                write_failure_exception => ['1'],
-                mapper_parsing_exception => %w(2 3)
-              }
-            },
-            import: { index: 3 })
+          {
+            'type' => 'mapper_parsing_exception',
+            'reason' => 'object mapping for [name] tried to parse field [name] as object, but found a concrete value'
+          }
         end
 
         specify do
-          skip_on_version_lt('2.0', 'format of exception was changed')
-          outer_payload = nil
-          ActiveSupport::Notifications.subscribe('import_objects.chewy') do |_name, _start, _finish, _id, payload|
-            outer_payload = payload
-          end
-
-          city.import dummy_cities, batch_size: 2
-          expect(outer_payload).to eq(type: CitiesIndex::City,
-            errors: {
-              index: {
-                {
-                  'type' => 'mapper_parsing_exception',
-                  'reason' => 'object mapping for [name] tried to parse field [name] as object, but found a concrete value'
-                } => %w(1 2 3)
-              }
-            },
-            import: { index: 3 })
+          payload = subscribe_notification
+          import dummy_cities, batch_size: 2
+          expect(payload).to eq(type: CitiesIndex::City,
+            errors: {index: {mapper_parsing_exception => %w[1 2 3]}},
+            import: {index: 3})
         end
       end
     end
 
-    context 'error handling' do
-      context do
-        before do
-          stub_index(:cities) do
-            define_type City do
-              field :name, type: 'object'
-            end
-          end
-        end
+    context 'fields' do
+      before { CitiesIndex::City.import!(dummy_cities.first(2)) }
 
-        specify { expect(city.import(dummy_cities)).to eq(false) }
-        specify { expect(city.import(dummy_cities.map(&:id))).to eq(false) }
-        specify { expect(city.import(dummy_cities, batch_size: 1)).to eq(false) }
+      context do
+        before { expect(Chewy.client).to receive(:bulk).twice.and_call_original }
+        specify { expect(import(dummy_cities, update_fields: [:name])).to eq(true) }
       end
 
       context do
-        before do
-          stub_index(:cities) do
-            define_type City do
-              field :name, type: 'object', value: -> { name == 'name1' ? name : { name: name } }
-            end
-          end
-        end
-
-        specify { expect(city.import(dummy_cities)).to eq(false) }
-        specify { expect(city.import(dummy_cities.map(&:id))).to eq(false) }
-        specify { expect(city.import(dummy_cities, batch_size: 2)).to eq(false) }
+        before { CitiesIndex::City.import!(dummy_cities.last) }
+        before { expect(Chewy.client).to receive(:bulk).once.and_call_original }
+        specify { expect(import(dummy_cities, update_fields: [:name])).to eq(true) }
       end
     end
 
-    context 'parent-child relationship', :orm do
-      let(:country) { Country.create(id: 1, name: 'country') }
-      let(:another_country) { Country.create(id: 2, name: 'another country') }
-
+    context 'fields integrational' do
       before do
-        stub_model(:country)
-        stub_model(:city)
-        adapter == :sequel ? City.many_to_one(:country) : City.belongs_to(:country)
-      end
-
-      before do
-        stub_index(:countries) do
-          define_type Country do
+        stub_index(:cities) do
+          define_type :city do
             field :name
-          end
-
-          define_type City do
-            root parent: { type: 'country' }, parent_id: -> { country_id } do
-              field :name
-            end
+            field :object, type: 'object'
           end
         end
       end
 
-      before { CountriesIndex::Country.import(country) }
-
-      let(:child_city) { City.create(id: 4, country_id: country.id, name: 'city') }
-      let(:city) { CountriesIndex::City }
+      let(:objects) do
+        [
+          double('Name1', id: 1, name: 'Name11', object: {foo: 11}),
+          double('Name2', id: 2, name: 'Name12', object: 'foo'),
+          double('Name3', id: 3, name: 'Name13', object: {foo: 13}),
+          double('Name4', id: 4, name: 'Name14', object: 'foo'),
+          double('Name5', id: 5, name: 'Name15', object: {foo: 15}),
+          double('Name6', id: '', name: 'Name16', object: {foo: 16})
+        ]
+      end
 
-      specify { expect(city.import(child_city)).to eq(true) }
-      specify { expect { city.import child_city }.to update_index(city).and_reindex(child_city) }
+      let(:old_objects) do
+        Array.new(6) do |i|
+          double("Name#{i + 1}", id: i + 1, name: "Name#{i + 1}", object: {foo: i + 1})
+        end
+      end
 
       specify do
-        expect(CountriesIndex.client).to receive(:bulk).with(hash_including(
-                                                               body: [{ index: { _id: child_city.id, parent: country.id, data: { 'name' => 'city' } } }]
-        ))
+        payload = subscribe_notification
+
+        expect(Chewy.client).to receive(:bulk).twice.and_call_original
+        import(objects, update_fields: %i[name])
+
+        expect(payload).to eq(
+          errors: {index: {{'type' => 'mapper_parsing_exception', 'reason' => 'object mapping for [object] tried to parse field [object] as object, but found a concrete value'} => %w[2 4]}},
+          import: {index: 6},
+          type: CitiesIndex::City
+        )
+        expect(imported_cities).to match_array([
+          {'id' => '1', 'name' => 'Name11', 'object' => {'foo' => 11}},
+          {'id' => '3', 'name' => 'Name13', 'object' => {'foo' => 13}},
+          {'id' => '5', 'name' => 'Name15', 'object' => {'foo' => 15}}
+        ])
+      end
 
-        city.import child_city
+      specify do
+        payload = subscribe_notification
+
+        expect(Chewy.client).to receive(:bulk).at_least(4).at_most(6).times.and_call_original
+        import(objects, batch_size: 2, update_fields: %i[name])
+
+        expect(payload).to eq(
+          errors: {index: {{'type' => 'mapper_parsing_exception', 'reason' => 'object mapping for [object] tried to parse field [object] as object, but found a concrete value'} => %w[2 4]}},
+          import: {index: 6},
+          type: CitiesIndex::City
+        )
+        expect(imported_cities).to match_array([
+          {'id' => '1', 'name' => 'Name11', 'object' => {'foo' => 11}},
+          {'id' => '3', 'name' => 'Name13', 'object' => {'foo' => 13}},
+          {'id' => '5', 'name' => 'Name15', 'object' => {'foo' => 15}}
+        ])
       end
 
-      context 'updating or deleting' do
-        before { city.import child_city }
+      context do
+        before { CitiesIndex::City.import!(objects[4]) }
 
         specify do
-          child_city.update_attributes(country_id: another_country.id)
+          payload = subscribe_notification
+
+          expect(Chewy.client).to receive(:bulk).at_least(3).at_most(5).times.and_call_original
+          import(objects, batch_size: 2, update_fields: %i[name])
+
+          expect(payload).to eq(
+            errors: {index: {{'type' => 'mapper_parsing_exception', 'reason' => 'object mapping for [object] tried to parse field [object] as object, but found a concrete value'} => %w[2 4]}},
+            import: {index: 6},
+            type: CitiesIndex::City
+          )
+          expect(imported_cities).to match_array([
+            {'id' => '1', 'name' => 'Name11', 'object' => {'foo' => 11}},
+            {'id' => '3', 'name' => 'Name13', 'object' => {'foo' => 13}},
+            {'id' => '5', 'name' => 'Name15', 'object' => {'foo' => 15}}
+          ])
+        end
+      end
 
-          expect(CountriesIndex.client).to receive(:bulk).with(hash_including(
-                                                                 body: [
-                                                                   { delete: { _id: child_city.id, parent: country.id.to_s } },
-                                                                   { index: { _id: child_city.id, parent: another_country.id, data: { 'name' => 'city' } } }
-                                                                 ]
-          ))
+      context do
+        before { CitiesIndex::City.import!(old_objects[1], old_objects[3], objects[4]) }
 
-          city.import child_city
+        specify do
+          payload = subscribe_notification
+
+          expect(Chewy.client).to receive(:bulk).twice.and_call_original
+          import(objects, update_fields: %i[name])
+
+          expect(payload).to eq(
+            import: {index: 6},
+            type: CitiesIndex::City
+          )
+          expect(imported_cities).to match_array([
+            {'id' => '1', 'name' => 'Name11', 'object' => {'foo' => 11}},
+            {'id' => '2', 'name' => 'Name12', 'object' => {'foo' => 2}},
+            {'id' => '3', 'name' => 'Name13', 'object' => {'foo' => 13}},
+            {'id' => '4', 'name' => 'Name14', 'object' => {'foo' => 4}},
+            {'id' => '5', 'name' => 'Name15', 'object' => {'foo' => 15}}
+          ])
         end
 
         specify do
-          child_city.destroy
-
-          expect(CountriesIndex.client).to receive(:bulk).with(hash_including(
-                                                                 body: [{ delete: { _id: child_city.id, parent: country.id.to_s } }]
-          ))
-
-          city.import child_city
+          payload = subscribe_notification
+
+          expect(Chewy.client).to receive(:bulk).at_least(3).at_most(5).times.and_call_original
+          import(objects, batch_size: 2, update_fields: %i[name])
+
+          expect(payload).to eq(
+            import: {index: 6},
+            type: CitiesIndex::City
+          )
+          expect(imported_cities).to match_array([
+            {'id' => '1', 'name' => 'Name11', 'object' => {'foo' => 11}},
+            {'id' => '2', 'name' => 'Name12', 'object' => {'foo' => 2}},
+            {'id' => '3', 'name' => 'Name13', 'object' => {'foo' => 13}},
+            {'id' => '4', 'name' => 'Name14', 'object' => {'foo' => 4}},
+            {'id' => '5', 'name' => 'Name15', 'object' => {'foo' => 15}}
+          ])
         end
 
         specify do
-          child_city.destroy
+          payload = subscribe_notification
 
-          expect(CountriesIndex.client).to receive(:bulk).with(hash_including(
-                                                                 body: [{ delete: { _id: child_city.id, parent: country.id.to_s } }]
-          ))
+          expect(Chewy.client).to receive(:bulk).once.and_call_original
+          import(objects, update_fields: %i[name], update_failover: false)
 
-          city.import child_city.id
+          # Full match doesn't work here.
+          expect(payload[:errors][:update].keys).to match([
+            hash_including('type' => 'document_missing_exception', 'reason' => '[city][1]: document missing'),
+            hash_including('type' => 'document_missing_exception', 'reason' => '[city][3]: document missing')
+          ])
+          expect(payload[:errors][:update].values).to eq([['1'], ['3']])
+          expect(imported_cities).to match_array([
+            {'id' => '2', 'name' => 'Name12', 'object' => {'foo' => 2}},
+            {'id' => '4', 'name' => 'Name14', 'object' => {'foo' => 4}},
+            {'id' => '5', 'name' => 'Name15', 'object' => {'foo' => 15}}
+          ])
         end
+      end
 
-        specify do
-          child_city.destroy
+      context do
+        before { CitiesIndex::City.import!(old_objects) }
 
-          expect(city.import(child_city)).to eq(true)
-          expect(city.import(child_city)).to eq(true)
+        specify do
+          payload = subscribe_notification
+
+          expect(Chewy.client).to receive(:bulk).once.and_call_original
+          import(objects, update_fields: %i[name])
+
+          expect(payload).to eq(
+            import: {index: 6},
+            type: CitiesIndex::City
+          )
+          expect(imported_cities).to match_array([
+            {'id' => '1', 'name' => 'Name11', 'object' => {'foo' => 1}},
+            {'id' => '2', 'name' => 'Name12', 'object' => {'foo' => 2}},
+            {'id' => '3', 'name' => 'Name13', 'object' => {'foo' => 3}},
+            {'id' => '4', 'name' => 'Name14', 'object' => {'foo' => 4}},
+            {'id' => '5', 'name' => 'Name15', 'object' => {'foo' => 5}},
+            {'id' => '6', 'name' => 'Name6', 'object' => {'foo' => 6}}
+          ])
         end
       end
-    end
 
-    context 'root id', :orm do
-      let(:canada) { Country.create(id: 1, name: 'Canada', country_code: 'CA', rating: 4) }
-      let(:country)  { CountriesIndex::Country }
+      context do
+        before { CitiesIndex::City.import!(old_objects) }
 
-      before do
-        stub_model(:country)
+        specify do
+          payload = subscribe_notification
+
+          expect(Chewy.client).to receive(:bulk).once.and_call_original
+          import(objects, update_fields: %i[object])
+
+          expect(payload).to eq(
+            errors: {update: {{'type' => 'mapper_parsing_exception', 'reason' => 'object mapping for [object] tried to parse field [object] as object, but found a concrete value'} => %w[2 4]}},
+            import: {index: 6},
+            type: CitiesIndex::City
+          )
+          expect(imported_cities).to match_array([
+            {'id' => '1', 'name' => 'Name1', 'object' => {'foo' => 11}},
+            {'id' => '2', 'name' => 'Name2', 'object' => {'foo' => 2}},
+            {'id' => '3', 'name' => 'Name3', 'object' => {'foo' => 13}},
+            {'id' => '4', 'name' => 'Name4', 'object' => {'foo' => 4}},
+            {'id' => '5', 'name' => 'Name5', 'object' => {'foo' => 15}},
+            {'id' => '6', 'name' => 'Name6', 'object' => {'foo' => 6}}
+          ])
+        end
       end
+    end
 
-      before do
-        stub_index(:countries) do
-          define_type Country do
-            root _id: -> { country_code } do
-              field :name
-              field :rating
+    context 'error handling' do
+      context do
+        before do
+          stub_index(:cities) do
+            define_type City do
+              field :name, type: 'object'
             end
           end
         end
-      end
-
-      specify { expect(country.import(canada)).to eq(true) }
-      specify { expect { country.import(canada) }.to update_index(country).and_reindex(canada.country_code) }
-
-      specify do
-        expect(CountriesIndex.client).to receive(:bulk).with(hash_including(
-                                                               body: [{ index: { _id: canada.country_code, data: { 'name' => 'Canada', 'rating' => 4 } } }]
-        ))
-
-        country.import canada
-      end
-
-      specify do
-        canada.update_attributes(rating: 9)
 
-        expect(CountriesIndex.client).to receive(:bulk).with(hash_including(
-                                                               body: [{ index: { _id: canada.country_code, data: { 'name' => 'Canada', 'rating' => 9 } } }]
-        ))
-
-        country.import canada
+        specify { expect(import(dummy_cities)).to eq(false) }
+        specify { expect(import(dummy_cities.map(&:id))).to eq(false) }
+        specify { expect(import(dummy_cities, batch_size: 1)).to eq(false) }
       end
 
-      specify do
-        canada.destroy
-
-        expect(CountriesIndex.client).to receive(:bulk).with(hash_including(
-                                                               body: [{ delete: { _id: canada.country_code } }]
-        ))
+      context do
+        before do
+          stub_index(:cities) do
+            define_type City do
+              field :name, type: 'object', value: -> { name == 'name1' ? name : {name: name} }
+            end
+          end
+        end
 
-        country.import canada
+        specify { expect(import(dummy_cities)).to eq(false) }
+        specify { expect(import(dummy_cities.map(&:id))).to eq(false) }
+        specify { expect(import(dummy_cities, batch_size: 2)).to eq(false) }
       end
     end
 
-    context 'default_import_options is set' do
+    context 'default_import_options are set' do
       before do
-        CitiesIndex::City.default_import_options(batch_size: 500, bulk_size: 1.megabyte)
+        CitiesIndex::City.default_import_options(batch_size: 500)
       end
 
       specify do
-        expect(CitiesIndex::City.adapter).to receive(:import).with(batch_size: 500, bulk_size: 1.megabyte)
+        expect(CitiesIndex::City.adapter).to receive(:import).with(any_args, hash_including(batch_size: 500))
         CitiesIndex::City.import
       end
     end
   end
 
+  describe '.import', :orm do
+    def import(*args)
+      CitiesIndex::City.import(*args)
+    end
+
+    it_behaves_like 'importing'
+
+    context 'parallel' do
+      def import(*args)
+        options = args.extract_options!
+        options[:parallel] = 0
+        CitiesIndex::City.import(*args, options)
+      end
+
+      it_behaves_like 'importing'
+    end
+  end
+
   describe '.import!', :orm do
-    specify { expect { city.import! }.not_to raise_error }
+    specify { expect { CitiesIndex::City.import! }.not_to raise_error }
 
     context do
       before do
@@ -443,21 +491,58 @@ describe Chewy::Type::Import do
         end
       end
 
-      specify { expect { city.import!(dummy_cities) }.to raise_error Chewy::ImportFailed }
+      specify { expect { CitiesIndex::City.import!(dummy_cities) }.to raise_error Chewy::ImportFailed }
     end
+  end
 
-    context 'when .import fails' do
-      before do
-        allow(city).to receive(:import) { raise }
+  describe '.compose' do
+    before do
+      stub_index(:cities) do
+        define_type :city do
+          crutch :names do |collection|
+            collection.map { |o| [o.name, o.name + '42'] }.to_h
+          end
+          field :name, value: ->(o, c) { c.names[o.name] }
+          field :rating
+        end
       end
+    end
+
+    specify do
+      expect(CitiesIndex::City.compose(double(name: 'Name', rating: 42)))
+        .to eq('name' => 'Name42', 'rating' => 42)
+    end
+
+    specify do
+      expect(CitiesIndex::City.compose(double(name: 'Name', rating: 42), fields: %i[name]))
+        .to eq('name' => 'Name42')
+    end
+
+    context 'witchcraft' do
+      before { CitiesIndex::City.witchcraft! }
 
       specify do
-        expect(ActiveSupport::Notifications).to receive(:unsubscribe)
-        begin
-          city.import!(dummy_cities)
-        rescue
-          nil
-        end
+        expect(CitiesIndex::City.compose(double(name: 'Name', rating: 42)))
+          .to eq('name' => 'Name42', 'rating' => 42)
+      end
+
+      specify do
+        expect(CitiesIndex::City.compose(double(name: 'Name', rating: 42), fields: %i[name]))
+          .to eq('name' => 'Name42')
+      end
+    end
+
+    context 'custom crutches' do
+      let(:crutches) { double(names: {'Name' => 'Name43'}) }
+
+      specify do
+        expect(CitiesIndex::City.compose(double(name: 'Name', rating: 42), crutches))
+          .to eq('name' => 'Name43', 'rating' => 42)
+      end
+
+      specify do
+        expect(CitiesIndex::City.compose(double(name: 'Name', rating: 42), crutches, fields: %i[name]))
+          .to eq('name' => 'Name43')
       end
     end
   end