chewy 0.8.4 → 5.1.0

data/spec/chewy/type/import/bulk_builder_spec.rb

@@ -0,0 +1,279 @@
+require 'spec_helper'
+
+describe Chewy::Type::Import::BulkBuilder do
+  before { Chewy.massacre }
+
+  subject { described_class.new(type, index: index, delete: delete, fields: fields) }
+  let(:type) { PlacesIndex::City }
+  let(:index) { [] }
+  let(:delete) { [] }
+  let(:fields) { [] }
+
+  describe '#bulk_body' do
+    context 'simple bulk', :orm do
+      before do
+        stub_model(:city)
+        stub_index(:places) do
+          define_type City do
+            field :name, :rating
+          end
+        end
+      end
+      let(:cities) { Array.new(3) { |i| City.create!(id: i + 1, name: "City#{i + 17}", rating: 42) } }
+
+      specify { expect(subject.bulk_body).to eq([]) }
+
+      context do
+        let(:index) { cities }
+        specify do
+          expect(subject.bulk_body).to eq([
+            {index: {_id: 1, data: {'name' => 'City17', 'rating' => 42}}},
+            {index: {_id: 2, data: {'name' => 'City18', 'rating' => 42}}},
+            {index: {_id: 3, data: {'name' => 'City19', 'rating' => 42}}}
+          ])
+        end
+      end
+
+      context do
+        let(:delete) { cities }
+        specify do
+          expect(subject.bulk_body).to eq([
+            {delete: {_id: 1}}, {delete: {_id: 2}}, {delete: {_id: 3}}
+          ])
+        end
+      end
+
+      context do
+        let(:index) { cities.first(2) }
+        let(:delete) { [cities.last] }
+        specify do
+          expect(subject.bulk_body).to eq([
+            {index: {_id: 1, data: {'name' => 'City17', 'rating' => 42}}},
+            {index: {_id: 2, data: {'name' => 'City18', 'rating' => 42}}},
+            {delete: {_id: 3}}
+          ])
+        end
+
+        context ':fields' do
+          let(:fields) { %w[name] }
+          specify do
+            expect(subject.bulk_body).to eq([
+              {update: {_id: 1, data: {doc: {'name' => 'City17'}}}},
+              {update: {_id: 2, data: {doc: {'name' => 'City18'}}}},
+              {delete: {_id: 3}}
+            ])
+          end
+        end
+      end
+    end
+
+    context 'parent-child relationship', :orm do
+      before do
+        stub_model(:country)
+        stub_model(:city)
+        adapter == :sequel ? City.many_to_one(:country) : City.belongs_to(:country)
+      end
+
+      before do
+        stub_index(:places) do
+          define_type Country do
+            field :name
+          end
+
+          define_type City do
+            root parent: 'country', parent_id: -> { country_id } do
+              field :name
+              field :rating
+            end
+          end
+        end
+      end
+
+      before { PlacesIndex::Country.import(country) }
+      let(:country) { Country.create!(id: 1, name: 'country') }
+      let(:another_country) { Country.create!(id: 2, name: 'another country') }
+      let(:city) { City.create!(id: 4, country_id: country.id, name: 'city', rating: 42) }
+
+      context 'indexing' do
+        let(:index) { [city] }
+
+        specify do
+          expect(subject.bulk_body).to eq([
+            {index: {_id: city.id, parent: country.id, data: {'name' => 'city', 'rating' => 42}}}
+          ])
+        end
+
+        context do
+          let(:fields) { %w[name] }
+
+          specify do
+            expect(subject.bulk_body).to eq([
+              {update: {_id: city.id, parent: country.id, data: {doc: {'name' => 'city'}}}}
+            ])
+          end
+        end
+      end
+
+      context 'updating parent' do
+        before do
+          PlacesIndex::City.import(city)
+          city.update_attributes(country_id: another_country.id)
+        end
+        let(:index) { [city] }
+
+        specify do
+          expect(subject.bulk_body).to eq([
+            {delete: {_id: city.id, parent: country.id.to_s}},
+            {index: {_id: city.id, parent: another_country.id, data: {'name' => 'city', 'rating' => 42}}}
+          ])
+        end
+
+        context do
+          let(:fields) { %w[name] }
+
+          specify do
+            expect(subject.bulk_body).to eq([
+              {delete: {_id: city.id, parent: country.id.to_s}},
+              {index: {_id: city.id, parent: another_country.id, data: {'name' => 'city', 'rating' => 42}}}
+            ])
+          end
+        end
+      end
+
+      context 'destroying' do
+        before { PlacesIndex::City.import(city) }
+        let(:delete) { [city] }
+
+        specify do
+          expect(subject.bulk_body).to eq([
+            {delete: {_id: city.id, parent: country.id.to_s}}
+          ])
+        end
+      end
+    end
+
+    context 'custom id', :orm do
+      before do
+        stub_model(:city)
+      end
+
+      before do
+        stub_index(:places) do
+          define_type City do
+            root id: -> { name } do
+              field :rating
+            end
+          end
+        end
+      end
+
+      let(:london) { City.create(id: 1, name: 'London', rating: 4) }
+
+      specify do
+        expect { PlacesIndex::City.import(london) }
+          .to update_index(PlacesIndex::City).and_reindex(london.name)
+      end
+
+      context 'indexing' do
+        let(:index) { [london] }
+
+        specify do
+          expect(subject.bulk_body).to eq([
+            {index: {_id: london.name, data: {'rating' => 4}}}
+          ])
+        end
+      end
+
+      context 'destroying' do
+        let(:delete) { [london] }
+
+        specify do
+          expect(subject.bulk_body).to eq([
+            {delete: {_id: london.name}}
+          ])
+        end
+      end
+    end
+
+    context 'crutches' do
+      before do
+        stub_index(:places) do
+          define_type :city do
+            crutch :names do |collection|
+              collection.map { |item| [item.id, "Name#{item.id}"] }.to_h
+            end
+
+            field :name, value: ->(o, c) { c.names[o.id] }
+          end
+        end
+      end
+
+      let(:index) { [double(id: 42)] }
+
+      specify do
+        expect(subject.bulk_body).to eq([
+          {index: {_id: 42, data: {'name' => 'Name42'}}}
+        ])
+      end
+
+      context 'witchcraft' do
+        before { PlacesIndex::City.witchcraft! }
+        specify do
+          expect(subject.bulk_body).to eq([
+            {index: {_id: 42, data: {'name' => 'Name42'}}}
+          ])
+        end
+      end
+    end
+
+    context 'empty ids' do
+      before do
+        stub_index(:places) do
+          define_type :city do
+            field :name
+          end
+        end
+      end
+
+      let(:index) { [{id: 1, name: 'Name0'}, double(id: '', name: 'Name1'), double(name: 'Name2')] }
+      let(:delete) { [double(id: '', name: 'Name3'), {name: 'Name4'}, '', 2] }
+
+      specify do
+        expect(subject.bulk_body).to eq([
+          {index: {_id: 1, data: {'name' => 'Name0'}}},
+          {index: {data: {'name' => 'Name1'}}},
+          {index: {data: {'name' => 'Name2'}}},
+          {delete: {_id: {'name' => 'Name4'}}},
+          {delete: {_id: 2}}
+        ])
+      end
+
+      context do
+        let(:fields) { %w[name] }
+
+        specify do
+          expect(subject.bulk_body).to eq([
+            {update: {_id: 1, data: {doc: {'name' => 'Name0'}}}},
+            {delete: {_id: {'name' => 'Name4'}}},
+            {delete: {_id: 2}}
+          ])
+        end
+      end
+    end
+  end
+
+  describe '#index_objects_by_id' do
+    before do
+      stub_index(:places) do
+        define_type :city do
+          field :name
+        end
+      end
+    end
+
+    let(:index) { [double(id: 1), double(id: 2), double(id: ''), double] }
+    let(:delete) { [double(id: 3)] }
+
+    specify { expect(subject.index_objects_by_id).to eq('1' => index.first, '2' => index.second) }
+  end
+end

data/spec/chewy/type/import/bulk_request_spec.rb

@@ -0,0 +1,102 @@
+require 'spec_helper'
+
+describe Chewy::Type::Import::BulkRequest do
+  before { Chewy.massacre }
+
+  subject { described_class.new(type, suffix: suffix, bulk_size: bulk_size, **bulk_options) }
+  let(:suffix) {}
+  let(:bulk_size) {}
+  let(:bulk_options) { {} }
+  let(:type) { PlacesIndex::City }
+
+  describe '#initialize' do
+    specify { expect { described_class.new(nil, bulk_size: 100) }.to raise_error(ArgumentError) }
+    specify { expect { described_class.new(nil, bulk_size: 100.kilobytes) }.not_to raise_error }
+  end
+
+  describe '#perform' do
+    before do
+      stub_model(:city)
+      stub_index(:places) do
+        define_type City do
+          field :name
+        end
+      end
+    end
+
+    specify do
+      expect(Chewy.client).not_to receive(:bulk)
+      subject.perform([])
+    end
+
+    specify do
+      expect(Chewy.client).to receive(:bulk).with(
+        index: 'places',
+        type: 'city',
+        body: [{index: {id: 42, data: {name: 'Name'}}}]
+      )
+      subject.perform([{index: {id: 42, data: {name: 'Name'}}}])
+    end
+
+    context ':suffix' do
+      let(:suffix) { 'suffix' }
+
+      specify do
+        expect(Chewy.client).to receive(:bulk).with(
+          index: 'places_suffix',
+          type: 'city',
+          body: [{index: {id: 42, data: {name: 'Name'}}}]
+        )
+        subject.perform([{index: {id: 42, data: {name: 'Name'}}}])
+      end
+    end
+
+    context ':bulk_size' do
+      let(:bulk_size) { 1.2.kilobyte }
+
+      specify do
+        expect(Chewy.client).to receive(:bulk).with(
+          index: 'places',
+          type: 'city',
+          body: "{\"index\":{\"id\":42}}\n{\"name\":\"#{'Name' * 10}\"}\n{\"index\":{\"id\":43}}\n{\"name\":\"#{'Shame' * 10}\"}\n"
+        )
+        subject.perform([
+          {index: {id: 42, data: {name: 'Name' * 10}}},
+          {index: {id: 43, data: {name: 'Shame' * 10}}}
+        ])
+      end
+
+      specify do
+        expect(Chewy.client).to receive(:bulk).with(
+          index: 'places',
+          type: 'city',
+          body: "{\"index\":{\"id\":42}}\n{\"name\":\"#{'Name' * 30}\"}\n"
+        )
+        expect(Chewy.client).to receive(:bulk).with(
+          index: 'places',
+          type: 'city',
+          body: "{\"index\":{\"id\":43}}\n{\"name\":\"#{'Shame' * 100}\"}\n"
+        )
+        expect(Chewy.client).to receive(:bulk).with(
+          index: 'places',
+          type: 'city',
+          body: "{\"index\":{\"id\":44}}\n{\"name\":\"#{'Blame' * 30}\"}\n"
+        )
+        subject.perform([
+          {index: {id: 42, data: {name: 'Name' * 30}}},
+          {index: {id: 43, data: {name: 'Shame' * 100}}},
+          {index: {id: 44, data: {name: 'Blame' * 30}}}
+        ])
+      end
+    end
+
+    context 'bulk_options' do
+      let(:bulk_options) { {refresh: true} }
+
+      specify do
+        expect(Chewy.client).to receive(:bulk).with(hash_including(bulk_options))
+        subject.perform([{index: {id: 42, data: {name: 'Name'}}}])
+      end
+    end
+  end
+end

data/spec/chewy/type/import/journal_builder_spec.rb

@@ -0,0 +1,95 @@
+require 'spec_helper'
+
+describe Chewy::Type::Import::JournalBuilder, :orm do
+  before do
+    stub_model(:country)
+    stub_index 'namespace/places' do
+      define_type :city
+      define_type Country
+    end
+    Timecop.freeze(time)
+  end
+  after { Timecop.return }
+
+  let(:time) { Time.parse('2017-07-14 12:00Z') }
+
+  let(:type) { Namespace::PlacesIndex::City }
+  let(:index) { [] }
+  let(:delete) { [] }
+  subject { described_class.new(type, index: index, delete: delete) }
+
+  describe '#bulk_body' do
+    specify { expect(subject.bulk_body).to eq([]) }
+
+    context do
+      let(:index) { [{id: 1, name: 'City'}] }
+      specify do
+        expect(subject.bulk_body).to eq([{
+          index: {
+            _index: 'chewy_journal',
+            _type: 'journal',
+            data: {
+              'index_name' => 'namespace/places',
+              'type_name' => 'city',
+              'action' => 'index',
+              'references' => [Base64.encode64('{"id":1,"name":"City"}')],
+              'created_at' => time.as_json
+            }
+          }
+        }])
+      end
+    end
+
+    context do
+      let(:delete) { [{id: 1, name: 'City'}] }
+      specify do
+        expect(subject.bulk_body).to eq([{
+          index: {
+            _index: 'chewy_journal',
+            _type: 'journal',
+            data: {
+              'index_name' => 'namespace/places',
+              'type_name' => 'city',
+              'action' => 'delete',
+              'references' => [Base64.encode64('{"id":1,"name":"City"}')],
+              'created_at' => time.as_json
+            }
+          }
+        }])
+      end
+    end
+
+    context do
+      let(:type) { Namespace::PlacesIndex::Country }
+      let(:index) { [Country.new(id: 1, name: 'City')] }
+      let(:delete) { [Country.new(id: 2, name: 'City')] }
+      specify do
+        expect(subject.bulk_body).to eq([{
+          index: {
+            _index: 'chewy_journal',
+            _type: 'journal',
+            data: {
+              'index_name' => 'namespace/places',
+              'type_name' => 'country',
+              'action' => 'index',
+              'references' => [Base64.encode64('1')],
+              'created_at' => time.as_json
+            }
+          }
+        }, {
+          index: {
+            _index: 'chewy_journal',
+            _type: 'journal',
+            data: {
+              'index_name' => 'namespace/places',
+              'type_name' => 'country',
+              'action' => 'delete',
+              'references' => [Base64.encode64('2')],
+              'created_at' => time.as_json
+            }
+          }
+        }])
+      end
+    end
+  end
+end

data/spec/chewy/type/import/routine_spec.rb

@@ -0,0 +1,110 @@
+require 'spec_helper'
+
+# TODO: add more specs here later
+describe Chewy::Type::Import::Routine do
+  before { Chewy.massacre }
+  before do
+    stub_index(:cities) do
+      define_type :city do
+        field :name
+        field :object, type: 'object'
+      end
+    end
+    CitiesIndex.create!
+  end
+
+  let(:index) { [double(id: 1, name: 'Name', object: {}), double(id: 2, name: 'Name', object: {})] }
+  let(:delete) { [double(id: 3, name: 'Name')] }
+
+  describe '#options' do
+    specify do
+      expect(described_class.new(CitiesIndex::City).options).to eq(
+        journal: nil,
+        refresh: true,
+        update_failover: true,
+        update_fields: [],
+        batch_size: 1000
+      )
+    end
+
+    specify do
+      expect(described_class.new(
+        CitiesIndex::City, batch_size: 100, bulk_size: 1.megabyte, refresh: false
+      ).options).to eq(
+        journal: nil,
+        refresh: false,
+        update_failover: true,
+        update_fields: [],
+        bulk_size: 1_048_576,
+        batch_size: 100
+      )
+    end
+
+    context do
+      before { allow(Chewy).to receive_messages(configuration: Chewy.configuration.merge(journal: true)) }
+      specify do
+        expect(described_class.new(CitiesIndex::City).options).to eq(
+          journal: true,
+          refresh: true,
+          update_failover: true,
+          update_fields: [],
+          batch_size: 1000
+        )
+      end
+    end
+
+    specify do
+      expect(CitiesIndex.client).to receive(:bulk).with(hash_including(refresh: true))
+      described_class.new(CitiesIndex::City).process(index: index)
+    end
+
+    specify do
+      expect(CitiesIndex.client).to receive(:bulk).with(hash_including(refresh: false))
+      described_class.new(CitiesIndex::City, refresh: false).process(index: index)
+    end
+  end
+
+  describe '#parallel_options' do
+    specify { expect(described_class.new(CitiesIndex::City).parallel_options).to be_nil }
+    specify { expect(described_class.new(CitiesIndex::City, parallel: true).parallel_options).to eq({}) }
+    specify { expect(described_class.new(CitiesIndex::City, parallel: 3).parallel_options).to eq(in_processes: 3) }
+    specify { expect(described_class.new(CitiesIndex::City, parallel: {in_threads: 2}).parallel_options).to eq(in_threads: 2) }
+  end
+
+  describe '#stats' do
+    subject { described_class.new(CitiesIndex::City) }
+
+    specify { expect(subject.stats).to eq({}) }
+    specify do
+      expect { subject.process(index: index) }
+        .to change { subject.stats }.to(index: 2)
+    end
+    specify do
+      expect { subject.process(index: index, delete: delete) }
+        .to change { subject.stats }.to(index: 2, delete: 1)
+    end
+    specify do
+      expect do
+        subject.process(index: index, delete: delete)
+        subject.process(index: index, delete: delete)
+      end.to change { subject.stats }.to(index: 4, delete: 2)
+    end
+  end
+
+  describe '#errors' do
+    subject { described_class.new(CitiesIndex::City) }
+    let(:index) { [double(id: 1, name: 'Name', object: ''), double(id: 2, name: 'Name', object: {})] }
+
+    specify { expect(subject.errors).to eq([]) }
+    specify do
+      expect { subject.process(index: index) }
+        .to change { subject.errors }.to(have(1).item)
+    end
+    specify do
+      expect do
+        subject.process(index: index)
+        subject.process(index: index)
+      end.to change { subject.errors }.to(have(2).items)
+    end
+  end
+end