chewy 0.8.4 → 0.9.0

data/spec/chewy/journal/apply_spec.rb

@@ -0,0 +1,120 @@
+require 'spec_helper'
+
+describe Chewy::Journal::Apply do
+  describe '.since' do
+    context 'with an index filter' do
+      let(:time) { Time.now }
+
+      before do
+        stub_model(:city) do
+          update_index 'city', :self
+        end
+        stub_model(:country) do
+          update_index 'country', :self
+        end
+
+        stub_index('city') do
+          define_type City do
+            default_import_options journal: true
+          end
+        end
+        stub_index('country') do
+          define_type Country do
+            default_import_options journal: true
+          end
+        end
+
+        Chewy.massacre
+        Timecop.freeze(time)
+      end
+
+      specify do
+        Chewy.strategy(:urgent) do
+          Array.new(2) { |i| City.create!(id: i + 1) }
+          Array.new(2) { |i| Country.create!(id: i + 1) }
+
+          # simulate lost data
+          Chewy.client.delete(index: 'city', type: 'city', id: 1, refresh: true)
+          Chewy.client.delete(index: 'country', type: 'country', id: 1, refresh: true)
+          expect(CityIndex.all.to_a.length).to eq 1
+          expect(CountryIndex.all.to_a.length).to eq 1
+
+          # Replay on specific index
+          described_class.since(time, only: [CityIndex])
+          expect(CityIndex.all.to_a.length).to eq 2
+          expect(CountryIndex.all.to_a.length).to eq 1
+
+          # Replay on both
+          Chewy.client.delete(index: 'city', type: 'city', id: 1, refresh: true)
+          expect(CityIndex.all.to_a.length).to eq 1
+          described_class.since(time, only: [CityIndex, CountryIndex])
+          expect(CityIndex.all.to_a.length).to eq 2
+          expect(CountryIndex.all.to_a.length).to eq 2
+        end
+      end
+    end
+
+    context 'retries' do
+      let(:time) { Time.now.to_i }
+      before do
+        stub_model(:city) do
+          update_index 'city', :self
+        end
+        stub_index('city') do
+          define_type City do
+            default_import_options journal: true
+          end
+        end
+        Chewy.massacre
+        Timecop.freeze
+        Chewy.strategy(:urgent)
+        City.create!(id: 1)
+      end
+
+      after do
+        Chewy.strategy.pop
+      end
+
+      specify 'journal was cleaned after the first call' do
+        expect(Chewy::Journal::Entry)
+          .to receive(:since).exactly(2).and_call_original
+        Chewy::Journal::Apply.since(time)
+      end
+
+      context 'endless journal' do
+        let(:count_of_checks) { 10 } # default
+        let!(:journal_records) do
+          record = Chewy::Journal::Entry.since(time).first
+          Array.new(count_of_checks) do |i|
+            r = record.dup
+            r.created_at = time.to_i + i
+            r.object_ids = [i]
+            r
+          end
+        end
+
+        specify '10 retries by default' do
+          expect(Chewy::Journal::Entry)
+            .to receive(:since).exactly(count_of_checks) { [journal_records.shift].compact }
+          Chewy::Journal::Apply.since(time)
+        end
+
+        specify 'with :once parameter set' do
+          expect(Chewy::Journal::Entry)
+            .to receive(:since).exactly(1) { [journal_records.shift].compact }
+          Chewy::Journal::Apply.since(time, once: true)
+        end
+
+        context 'with retries parameter set' do
+          let(:retries) { 5 }
+
+          specify do
+            expect(Chewy::Journal::Entry)
+              .to receive(:since).exactly(retries) { [journal_records.shift].compact }
+            Chewy::Journal::Apply.since(time, retries: retries)
+          end
+        end
+      end
+    end
+  end
+end

data/spec/chewy/journal/entry_spec.rb

@@ -0,0 +1,237 @@
+require 'spec_helper'
+
+describe Chewy::Journal::Entry do
+  before do
+    stub_model(:city) do
+      update_index 'city', :self
+    end
+    stub_index('city') do
+      define_type City do
+        default_import_options journal: true
+      end
+    end
+  end
+
+  describe '.since' do
+    let(:time) { Time.now.to_i }
+    before do
+      Timecop.freeze(time)
+      Chewy.massacre
+      Chewy.strategy(:urgent) { City.create!(id: 1) }
+    end
+    subject { described_class.since(time) }
+
+    its(:length) { is_expected.to eq(1) }
+
+    context 'with indices parameter provided' do
+      subject { described_class.since(time, indices) }
+
+      context 'it ignores empty array' do
+        let(:indices) { [] }
+        its(:length) { is_expected.to eq(1) }
+      end
+
+      context do
+        let(:indices) { [CityIndex::City2] }
+        before do
+          stub_index('city') do
+            define_type :city2 do
+              default_import_options journal: true
+            end
+          end
+        end
+
+        its(:length) { is_expected.to eq(1) }
+      end
+    end
+  end
+
+  describe '.group' do
+    let(:full_type_name) { 'type' }
+    let(:another_full_type_name) { 'type' }
+    let(:entry) { described_class.new('object_ids' => [1]) }
+    let(:another_entry) { described_class.new('object_ids' => [2]) }
+    before do
+      allow(entry)
+        .to receive(:full_type_name).and_return(full_type_name)
+      allow(another_entry)
+        .to receive(:full_type_name).and_return(another_full_type_name)
+    end
+    subject { described_class.group([entry, another_entry]) }
+
+    specify do
+      expect(subject.length).to eq(1)
+      expect(subject.first.object_ids).to eq([1, 2])
+    end
+
+    context do
+      let(:another_full_type_name) { 'whatever' }
+
+      specify do
+        expect(subject.length).to eq(2)
+        expect(subject.first.object_ids).to eq(entry.object_ids)
+        expect(subject.last.object_ids).to eq(another_entry.object_ids)
+      end
+    end
+  end
+
+  describe '.recent_timestamp' do
+    let(:time) { Time.now.to_i }
+    let(:entry) { described_class.new('created_at' => time) }
+    let(:another_entry) { described_class.new('created_at' => time + 1) }
+    subject { described_class.recent_timestamp([entry, another_entry]) }
+
+    it { is_expected.to eq(time + 1) }
+  end
+
+  describe '.subtract' do
+    let(:full_type_name) { 'type' }
+    let(:another_full_type_name) { 'type' }
+    let(:entry) { described_class.new('object_ids' => [1]) }
+    let(:another_entry) { described_class.new('object_ids' => [2]) }
+    before do
+      allow(entry)
+        .to receive(:full_type_name).and_return(full_type_name)
+      allow(another_entry)
+        .to receive(:full_type_name).and_return(another_full_type_name)
+    end
+    let(:from) { [entry] }
+    let(:what) { [another_entry] }
+    subject { described_class.subtract(from, what) }
+
+    specify { expect { subject }.not_to change { from.length } }
+    specify { expect(entry.object_ids).to eq([1]) }
+
+    context 'object_ids have same elements' do
+      let(:another_entry) { described_class.new('object_ids' => [1, 2]) }
+
+      specify { expect { subject }.to change { from.length }.from(1).to(0) }
+      specify do
+        subject
+        expect(entry.object_ids).to eq([])
+      end
+
+      context 'not all elements are covered by subtracting array' do
+        let(:entry) { described_class.new('object_ids' => [1, 3]) }
+
+        specify { expect { subject }.not_to change { from.length } }
+        specify do
+          subject
+          expect(entry.object_ids).to eq([3])
+        end
+      end
+    end
+  end
+
+  describe '#merge' do
+    let(:time) { Time.now.to_i }
+    let(:index_name) { 'index' }
+    let(:type_name) { 'type' }
+    let(:object_ids) { [1] }
+    let(:entry) do
+      described_class.new('index_name' => index_name,
+                          'type_name' => type_name,
+                          'object_ids' => object_ids,
+                          'created_at' => time)
+    end
+    let(:another_index_name) { 'index' }
+    let(:another_type_name) { 'type' }
+    let(:another_object_ids) { [2] }
+    let(:another_time) { time + 1 }
+    let(:another_entry) do
+      described_class.new('index_name' => another_index_name,
+                          'type_name' => another_type_name,
+                          'object_ids' => another_object_ids,
+                          'created_at' => another_time)
+    end
+    subject { entry.merge(another_entry) }
+
+    specify do
+      expect(subject).to eq(entry)
+      expect(entry.created_at).to eq(another_entry.created_at)
+      expect(entry.object_ids).to eq([1, 2])
+    end
+
+    shared_examples_for 'merge does not change the entry' do
+      it { is_expected.to eq(entry) }
+      specify { expect { subject }.not_to change(entry, :created_at) }
+      specify { expect { subject }.not_to change(entry, :object_ids) }
+    end
+
+    context 'different types' do
+      let(:another_type_name) { 'whatever' }
+      include_examples 'merge does not change the entry'
+    end
+
+    context 'merge with nil' do
+      let(:another_entry) { nil }
+      include_examples 'merge does not change the entry'
+    end
+
+    context 'original entry has more recent time' do
+      let(:another_time) { time - 1 }
+      specify { expect { subject }.not_to change(entry, :created_at) }
+    end
+  end
+
+  describe '#empty?' do
+    let(:object_ids) { [] }
+    subject { described_class.new('object_ids' => object_ids).empty? }
+
+    it { is_expected.to eq(true) }
+
+    context do
+      let(:object_ids) { nil }
+      it { is_expected.to eq(true) }
+    end
+
+    context do
+      let(:object_ids) { [1] }
+      it { is_expected.to eq(false) }
+    end
+  end
+
+  describe '#full_type_name' do
+    subject do
+      described_class.new('index_name' => 'index', 'type_name' => 'type')
+        .full_type_name
+    end
+    it { is_expected.to eq('index#type') }
+  end
+
+  describe '#index' do
+    let(:index_name) { 'wrong_index_name' }
+    let(:type_name) { 'city' }
+    subject do
+      described_class
+        .new('index_name' => index_name, 'type_name' => type_name).index
+    end
+
+    specify { expect { subject }.to raise_error(Chewy::UnderivableType) }
+
+    context do
+      let(:index_name) { 'city' }
+      it { is_expected.to eq(CityIndex::City) }
+    end
+  end
+
+  describe '#==' do
+    let(:full_type_name) { 'type' }
+    let(:another_full_type_name) { 'type' }
+    let(:entry) { described_class.new }
+    let(:another_entry) { described_class.new }
+    before do
+      expect(entry).to receive(:full_type_name).and_return(full_type_name)
+      expect(another_entry)
+        .to receive(:full_type_name).and_return(another_full_type_name)
+    end
+    subject { entry == another_entry }
+
+    it { is_expected.to eq(true) }
+
+    context do
+      let(:another_full_type_name) { 'whatever' }
+      it { is_expected.to eq(false) }
+    end
+  end
+end

data/spec/chewy/journal_spec.rb

@@ -0,0 +1,173 @@
+require 'spec_helper'
+
+describe Chewy::Journal do
+  context 'journaling', orm: true do
+    ['', 'namespace/'].each do |namespace|
+      context namespace.present? ? 'with namespace' : 'without namespace' do
+        before do
+          stub_model(:city) do
+            update_index "#{namespace}places#city", :self
+          end
+          stub_model(:country) do
+            update_index "#{namespace}places#country", :self
+          end
+
+          stub_index("#{namespace}places") do
+            define_type City do
+              default_import_options journal: true
+            end
+            define_type Country do
+              default_import_options journal: true
+            end
+          end
+
+          Chewy.massacre
+          begin
+            Chewy.client.indices.delete(index: Chewy::Journal.index_name)
+          rescue Elasticsearch::Transport::Transport::Errors::NotFound
+            nil
+          end
+          Chewy.settings[:prefix] = 'some_prefix'
+          Timecop.freeze(time)
+        end
+
+        after do
+          Chewy.settings[:prefix] = nil
+          Timecop.return
+        end
+
+        let(:time) { Time.now }
+        let(:import_time) { time + 1 }
+        let(:update_time) { time + 2 }
+        let(:destroy_time) { time + 3 }
+
+        def timestamp(time)
+          time.to_i
+        end
+
+        specify do
+          places_index = namespace.present? ? Namespace::PlacesIndex : PlacesIndex
+          Chewy.strategy(:urgent) do
+            cities = Array.new(2) { |i| City.create!(id: i + 1) }
+            countries = Array.new(2) { |i| Country.create!(id: i + 1) }
+            Country.create!(id: 3)
+
+            Timecop.freeze(import_time)
+
+            places_index.import
+
+            expect(Chewy.client.indices.exists?(index: Chewy::Journal.index_name)).to eq true
+
+            Timecop.freeze(update_time)
+            cities.first.update_attributes!(name: 'Supername')
+
+            Timecop.freeze(destroy_time)
+            countries.last.destroy
+
+            journal_records = Chewy.client.search(index: Chewy::Journal.index_name, type: Chewy::Journal.type_name, sort: 'created_at')['hits']['hits'].map { |r| r['_source'] }
+
+            expected_journal = [
+              {
+                'index_name' => "#{namespace}places",
+                'type_name' => 'city',
+                'action' => 'index',
+                'object_ids' => [1],
+                'created_at' => time.to_i
+              },
+              {
+                'index_name' => "#{namespace}places",
+                'type_name' => 'city',
+                'action' => 'index',
+                'object_ids' => [2],
+                'created_at' => time.to_i
+              },
+              {
+                'index_name' => "#{namespace}places",
+                'type_name' => 'country',
+                'action' => 'index',
+                'object_ids' => [1],
+                'created_at' => time.to_i
+              },
+              {
+                'index_name' => "#{namespace}places",
+                'type_name' => 'country',
+                'action' => 'index',
+                'object_ids' => [2],
+                'created_at' => time.to_i
+              },
+              {
+                'index_name' => "#{namespace}places",
+                'type_name' => 'country',
+                'action' => 'index',
+                'object_ids' => [3],
+                'created_at' => time.to_i
+              },
+              {
+                'index_name' => "#{namespace}places",
+                'type_name' => 'city',
+                'action' => 'index',
+                'object_ids' => [1, 2],
+                'created_at' => import_time.to_i
+              },
+              {
+                'index_name' => "#{namespace}places",
+                'type_name' => 'country',
+                'action' => 'index',
+                'object_ids' => [1, 2, 3],
+                'created_at' => import_time.to_i
+              },
+              {
+                'index_name' => "#{namespace}places",
+                'type_name' => 'city',
+                'action' => 'index',
+                'object_ids' => [1],
+                'created_at' => update_time.to_i
+              },
+              {
+                'index_name' => "#{namespace}places",
+                'type_name' => 'country',
+                'action' => 'delete',
+                'object_ids' => [2],
+                'created_at' => destroy_time.to_i
+              }
+            ]
+
+            expect(Chewy.client.count(index: Chewy::Journal.index_name)['count']).to eq 9
+            expect(journal_records).to eq expected_journal
+
+            journal_entries = Chewy::Journal::Entry.since(import_time)
+            expect(journal_entries.length).to eq 4
+            # we have only 2 types, so we can group all journal entries(4) into 2
+            expect(Chewy::Journal::Entry.group(journal_entries)).to eq [
+              Chewy::Journal::Entry.new('index_name' => "#{namespace}places",
+                                        'type_name' => 'city',
+                                        'action' => nil,
+                                        'object_ids' => [1, 2],
+                                        'created_at' => nil),
+              Chewy::Journal::Entry.new('index_name' => "#{namespace}places",
+                                        'type_name' => 'country',
+                                        'action' => 'delete',
+                                        'object_ids' => [1, 2, 3],
+                                        'created_at' => destroy_time.to_i)
+            ]
+
+            # simulate lost data
+            Chewy.client.delete(index: "#{Chewy.settings[:prefix]}_places", type: 'city', id: 1, refresh: true)
+            expect(places_index::City.all.to_a.length).to eq 1
+
+            Chewy::Journal::Apply.since(time)
+            expect(places_index::City.all.to_a.length).to eq 2
+
+            expect(Chewy::Journal::Clean.until(import_time)).to eq 7
+            expect(Chewy.client.count(index: Chewy::Journal.index_name)['count']).to eq 2
+
+            expect(Chewy::Journal.delete!).to be_truthy
+            expect { Chewy::Journal.delete! }.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
+            expect(Chewy::Journal.delete).to eq false
+            expect(Chewy::Journal.exists?).to eq false
+          end
+        end
+      end
+    end
+  end
+end