chewy 0.8.4 → 7.3.4

data/spec/chewy/index_spec.rb

@@ -2,9 +2,7 @@ require 'spec_helper'
 
 describe Chewy::Index do
   before do
-    stub_index(:dummies) do
-      define_type :dummy
-    end
+    stub_index(:dummies)
   end
 
   describe '.import', :orm do
@@ -12,33 +10,31 @@ describe Chewy::Index do
       stub_model(:city)
       stub_model(:country)
 
-      stub_index(:places) do
-        define_type City
-        define_type Country
+      stub_index(:cities) do
+        index_scope City
       end
-    end
 
-    let!(:cities) { 2.times.map { |i| City.create! id: i + 1 } }
-    let!(:countries) { 2.times.map { |i| Country.create! id: i + 1 } }
-
-    specify do
-      expect { PlacesIndex.import }.to update_index(PlacesIndex::City).and_reindex(cities)
-      expect { PlacesIndex.import }.to update_index(PlacesIndex::Country).and_reindex(countries)
+      stub_index(:countries) do
+        index_scope Country
+      end
     end
 
+    let!(:cities) { Array.new(2) { |i| City.create! id: i + 1 } }
+    let!(:countries) { Array.new(2) { |i| Country.create! id: i + 1 } }
+
     specify do
-      expect { PlacesIndex.import city: cities.first }.to update_index(PlacesIndex::City).and_reindex(cities.first).only
-      expect { PlacesIndex.import city: cities.first }.to update_index(PlacesIndex::Country).and_reindex(countries)
+      expect { CitiesIndex.import }.to update_index(CitiesIndex).and_reindex(cities)
+      expect { CountriesIndex.import }.to update_index(CountriesIndex).and_reindex(countries)
     end
 
     specify do
-      expect { PlacesIndex.import city: cities.first, country: countries.last }.to update_index(PlacesIndex::City).and_reindex(cities.first).only
-      expect { PlacesIndex.import city: cities.first, country: countries.last }.to update_index(PlacesIndex::Country).and_reindex(countries.last).only
+      expect { CitiesIndex.import cities.first }.to update_index(CitiesIndex).and_reindex(cities.first).only
+      expect { CountriesIndex.import countries.last }.to update_index(CountriesIndex).and_reindex(countries.last).only
     end
 
     specify do
-      expect(PlacesIndex.client).to receive(:bulk).with(hash_including(refresh: false)).twice
-      PlacesIndex.import city: cities.first, refresh: false
+      expect(CitiesIndex.client).to receive(:bulk).with(hash_including(refresh: false)).once
+      CitiesIndex.import cities.first, refresh: false
     end
   end
 
@@ -61,181 +57,213 @@ describe Chewy::Index do
     specify { expect(stub_const('DeveloperIndex', Class.new(Chewy::Index)).index_name).to eq('developer') }
     specify { expect(stub_const('DevelopersIndex', Class.new(Chewy::Index)).index_name).to eq('developers') }
 
+    specify do
+      expect(stub_const('DevelopersIndex', Class.new(Chewy::Index)).index_name(suffix: '')).to eq('developers')
+    end
+    specify do
+      expect(stub_const('DevelopersIndex', Class.new(Chewy::Index)).index_name(suffix: '2013')).to eq('developers_2013')
+    end
+    specify do
+      expect(stub_const('DevelopersIndex', Class.new(Chewy::Index)).index_name(prefix: '')).to eq('developers')
+    end
+    specify do
+      expect(stub_const('DevelopersIndex', Class.new(Chewy::Index)).index_name(prefix: 'test')).to eq('test_developers')
+    end
+
     context do
       before { allow(Chewy).to receive_messages(configuration: {prefix: 'testing'}) }
       specify { expect(DummiesIndex.index_name).to eq('testing_dummies') }
       specify { expect(stub_index(:dummies) { index_name :users }.index_name).to eq('testing_users') }
+      specify { expect(stub_index(:dummies) { index_name :users }.index_name(prefix: '')).to eq('users') }
     end
   end
 
-  describe '.default_prefix' do
+  describe '.derivable_name' do
+    specify { expect(Class.new(Chewy::Index).derivable_name).to be_nil }
+    specify { expect(stub_index(:places).derivable_name).to eq('places') }
+    specify { expect(stub_index('namespace/places').derivable_name).to eq('namespace/places') }
+  end
+
+  describe '.prefix' do
     before { allow(Chewy).to receive_messages(configuration: {prefix: 'testing'}) }
-    specify { expect(Class.new(Chewy::Index).default_prefix).to eq('testing') }
+    specify { expect(Class.new(Chewy::Index).prefix).to eq('testing') }
   end
 
-  describe '.define_type' do
-    specify { expect(DummiesIndex.type_hash['dummy']).to eq(DummiesIndex::Dummy) }
+  describe '.index_scope' do
+    specify { expect(DummiesIndex.adapter.name).to eq('Default') }
 
     context do
-      before { stub_index(:dummies) { define_type :dummy, name: :borogoves } }
-      specify { expect(DummiesIndex.type_hash['borogoves']).to eq(DummiesIndex::Borogoves) }
+      before { stub_index(:dummies) { index_scope :dummy, name: :borogoves } }
+      specify { expect(DummiesIndex.adapter.name).to eq('Borogoves') }
     end
 
     context do
       before { stub_class(:city) }
-      before { stub_index(:dummies) { define_type City, name: :country } }
-      specify { expect(DummiesIndex.type_hash['country']).to eq(DummiesIndex::Country) }
+      before { stub_index(:dummies) { index_scope City, name: :country } }
+      specify { expect(DummiesIndex.adapter.name).to eq('Country') }
     end
 
     context do
       before { stub_class('City') }
-      before { stub_class('City::District', City) }
+      before { stub_class('Country') }
 
       specify do
-        expect {
+        expect do
           Kernel.eval <<-DUMMY_CITY_INDEX
-            class DummyCityIndex < Chewy::Index
-              define_type City
-              define_type City::District
+            class DummyCityIndex2 < Chewy::Index
+              index_scope City
+              index_scope Country
             end
           DUMMY_CITY_INDEX
-        }.not_to raise_error
-      end
+        end.to raise_error(/Index scope is already defined/)
 
-      specify do
-        expect {
+        expect do
           Kernel.eval <<-DUMMY_CITY_INDEX
             class DummyCityIndex2 < Chewy::Index
-              define_type City
-              define_type City::Nothing
+              index_scope City::Nothing
             end
           DUMMY_CITY_INDEX
-        }.to raise_error(NameError)
+        end.to raise_error(NameError)
       end
     end
-
-    context 'type methods should be deprecated and can\'t redefine existion ones' do
-      before do
-        stub_index(:places) do
-          def self.city
-          end
-          define_type :city
-          define_type :country
-        end
-      end
-
-      specify { expect(PlacesIndex.city).to be_nil }
-      specify { expect(PlacesIndex.country).to be < Chewy::Type }
-    end
-  end
-
-  describe '.type_hash' do
-    specify { expect(DummiesIndex.type_hash['dummy']).to eq(DummiesIndex::Dummy) }
-    specify { expect(DummiesIndex.type_hash).to have_key 'dummy' }
-    specify { expect(DummiesIndex.type_hash['dummy']).to be < Chewy::Type }
-    specify { expect(DummiesIndex.type_hash['dummy'].type_name).to eq('dummy') }
-  end
-
-  specify { expect(DummiesIndex.type_names).to eq(DummiesIndex.type_hash.keys) }
-
-  describe '.types' do
-    specify { expect(DummiesIndex.types).to eq(DummiesIndex.type_hash.values) }
-    specify { expect(DummiesIndex.types(:dummy)).to be_a Chewy::Query }
-    specify { expect(DummiesIndex.types(:user)).to be_a Chewy::Query }
   end
 
   describe '.settings' do
     before do
       allow(Chewy).to receive_messages(config: Chewy::Config.send(:new))
 
-      Chewy.analyzer :name, filter: ['lowercase', 'icu_folding', 'names_nysiis']
+      Chewy.analyzer :name, filter: %w[lowercase icu_folding names_nysiis]
       Chewy.analyzer :phone, tokenizer: 'ngram', char_filter: ['phone']
       Chewy.tokenizer :ngram, type: 'nGram', min_gram: 3, max_gram: 3
       Chewy.char_filter :phone, type: 'pattern_replace', pattern: '[^\d]', replacement: ''
       Chewy.filter :names_nysiis, type: 'phonetic', encoder: 'nysiis', replace: false
     end
 
-    let(:documents) { stub_index(:documents) { settings analysis: {analyzer: [:name, :phone, {sorted: {option: :baz}}]} } }
+    let(:documents) do
+      stub_index(:documents) do
+        settings analysis: {analyzer: [:name, :phone, {sorted: {option: :baz}}]}
+      end
+    end
 
-    specify { expect { documents.settings_hash }.to_not change(documents._settings, :inspect)  }
-    specify { expect(documents.settings_hash).to eq({settings: {analysis: {
-      analyzer: {name: {filter: ['lowercase', 'icu_folding', 'names_nysiis']},
-                 phone: {tokenizer: 'ngram', char_filter: ['phone']},
-                 sorted: {option: :baz}},
-      tokenizer: {ngram: {type: 'nGram', min_gram: 3, max_gram: 3}},
-      char_filter: {phone: {type: 'pattern_replace', pattern: '[^\d]', replacement: ''}},
-      filter: {names_nysiis: {type: 'phonetic', encoder: 'nysiis', replace: false}}
-    }}}) }
+    specify { expect { documents.settings_hash }.to_not change(documents._settings, :inspect) }
+    specify do
+      expect(documents.settings_hash).to eq(settings: {analysis: {
+        analyzer: {name: {filter: %w[lowercase icu_folding names_nysiis]},
+                   phone: {tokenizer: 'ngram', char_filter: ['phone']},
+                   sorted: {option: :baz}},
+        tokenizer: {ngram: {type: 'nGram', min_gram: 3, max_gram: 3}},
+        char_filter: {phone: {type: 'pattern_replace', pattern: '[^\d]', replacement: ''}},
+        filter: {names_nysiis: {type: 'phonetic', encoder: 'nysiis', replace: false}}
+      }})
+    end
   end
 
   describe '.scopes' do
     before do
       stub_index(:places) do
-        def self.by_rating
-        end
+        def self.by_rating; end
 
-        def self.by_name
+        def self.colors(*colors)
+          filter(terms: {colors: colors.flatten(1).map(&:to_s)})
         end
 
-        define_type :city do
-          def self.by_id
-          end
-        end
+        def self.by_id; end
+        field :colors
       end
     end
 
     specify { expect(described_class.scopes).to eq([]) }
-    specify { expect(PlacesIndex.scopes).to match_array([:by_rating, :by_name]) }
-  end
+    specify { expect(PlacesIndex.scopes).to match_array(%i[by_rating colors by_id]) }
+
+    context do
+      before do
+        Chewy.massacre
+        PlacesIndex.import!(
+          double(colors: ['red']),
+          double(colors: %w[red green]),
+          double(colors: %w[green yellow])
+        )
+      end
+
+      specify do
+        expect(PlacesIndex.colors(:green).map(&:colors))
+          .to contain_exactly(%w[red green], %w[green yellow])
+      end
 
-  describe '.build_index_name' do
-    specify { expect(stub_const('DevelopersIndex', Class.new(Chewy::Index)).build_index_name(suffix: '')).to eq('developers') }
-    specify { expect(stub_const('DevelopersIndex', Class.new(Chewy::Index)).build_index_name(suffix: '2013')).to eq('developers_2013') }
-    specify { expect(stub_const('DevelopersIndex', Class.new(Chewy::Index)).build_index_name(prefix: '')).to eq('developers') }
-    specify { expect(stub_const('DevelopersIndex', Class.new(Chewy::Index)).build_index_name(prefix: 'test')).to eq('test_developers') }
-    specify { expect(stub_const('DevelopersIndex', Class.new(Chewy::Index)).build_index_name(:users, prefix: 'test', suffix: '2013')).to eq('test_users_2013') }
+      specify do
+        expect(PlacesIndex.colors(:green).map(&:colors))
+          .to contain_exactly(%w[red green], %w[green yellow])
+      end
+    end
   end
 
   describe '.settings_hash' do
     before { allow(Chewy).to receive_messages(config: Chewy::Config.send(:new)) }
 
     specify { expect(stub_index(:documents).settings_hash).to eq({}) }
-    specify { expect(stub_index(:documents) { settings number_of_shards: 1 }.settings_hash).to eq({settings: {number_of_shards: 1}}) }
+    specify do
+      expect(stub_index(:documents) do
+               settings number_of_shards: 1
+             end.settings_hash).to eq(settings: {number_of_shards: 1})
+    end
   end
 
   describe '.mappings_hash' do
     specify { expect(stub_index(:documents).mappings_hash).to eq({}) }
-    specify { expect(stub_index(:documents) { define_type :document }.mappings_hash).to eq({}) }
-    specify { expect(stub_index(:documents) do
-      define_type :document do
-        field :name, type: 'string'
-      end
-    end.mappings_hash).to eq({mappings: {document: {properties: {name: {type: 'string'}}}}}) }
-    specify { expect(stub_index(:documents) do
-      define_type :document do
-        field :name, type: 'string'
-      end
-      define_type :document2 do
-        field :name, type: 'string'
-      end
-    end.mappings_hash[:mappings].keys).to match_array([:document, :document2]) }
+    specify { expect(stub_index(:documents) { index_scope :document }.mappings_hash).to eq({}) }
+    specify do
+      expect(stub_index(:documents) do
+               field :date, type: 'date'
+             end.mappings_hash).to eq(mappings: {properties: {date: {type: 'date'}}})
+    end
   end
 
-  describe '.index_params' do
+  describe '.specification_hash' do
     before { allow(Chewy).to receive_messages(config: Chewy::Config.send(:new)) }
 
-    specify { expect(stub_index(:documents).index_params).to eq({}) }
-    specify { expect(stub_index(:documents) { settings number_of_shards: 1 }.index_params.keys).to eq([:settings]) }
-    specify { expect(stub_index(:documents) do
-      define_type :document do
-        field :name, type: 'string'
+    specify { expect(stub_index(:documents).specification_hash).to eq({}) }
+    specify do
+      expect(stub_index(:documents) do
+               settings number_of_shards: 1
+             end.specification_hash.keys).to eq([:settings])
+    end
+    specify do
+      expect(stub_index(:documents) do
+               field :name
+             end.specification_hash.keys).to eq([:mappings])
+    end
+    specify do
+      expect(stub_index(:documents) do
+               settings number_of_shards: 1
+               field :name
+             end.specification_hash.keys).to match_array(%i[mappings settings])
+    end
+  end
+
+  describe '.specification' do
+    subject { stub_index(:documents) }
+    specify { expect(subject.specification).to be_a(Chewy::Index::Specification) }
+    specify { expect(subject.specification).to equal(subject.specification) }
+  end
+
+  context 'index call inside index', :orm do
+    before do
+      stub_index(:cities) do
+        field :country_name, value: (lambda do |city|
+          CountriesIndex.filter(term: {_id: city.country_id}).first.name
+        end)
       end
-    end.index_params.keys).to eq([:mappings]) }
-    specify { expect(stub_index(:documents) do
-      settings number_of_shards: 1
-      define_type :document do
-        field :name, type: 'string'
+
+      stub_index(:countries) do
+        field :name
       end
-    end.index_params.keys).to match_array([:mappings, :settings]) }
+
+      CountriesIndex.import!(double(id: 1, name: 'Country'))
+    end
+
+    specify do
+      expect { CitiesIndex.import!(double(country_id: 1)) }
+        .to update_index(CitiesIndex).and_reindex(country_name: 'Country')
+    end
   end
 end

data/spec/chewy/journal_spec.rb

@@ -0,0 +1,223 @@
+require 'spec_helper'
+
+describe Chewy::Journal do
+  context 'journaling', :orm do
+    ['', 'namespace/'].each do |namespace|
+      context namespace.present? ? 'with namespace' : 'without namespace' do
+        before do
+          stub_model(:city) do
+            update_index "#{namespace}cities", :self
+          end
+          stub_model(:country) do
+            update_index "#{namespace}countries", :self
+          end
+
+          stub_index("#{namespace}cities") do
+            index_scope City
+            default_import_options journal: true
+          end
+          stub_index("#{namespace}countries") do
+            index_scope Country
+            default_import_options journal: true
+          end
+
+          Chewy.massacre
+          Chewy.settings[:prefix] = 'some_prefix'
+          Timecop.freeze(time)
+        end
+
+        after do
+          Chewy.settings[:prefix] = nil
+          Timecop.return
+        end
+
+        let(:time) { Time.now }
+        let(:import_time) { time + 1 }
+        let(:update_time) { time + 2 }
+        let(:destroy_time) { time + 3 }
+
+        def timestamp(time)
+          time.to_i
+        end
+
+        specify do
+          cities_index = namespace.present? ? Namespace::CitiesIndex : CitiesIndex
+          countries_index = namespace.present? ? Namespace::CountriesIndex : CountriesIndex
+          Chewy.strategy(:urgent) do
+            cities = Array.new(2) { |i| City.create!(id: i + 1) }
+            countries = Array.new(2) { |i| Country.create!(id: i + 1) }
+            Country.create!(id: 3)
+
+            Timecop.freeze(import_time)
+
+            cities_index.import
+            countries_index.import
+
+            expect(Chewy::Stash::Journal.exists?).to eq true
+
+            Timecop.freeze(update_time)
+            cities.first.update!(name: 'Supername')
+
+            Timecop.freeze(destroy_time)
+            countries.last.destroy
+
+            journal_entries = Chewy::Stash::Journal.order(:created_at).hits.map { |r| r['_source'] }
+            expected_journal = [
+              {
+                'index_name' => "#{namespace}cities",
+                'action' => 'index',
+                'references' => ['1'].map(&Base64.method(:encode64)),
+                'created_at' => time.utc.as_json
+              },
+              {
+                'index_name' => "#{namespace}cities",
+                'action' => 'index',
+                'references' => ['2'].map(&Base64.method(:encode64)),
+                'created_at' => time.utc.as_json
+              },
+              {
+                'index_name' => "#{namespace}countries",
+                'action' => 'index',
+                'references' => ['1'].map(&Base64.method(:encode64)),
+                'created_at' => time.utc.as_json
+              },
+              {
+                'index_name' => "#{namespace}countries",
+                'action' => 'index',
+                'references' => ['2'].map(&Base64.method(:encode64)),
+                'created_at' => time.utc.as_json
+              },
+              {
+                'index_name' => "#{namespace}countries",
+                'action' => 'index',
+                'references' => ['3'].map(&Base64.method(:encode64)),
+                'created_at' => time.utc.as_json
+              },
+              {
+                'index_name' => "#{namespace}cities",
+                'action' => 'index',
+                'references' => %w[1 2].map(&Base64.method(:encode64)),
+                'created_at' => import_time.utc.as_json
+              },
+              {
+                'index_name' => "#{namespace}countries",
+                'action' => 'index',
+                'references' => %w[1 2 3].map(&Base64.method(:encode64)),
+                'created_at' => import_time.utc.as_json
+              },
+              {
+                'index_name' => "#{namespace}cities",
+                'action' => 'index',
+                'references' => ['1'].map(&Base64.method(:encode64)),
+                'created_at' => update_time.utc.as_json
+              },
+              {
+                'index_name' => "#{namespace}countries",
+                'action' => 'delete',
+                'references' => ['2'].map(&Base64.method(:encode64)),
+                'created_at' => destroy_time.utc.as_json
+              }
+            ]
+
+            expect(Chewy::Stash::Journal.count).to eq 9
+            expect(journal_entries).to eq expected_journal
+
+            journal_entries = Chewy::Stash::Journal.entries(import_time - 1)
+            expect(journal_entries.size).to eq 4
+
+            # simulate lost data
+            Chewy.client.delete(index: "#{Chewy.settings[:prefix]}_cities", id: 1, refresh: true)
+            expect(cities_index.count).to eq 1
+
+            described_class.new.apply(time)
+            expect(cities_index.count).to eq 2
+
+            clean_response = described_class.new.clean(import_time)
+            expect(clean_response['deleted'] || clean_response['_indices']['_all']['deleted']).to eq 7
+            Chewy.client.indices.refresh
+            expect(Chewy::Stash::Journal.count).to eq 2
+
+            Timecop.return
+          end
+        end
+      end
+    end
+  end
+
+  context do
+    before { Chewy.massacre }
+    before do
+      stub_model(:city) do
+        update_index 'cities', :self
+      end
+      stub_model(:country) do
+        update_index 'countries', :self
+      end
+
+      stub_index(:cities) do
+        index_scope City
+        default_import_options journal: true
+      end
+      stub_index(:countries) do
+        index_scope Country
+        default_import_options journal: true
+      end
+    end
+
+    describe '#apply' do
+      specify { expect(described_class.new(CitiesIndex).apply(2.minutes.ago)).to eq(0) }
+
+      context 'with an index filter' do
+        let(:time) { Time.now }
+
+        before { Timecop.freeze(time) }
+        after { Timecop.return }
+
+        specify do
+          Chewy.strategy(:urgent) do
+            Array.new(2) { |i| City.create!(id: i + 1) }
+            Array.new(2) { |i| Country.create!(id: i + 1) }
+
+            # simulate lost data
+            Chewy.client.delete(index: 'cities', id: 1, refresh: true)
+            Chewy.client.delete(index: 'countries', id: 1, refresh: true)
+            expect(CitiesIndex.all.to_a.length).to eq 1
+            expect(CountriesIndex.all.to_a.length).to eq 1
+
+            # Replay on specific index
+            expect(described_class.new(CitiesIndex).apply(time)).to eq(2)
+            expect(CitiesIndex.all.to_a.length).to eq 2
+            expect(CountriesIndex.all.to_a.length).to eq 1
+
+            # Replay on both
+            Chewy.client.delete(index: 'cities', id: 1, refresh: true)
+            expect(CitiesIndex.all.to_a.length).to eq 1
+            expect(described_class.new(CitiesIndex, CountriesIndex).apply(time)).to eq(4)
+            expect(CitiesIndex.all.to_a.length).to eq 2
+            expect(CountriesIndex.all.to_a.length).to eq 2
+          end
+        end
+      end
+
+      context 'when order is not preserved' do
+        let(:time) { Time.now }
+
+        it 'paginates properly through all items' do
+          Chewy.strategy(:urgent) do
+            Timecop.travel(time + 1.minute) { City.create!(id: 2) }
+            Timecop.travel(time + 3.minute) { City.create!(id: 4) }
+            Timecop.travel(time + 2.minute) { City.create!(id: 1) }
+            Timecop.travel(time + 4.minute) { City.create!(id: 3) }
+          end
+
+          CitiesIndex.purge!
+          expect(CitiesIndex.all.to_a.length).to eq 0
+
+          # Replay on specific index
+          expect(described_class.new(CitiesIndex).apply(time, fetch_limit: 2)).to eq(4)
+          expect(CitiesIndex.all.to_a.map(&:id).sort).to eq([1, 2, 3, 4])
+        end
+      end
+    end
+  end
+end