chewy 5.2.0 → 6.0.0

data/lib/chewy/type/import.rb

@@ -67,6 +67,7 @@ module Chewy
         # @option options [String] suffix an index name suffix, used for zero-downtime reset mostly, no suffix by default
         # @option options [Integer] bulk_size bulk API chunk size in bytes; if passed, the request is performed several times for each chunk, empty by default
         # @option options [Integer] batch_size passed to the adapter import method, used to split imported objects in chunks, 1000 by default
+        # @option options [Boolean] direct_import skips object reloading in ORM adapter, `false` by default
         # @option options [true, false] journal enables imported objects journaling, false by default
         # @option options [Array<Symbol, String>] update_fields list of fields for the partial import, empty by default
         # @option options [true, false] update_failover enables full objects reimport in cases of partial update errors, `true` by default

data/lib/chewy/type/syncer.rb

@@ -2,7 +2,7 @@ module Chewy
   class Type
     # This class is able to find missing and outdated documents in the ES
     # comparing ids from the data source and the ES index. Also, if `outdated_sync_field`
-    # existss in the index definition, it performs comparison of this field
+    # exists in the index definition, it performs comparison of this field
     # values for each source object and corresponding ES document. Usually,
     # this field is `updated_at` and if its value in the source is not equal
     # to the value in the index - this means that this document outdated and
@@ -205,10 +205,9 @@ module Chewy
         return @outdated_sync_field_type if instance_variable_defined?(:@outdated_sync_field_type)
         return unless @type.outdated_sync_field
 
-        mappings = @type.client.indices.get_mapping(
-          index: @type.index_name,
-          type: @type.type_name
-        ).values.first.fetch('mappings', {})
+        args = {index: @type.index_name, type: @type.type_name}
+        args[:include_type_name] = true if Runtime.version >= '6.7.0'
+        mappings = @type.client.indices.get_mapping(**args).values.first.fetch('mappings', {})
 
         @outdated_sync_field_type = mappings
           .fetch(@type.type_name, {})

data/lib/chewy/type/witchcraft.rb

@@ -79,7 +79,9 @@ module Chewy
           if field.children.present? && !field.multi_field?
             <<-RUBY
               (result#{nesting} = #{fetcher}
-              if result#{nesting}.respond_to?(:to_ary)
+              if result#{nesting}.nil?
+                nil
+              elsif result#{nesting}.respond_to?(:to_ary)
                 result#{nesting}.map do |object#{nesting}|
                   #{composed_values(field, nesting)}
                 end

data/lib/chewy/type/wrapper.rb

@@ -49,6 +49,8 @@ module Chewy
         m = method.to_s
         if (name = highlight_name(m))
           highlight(name)
+        elsif (name = highlight_names(m))
+          highlights(name)
         elsif @attributes.key?(m)
           @attributes[m]
         elsif attribute_defined?(m)
@@ -60,7 +62,7 @@ module Chewy
 
       def respond_to_missing?(method, include_private = false)
         m = method.to_s
-        highlight_name(m) || @attributes.key?(m) || attribute_defined?(m) || super
+        highlight_name(m) || highlight_names(m) || @attributes.key?(m) || attribute_defined?(m) || super
       end
 
     private
@@ -69,6 +71,10 @@ module Chewy
         method.sub(/_highlight\z/, '') if method.end_with?('_highlight')
       end
 
+      def highlight_names(method)
+        method.sub(/_highlights\z/, '') if method.end_with?('_highlights')
+      end
+
       def attribute_defined?(attribute)
         self.class.root && self.class.root.children.find { |a| a.name.to_s == attribute }.present?
       end
@@ -77,6 +83,10 @@ module Chewy
         _data['highlight'][attribute].first if highlight?(attribute)
       end
 
+      def highlights(attribute)
+        _data['highlight'][attribute] if highlight?(attribute)
+      end
+
       def highlight?(attribute)
         _data.key?('highlight') && _data['highlight'].key?(attribute)
       end

data/lib/chewy/version.rb

@@ -1,3 +1,3 @@
 module Chewy
-  VERSION = '5.2.0'.freeze
+  VERSION = '6.0.0'.freeze
 end

data/migration_guide.md

@@ -0,0 +1,18 @@
+# Migration guide
+
+## Chewy 5 / ES 5 to Chewy 6 / ES 6
+
+In order to upgrade chewy5/ES5 to chewy6/ES6 in the most seamless manner you have to:
+* Upgrade to the latest 5.x stable releases: Chewy 5.2, ES 5.6
+* [Migrate any multi-typed indexes into single-typed](https://www.elastic.co/guide/en/elasticsearch/reference/6.8/removal-of-types.html)
+  * Using [multi-index queries](https://github.com/toptal/chewy/pull/657) could be helpful
+  * Parent/Child [relationship is deprecated](https://www.elastic.co/guide/en/elasticsearch/reference/6.8/removal-of-types.html#parent-child-mapping-types) in favor of the [join field](https://www.elastic.co/guide/en/elasticsearch/reference/6.8/parent-join.html) 
+* Handle deprecation of `string` type & `not_analyzed` value for the `index` mapping parameter:
+  * replace fields with `{ type: 'string', index: 'not_analyzed'}` by `{type: 'keyword'}`
+  * replace fields with `{ type: 'string', index: 'analyzed'}` by `{type: 'text'}`
+* `PathHierarchy` tokenizer' param `delimiter` now accepts only one argument, [others should be replaced by character filter ](https://discuss.elastic.co/t/multichar-delimiter-in-path-hierarchy-tokenizer/16203)
+* Make sure you don't use any other of the [deprecated ES5 features](https://www.elastic.co/guide/en/elasticsearch/reference/6.8/breaking-changes-6.0.html)
+* Run your test suite on ES6
+* Run manual tests on ES6
+* Perform a [rolling upgrade](https://www.elastic.co/guide/en/elasticsearch/reference/6.8/rolling-upgrades.html) of Elasticsearch
+* Upgrade to Chewy 6

data/spec/chewy/config_spec.rb

@@ -6,9 +6,6 @@ describe Chewy::Config do
   its(:logger) { should be_nil }
   its(:transport_logger) { should be_nil }
   its(:transport_logger) { should be_nil }
-  its(:query_mode) { should == :must }
-  its(:filter_mode) { should == :and }
-  its(:post_filter_mode) { should be_nil }
   its(:root_strategy) { should == :base }
   its(:request_strategy) { should == :atomic }
   its(:use_after_commit_callbacks) { should == true }
@@ -54,24 +51,6 @@ describe Chewy::Config do
     end
   end
 
-  describe '#search_class=' do
-    specify do
-      expect { subject.search_class = Chewy::Query }
-        .to change { subject.search_class }
-        .from(be < Chewy::Search::Request)
-        .to(be < Chewy::Query)
-    end
-
-    context do
-      before { hide_const('Kaminari') }
-
-      specify do
-        expect(subject.search_class.included_modules)
-          .to include(Chewy::Search::Pagination::WillPaginate)
-      end
-    end
-  end
-
   describe '#search_class' do
     context 'nothing is defined' do
       before do

data/spec/chewy/index/actions_spec.rb

@@ -281,6 +281,18 @@ describe Chewy::Index::Actions do
 
     specify { expect(CitiesIndex.import).to eq(true) }
 
+    specify 'with an empty array' do
+      expect(CitiesIndex).not_to receive(:exists?)
+      expect(CitiesIndex).not_to receive(:create!)
+      expect(CitiesIndex.import([])).to eq(true)
+    end
+
+    specify 'with an empty relation' do
+      expect(CitiesIndex).not_to receive(:exists?)
+      expect(CitiesIndex).not_to receive(:create!)
+      expect(CitiesIndex.import(City.where('1 = 2'))).to eq(true)
+    end
+
     context do
       before do
         stub_index(:cities) do
@@ -305,6 +317,18 @@ describe Chewy::Index::Actions do
 
     specify { expect(CitiesIndex.import!).to eq(true) }
 
+    specify 'with an empty array' do
+      expect(CitiesIndex).not_to receive(:exists?)
+      expect(CitiesIndex).not_to receive(:create!)
+      expect(CitiesIndex.import!([])).to eq(true)
+    end
+
+    specify 'with an empty relation' do
+      expect(CitiesIndex).not_to receive(:exists?)
+      expect(CitiesIndex).not_to receive(:create!)
+      expect(CitiesIndex.import!(City.where('1 = 2'))).to eq(true)
+    end
+
     context do
       before do
         stub_index(:cities) do

data/spec/chewy/index_spec.rb

@@ -12,8 +12,11 @@ describe Chewy::Index do
       stub_model(:city)
       stub_model(:country)
 
-      stub_index(:places) do
+      stub_index(:cities) do
         define_type City
+      end
+
+      stub_index(:countries) do
         define_type Country
       end
     end
@@ -22,23 +25,23 @@ describe Chewy::Index do
     let!(:countries) { Array.new(2) { |i| Country.create! id: i + 1 } }
 
     specify do
-      expect { PlacesIndex.import }.to update_index(PlacesIndex::City).and_reindex(cities)
-      expect { PlacesIndex.import }.to update_index(PlacesIndex::Country).and_reindex(countries)
+      expect { CitiesIndex.import }.to update_index(CitiesIndex).and_reindex(cities)
+      expect { CountriesIndex.import }.to update_index(CountriesIndex).and_reindex(countries)
     end
 
     specify do
-      expect { PlacesIndex.import city: cities.first }.to update_index(PlacesIndex::City).and_reindex(cities.first).only
-      expect { PlacesIndex.import city: cities.first }.to update_index(PlacesIndex::Country).and_reindex(countries)
+      expect { CitiesIndex.import city: cities.first }.to update_index(CitiesIndex).and_reindex(cities.first).only
+      expect { CountriesIndex.import city: cities.first }.to update_index(CountriesIndex).and_reindex(countries)
     end
 
     specify do
-      expect { PlacesIndex.import city: cities.first, country: countries.last }.to update_index(PlacesIndex::City).and_reindex(cities.first).only
-      expect { PlacesIndex.import city: cities.first, country: countries.last }.to update_index(PlacesIndex::Country).and_reindex(countries.last).only
+      expect { CitiesIndex.import city: cities.first, country: countries.last }.to update_index(CitiesIndex).and_reindex(cities.first).only
+      expect { CountriesIndex.import city: cities.first, country: countries.last }.to update_index(CountriesIndex).and_reindex(countries.last).only
     end
 
     specify do
-      expect(PlacesIndex.client).to receive(:bulk).with(hash_including(refresh: false)).twice
-      PlacesIndex.import city: cities.first, refresh: false
+      expect(CitiesIndex.client).to receive(:bulk).with(hash_including(refresh: false)).once
+      CitiesIndex.import city: cities.first, refresh: false
     end
   end
 
@@ -101,43 +104,27 @@ describe Chewy::Index do
 
     context do
       before { stub_class('City') }
-      before { stub_class('City::District', City) }
+      before { stub_class('Country') }
 
       specify do
         expect do
           Kernel.eval <<-DUMMY_CITY_INDEX
-            class DummyCityIndex < Chewy::Index
+            class DummyCityIndex2 < Chewy::Index
               define_type City
-              define_type City::District
+              define_type Country
             end
           DUMMY_CITY_INDEX
-        end.not_to raise_error
-      end
+        end.to raise_error(/Multiple types are deprecated/)
 
-      specify do
         expect do
           Kernel.eval <<-DUMMY_CITY_INDEX
             class DummyCityIndex2 < Chewy::Index
-              define_type City
               define_type City::Nothing
             end
           DUMMY_CITY_INDEX
         end.to raise_error(NameError)
       end
     end
-
-    context 'type methods should be deprecated and can\'t redefine existing ones' do
-      before do
-        stub_index(:places) do
-          def self.city; end
-          define_type :city
-          define_type :country
-        end
-      end
-
-      specify { expect(PlacesIndex.city).to be_nil }
-      specify { expect(PlacesIndex::Country).to be < Chewy::Type }
-    end
   end
 
   describe '.type_hash' do
@@ -248,16 +235,6 @@ describe Chewy::Index do
                end
              end.mappings_hash).to eq(mappings: {document: {properties: {date: {type: 'date'}}}})
     end
-    specify do
-      expect(stub_index(:documents) do
-               define_type :document do
-                 field :name
-               end
-               define_type :document2 do
-                 field :name
-               end
-             end.mappings_hash[:mappings].keys).to match_array(%i[document document2])
-    end
   end
 
   describe '.specification_hash' do

data/spec/chewy/journal_spec.rb

@@ -6,16 +6,18 @@ describe Chewy::Journal do
       context namespace.present? ? 'with namespace' : 'without namespace' do
         before do
           stub_model(:city) do
-            update_index "#{namespace}places#city", :self
+            update_index "#{namespace}cities", :self
           end
           stub_model(:country) do
-            update_index "#{namespace}places#country", :self
+            update_index "#{namespace}countries", :self
           end
 
-          stub_index("#{namespace}places") do
+          stub_index("#{namespace}cities") do
             define_type City do
               default_import_options journal: true
             end
+          end
+          stub_index("#{namespace}countries") do
             define_type Country do
               default_import_options journal: true
             end
@@ -41,7 +43,8 @@ describe Chewy::Journal do
         end
 
         specify do
-          places_index = namespace.present? ? Namespace::PlacesIndex : PlacesIndex
+          cities_index = namespace.present? ? Namespace::CitiesIndex : CitiesIndex
+          countries_index = namespace.present? ? Namespace::CountriesIndex : CountriesIndex
           Chewy.strategy(:urgent) do
             cities = Array.new(2) { |i| City.create!(id: i + 1) }
             countries = Array.new(2) { |i| Country.create!(id: i + 1) }
@@ -49,7 +52,8 @@ describe Chewy::Journal do
 
             Timecop.freeze(import_time)
 
-            places_index.import
+            cities_index.import
+            countries_index.import
 
             expect(Chewy::Stash::Journal.exists?).to eq true
 
@@ -62,63 +66,63 @@ describe Chewy::Journal do
             journal_entries = Chewy::Stash::Journal.order(:created_at).hits.map { |r| r['_source'] }
             expected_journal = [
               {
-                'index_name' => "#{namespace}places",
+                'index_name' => "#{namespace}cities",
                 'type_name' => 'city',
                 'action' => 'index',
                 'references' => ['1'].map(&Base64.method(:encode64)),
                 'created_at' => time.utc.as_json
               },
               {
-                'index_name' => "#{namespace}places",
+                'index_name' => "#{namespace}cities",
                 'type_name' => 'city',
                 'action' => 'index',
                 'references' => ['2'].map(&Base64.method(:encode64)),
                 'created_at' => time.utc.as_json
               },
               {
-                'index_name' => "#{namespace}places",
+                'index_name' => "#{namespace}countries",
                 'type_name' => 'country',
                 'action' => 'index',
                 'references' => ['1'].map(&Base64.method(:encode64)),
                 'created_at' => time.utc.as_json
               },
               {
-                'index_name' => "#{namespace}places",
+                'index_name' => "#{namespace}countries",
                 'type_name' => 'country',
                 'action' => 'index',
                 'references' => ['2'].map(&Base64.method(:encode64)),
                 'created_at' => time.utc.as_json
               },
               {
-                'index_name' => "#{namespace}places",
+                'index_name' => "#{namespace}countries",
                 'type_name' => 'country',
                 'action' => 'index',
                 'references' => ['3'].map(&Base64.method(:encode64)),
                 'created_at' => time.utc.as_json
               },
               {
-                'index_name' => "#{namespace}places",
+                'index_name' => "#{namespace}cities",
                 'type_name' => 'city',
                 'action' => 'index',
                 'references' => %w[1 2].map(&Base64.method(:encode64)),
                 'created_at' => import_time.utc.as_json
               },
               {
-                'index_name' => "#{namespace}places",
+                'index_name' => "#{namespace}countries",
                 'type_name' => 'country',
                 'action' => 'index',
                 'references' => %w[1 2 3].map(&Base64.method(:encode64)),
                 'created_at' => import_time.utc.as_json
               },
               {
-                'index_name' => "#{namespace}places",
+                'index_name' => "#{namespace}cities",
                 'type_name' => 'city',
                 'action' => 'index',
                 'references' => ['1'].map(&Base64.method(:encode64)),
                 'created_at' => update_time.utc.as_json
               },
               {
-                'index_name' => "#{namespace}places",
+                'index_name' => "#{namespace}countries",
                 'type_name' => 'country',
                 'action' => 'delete',
                 'references' => ['2'].map(&Base64.method(:encode64)),
@@ -133,11 +137,11 @@ describe Chewy::Journal do
             expect(journal_entries.size).to eq 4
 
             # simulate lost data
-            Chewy.client.delete(index: "#{Chewy.settings[:prefix]}_places", type: 'city', id: 1, refresh: true)
-            expect(places_index::City.count).to eq 1
+            Chewy.client.delete(index: "#{Chewy.settings[:prefix]}_cities", type: 'city', id: 1, refresh: true)
+            expect(cities_index.count).to eq 1
 
             described_class.new.apply(time)
-            expect(places_index::City.count).to eq 2
+            expect(cities_index.count).to eq 2
 
             clean_response = described_class.new.clean(import_time)
             expect(clean_response['deleted'] || clean_response['_indices']['_all']['deleted']).to eq 7

data/spec/chewy/minitest/search_index_receiver_spec.rb

@@ -29,7 +29,9 @@ describe :search_index_receiver do
       define_type :fizz do
         root value: ->(_o) { {} }
       end
+    end
 
+    stub_index(:dummies2) do
       define_type :buzz do
         root value: ->(_o) { {} }
       end
@@ -39,15 +41,15 @@ describe :search_index_receiver do
   context 'catch' do
     specify 'archives more than one type' do
       receiver.catch search_request(2), DummiesIndex::Fizz
-      receiver.catch search_request(3), DummiesIndex::Buzz
-      expect(receiver.indexes.keys).to match_array([DummiesIndex::Fizz, DummiesIndex::Buzz])
+      receiver.catch search_request(3), Dummies2Index::Buzz
+      expect(receiver.indexes.keys).to match_array([DummiesIndex::Fizz, Dummies2Index::Buzz])
     end
   end
 
   context 'indexes_for' do
     before do
       receiver.catch search_request(2), DummiesIndex::Fizz
-      receiver.catch search_request(3), DummiesIndex::Buzz
+      receiver.catch search_request(3), Dummies2Index::Buzz
     end
 
     specify 'returns indexes for a specific type' do
@@ -56,7 +58,7 @@ describe :search_index_receiver do
 
     specify 'returns only indexes for all types' do
       index_responses = receiver.indexes
-      expect(index_responses.keys).to match_array([DummiesIndex::Fizz, DummiesIndex::Buzz])
+      expect(index_responses.keys).to match_array([DummiesIndex::Fizz, Dummies2Index::Buzz])
       expect(parse_request(index_responses.values.flatten)).to match_array([1, 2, 1, 2, 3])
     end
   end
@@ -64,16 +66,16 @@ describe :search_index_receiver do
   context 'deletes_for' do
     before do
       receiver.catch search_request(2, verb: :delete), DummiesIndex::Fizz
-      receiver.catch search_request(3, verb: :delete), DummiesIndex::Buzz
+      receiver.catch search_request(3, verb: :delete), Dummies2Index::Buzz
     end
 
     specify 'returns deletes for a specific type' do
-      expect(receiver.deletes_for(DummiesIndex::Buzz)).to match_array([1, 2, 3])
+      expect(receiver.deletes_for(Dummies2Index::Buzz)).to match_array([1, 2, 3])
     end
 
     specify 'returns only deletes for all types' do
       deletes = receiver.deletes
-      expect(deletes.keys).to match_array([DummiesIndex::Fizz, DummiesIndex::Buzz])
+      expect(deletes.keys).to match_array([DummiesIndex::Fizz, Dummies2Index::Buzz])
       expect(deletes.values.flatten).to match_array([1, 2, 1, 2, 3])
     end
   end
@@ -113,8 +115,8 @@ describe :search_index_receiver do
   context 'updated_indexes' do
     specify 'provides a list of indices updated' do
       receiver.catch search_request(2, verb: :delete), DummiesIndex::Fizz
-      receiver.catch search_request(3, verb: :delete), DummiesIndex::Buzz
-      expect(receiver.updated_indexes).to match_array([DummiesIndex::Fizz, DummiesIndex::Buzz])
+      receiver.catch search_request(3, verb: :delete), Dummies2Index::Buzz
+      expect(receiver.updated_indexes).to match_array([DummiesIndex::Fizz, Dummies2Index::Buzz])
     end
   end
 end