chewy 7.2.3 → 7.2.4

data/lib/chewy/index/import/bulk_builder.rb

@@ -4,7 +4,7 @@ module Chewy
       # This class purpose is to build ES client-acceptable bulk
       # request body from the passed objects for index and deletion.
       # It handles parent-child relationships as well by fetching
-      # existing documents from ES, taking their `_parent` field and
+      # existing documents from ES and database, taking their join field values and
       # using it in the bulk body.
       # If fields are passed - it creates partial update entries except for
       # the cases when the type has parent and parent_id has been changed.
@@ -24,9 +24,11 @@ module Chewy
         # @see https://github.com/elastic/elasticsearch-ruby/blob/master/elasticsearch-api/lib/elasticsearch/api/actions/bulk.rb
         # @return [Array<Hash>] bulk body
         def bulk_body
+          populate_cache
+
           @bulk_body ||= @to_index.flat_map(&method(:index_entry)).concat(
             @delete.flat_map(&method(:delete_entry))
-          )
+          ).uniq
         end
 
         # The only purpose of this method is to cache document ids for
@@ -39,64 +41,250 @@ module Chewy
 
       private
 
-        def crutches
-          @crutches ||= Chewy::Index::Crutch::Crutches.new @index, @to_index
-        end
-
-        def parents
-          return unless type_root.parent_id
-
-          @parents ||= begin
-            ids = @index.map do |object|
-              object.respond_to?(:id) ? object.id : object
-            end
-            ids.concat(@delete.map do |object|
-              object.respond_to?(:id) ? object.id : object
-            end)
-            @index.filter(ids: {values: ids}).order('_doc').pluck(:_id, :_parent).to_h
-          end
+        def crutches_for_index
+          @crutches_for_index ||= Chewy::Index::Crutch::Crutches.new @index, @to_index
         end
 
         def index_entry(object)
           entry = {}
           entry[:_id] = index_object_ids[object] if index_object_ids[object]
 
-          if parents
-            entry[:parent] = type_root.compose_parent(object)
-            parent = entry[:_id].present? && parents[entry[:_id].to_s]
-          end
+          data = data_for(object)
+          parent = cache(entry[:_id])
 
-          if parent && entry[:parent].to_s != parent
-            entry[:data] = @index.compose(object, crutches)
-            [{delete: entry.except(:data).merge(parent: parent)}, {index: entry}]
+          entry[:routing] = routing(object) if join_field?
+          if parent_changed?(data, parent)
+            reindex_entries(object, data) + reindex_descendants(object)
           elsif @fields.present?
             return [] unless entry[:_id]
 
-            entry[:data] = {doc: @index.compose(object, crutches, fields: @fields)}
+            entry[:data] = {doc: data_for(object, fields: @fields)}
             [{update: entry}]
           else
-            entry[:data] = @index.compose(object, crutches)
+            entry[:data] = data
             [{index: entry}]
           end
         end
 
+        def reindex_entries(object, data, root: object)
+          entry = {}
+          entry[:_id] = index_object_ids[object] || entry_id(object)
+          entry[:data] = data
+          entry[:routing] = routing(root) || routing(object) if join_field?
+          delete = delete_single_entry(object, root: root).first
+          index = {index: entry}
+          [delete, index]
+        end
+
+        def reindex_descendants(root)
+          descendants = load_descendants(root)
+          crutches = Chewy::Index::Crutch::Crutches.new @index, [root, *descendants]
+          descendants.flat_map do |object|
+            reindex_entries(
+              object,
+              data_for(object, crutches: crutches),
+              root: root
+            )
+          end
+        end
+
         def delete_entry(object)
+          delete_single_entry(object)
+        end
+
+        def delete_single_entry(object, root: object)
           entry = {}
           entry[:_id] = entry_id(object)
           entry[:_id] ||= object.as_json
 
           return [] if entry[:_id].blank?
 
-          if parents
-            parent = entry[:_id].present? && parents[entry[:_id].to_s]
-            return [] unless parent
+          if join_field?
+            cached_parent = cache(entry[:_id])
+            entry_parent_id =
+              if cached_parent
+                cached_parent[:parent_id]
+              else
+                find_parent_id(object)
+              end
 
-            entry[:parent] = parent
+            entry[:routing] = existing_routing(root.try(:id)) || existing_routing(object.id)
+            entry[:parent] = entry_parent_id if entry_parent_id
           end
 
           [{delete: entry}]
         end
 
+        def load_descendants(root)
+          root_type = join_field_type(root)
+          return [] unless root_type
+
+          descendant_ids = []
+          grouped_parents = {root_type => [root.id]}
+          # iteratively fetch all the descendants (with grouped_parents as a queue for next iteration)
+          until grouped_parents.empty?
+            children_data = grouped_parents.flat_map do |parent_type, parent_ids|
+              @index.query(
+                has_parent: {
+                  parent_type: parent_type,
+                  # ignore_unmapped to avoid error for the leaves of the tree
+                  # (types without children)
+                  ignore_unmapped: true,
+                  query: {ids: {values: parent_ids}}
+                }
+              ).pluck(:_id, join_field).map { |id, join| [join['name'], id] }
+            end
+            descendant_ids |= children_data.map(&:last)
+
+            grouped_parents = {}
+            children_data.each do |name, id|
+              next unless name
+
+              grouped_parents[name] ||= []
+              grouped_parents[name] << id
+            end
+          end
+          # query the primary database to load the descentants' records
+          @index.adapter.load(descendant_ids, _index: @index.base_name, raw_import: @index._default_import_options[:raw_import])
+        end
+
+        def populate_cache
+          @cache = load_cache
+        end
+
+        def cache(id)
+          @cache[id.to_s]
+        end
+
+        def load_cache
+          return {} unless join_field?
+
+          @index
+            .filter(ids: {values: ids_for_cache})
+            .order('_doc')
+            .pluck(:_id, :_routing, join_field)
+            .map do |id, routing, join|
+              [
+                id,
+                {routing: routing, parent_id: join['parent']}
+              ]
+            end.to_h
+        end
+
+        def existing_routing(id)
+          # All objects needed here should be cached in #load_cache,
+          # if not, we return nil. In some cases we don't have existing routing cached,
+          # e.g. for loaded descendants
+          return unless cache(id)
+
+          cache(id)[:routing]
+        end
+
+        # Two types of ids:
+        # * of parents of the objects to be indexed
+        # * of objects to be deleted
+        def ids_for_cache
+          ids = @to_index.flat_map do |object|
+            [find_parent_id(object), object.id] if object.respond_to?(:id)
+          end
+          ids.concat(@delete.map do |object|
+            object.id if object.respond_to?(:id)
+          end)
+          ids.uniq.compact
+        end
+
+        def routing(object)
+          # filter out non-model objects, early return on object==nil
+          return unless object.respond_to?(:id)
+
+          parent_id = find_parent_id(object)
+          if parent_id
+            routing(index_objects_by_id[parent_id.to_s]) || existing_routing(parent_id)
+          else
+            object.id.to_s
+          end
+        end
+
+        def find_parent_id(object)
+          return unless object.respond_to?(:id)
+
+          join = data_for(object, fields: [join_field.to_sym])[join_field]
+          join['parent'] if join
+        end
+
+        def join_field
+          return @join_field if defined?(@join_field)
+
+          @join_field = find_join_field
+        end
+
+        def find_join_field
+          type_settings = @index.mappings_hash[:mappings]
+          return unless type_settings
+
+          properties = type_settings[:properties]
+          join_fields = properties.find { |_, options| options[:type] == :join }
+          return unless join_fields
+
+          join_fields.first.to_s
+        end
+
+        def join_field_type(object)
+          return unless join_field?
+
+          raw_object =
+            if @index._default_import_options[:raw_import]
+              @index._default_import_options[:raw_import].call(object.attributes)
+            else
+              object
+            end
+
+          join_field_value = data_for(
+            raw_object,
+            fields: [join_field.to_sym], # build only the field that is needed
+            crutches: Chewy::Index::Crutch::Crutches.new(@index, [raw_object])
+          )[join_field]
+
+          case join_field_value
+          when String
+            join_field_value
+          when Hash
+            join_field_value['name']
+          end
+        end
+
+        def join_field?
+          join_field && !join_field.empty?
+        end
+
+        def data_for(object, fields: [], crutches: crutches_for_index)
+          @index.compose(object, crutches, fields: fields)
+        end
+
+        def parent_changed?(data, old_parent)
+          return false unless old_parent
+          return false unless join_field?
+          return false unless @fields.include?(join_field.to_sym)
+          return false unless data.key?(join_field)
+
+          # The join field value can be a hash, e.g.:
+          # {"name": "child", "parent": "123"} for a child
+          # {"name": "parent"} for a parent
+          # but it can also be a string: (e.g. "parent") for a parent:
+          # https://www.elastic.co/guide/en/elasticsearch/reference/current/parent-join.html#parent-join
+          new_join_field_value = data[join_field]
+          if new_join_field_value.is_a? Hash
+            # If we have a hash in the join field,
+            # we're taking the `parent` field that holds the parent id.
+            new_parent_id = new_join_field_value['parent']
+            new_parent_id != old_parent[:parent_id]
+          else
+            # If there is a non-hash value (String or nil), it means that the join field is changed
+            # and the current object is no longer a child.
+            true
+          end
+        end
+
         def entry_id(object)
           if type_root.id
             type_root.compose_id(object)

data/lib/chewy/index/import/bulk_request.rb

@@ -33,7 +33,7 @@ module Chewy
           return [] if body.blank?
 
           request_bodies(body).each_with_object([]) do |request_body, results|
-            response = @index.client.bulk request_base.merge(body: request_body) if request_body.present?
+            response = @index.client.bulk(**request_base.merge(body: request_body)) if request_body.present?
 
             next unless response.try(:[], 'errors')
 

data/lib/chewy/index/import.rb

@@ -36,8 +36,7 @@ module Chewy
         # passed objects from the index if they are not in the default scope
         # or marked for destruction.
         #
-        # It handles parent-child relationships: if the object parent_id has been
-        # changed it destroys the object and recreates it from scratch.
+        # It handles parent-child relationships with a join field reindexing children when the parent is reindexed.
         #
         # Performs journaling if enabled: it stores all the ids of the imported
         # objects to a specialized index. It is possible to replay particular import
@@ -73,7 +72,7 @@ module Chewy
         # @option options [true, false] update_failover enables full objects reimport in cases of partial update errors, `true` by default
         # @option options [true, Integer, Hash] parallel enables parallel import processing with the Parallel gem, accepts the number of workers or any Parallel gem acceptable options
         # @return [true, false] false in case of errors
-        def import(*args)
+        ruby2_keywords def import(*args)
           import_routine(*args).blank?
         end
 
@@ -84,7 +83,7 @@ module Chewy
         # in case of any import errors.
         #
         # @raise [Chewy::ImportFailed] in case of errors
-        def import!(*args)
+        ruby2_keywords def import!(*args)
           errors = import_routine(*args)
           raise Chewy::ImportFailed.new(self, errors) if errors.present?
 

data/lib/chewy/index/mapping.rb

@@ -170,8 +170,8 @@ module Chewy
         #   template /tit.+/, type: 'text', mapping_hash # "match_mapping_type" as an optional second argument
         #   template template42: {match: 'hello*', mapping: {type: 'object'}} # or even pass a template as is
         #
-        def template(*args)
-          root.dynamic_template(*args)
+        def template(*args, **options)
+          root.dynamic_template(*args, **options)
         end
         alias_method :dynamic_template, :template
 

data/lib/chewy/index/observe.rb

@@ -48,7 +48,7 @@ module Chewy
       extend Helpers
 
       module ActiveRecordMethods
-        def update_index(type_name, *args, &block)
+        ruby2_keywords def update_index(type_name, *args, &block)
           callback_options = Observe.extract_callback_options!(args)
           update_proc = Observe.update_proc(type_name, *args, &block)
 

data/lib/chewy/rspec/update_index.rb

@@ -1,4 +1,4 @@
-require 'i18n/core_ext/hash'
+require 'active_support/core_ext/hash/keys'
 
 # Rspec matcher `update_index`
 # To use it - add `require 'chewy/rspec'` to the `spec_helper.rb`

data/lib/chewy/search/request.rb

@@ -18,7 +18,7 @@ module Chewy
       include Scoping
       include Scrolling
       UNDEFINED = Class.new.freeze
-      EVERFIELDS = %w[_index _type _id _parent].freeze
+      EVERFIELDS = %w[_index _type _id _parent _routing].freeze
       DELEGATED_METHODS = %i[
         query filter post_filter order reorder docvalue_fields
         track_scores track_total_hits request_cache explain version profile
@@ -914,7 +914,7 @@ module Chewy
 
       # Returns and array of values for specified fields.
       # Uses `source` to restrict the list of returned fields.
-      # Fields `_id`, `_type` and `_index` are also supported.
+      # Fields `_id`, `_type`, `_routing` and `_index` are also supported.
       #
       # @overload pluck(field)
       #   If single field is passed - it returns and array of values.

data/lib/chewy/search.rb

@@ -56,7 +56,7 @@ module Chewy
       #
       # @example
       #   PlacesIndex.query(match: {name: 'Moscow'})
-      def method_missing(name, *args, &block)
+      ruby2_keywords def method_missing(name, *args, &block)
         if search_class::DELEGATED_METHODS.include?(name)
           all.send(name, *args, &block)
         else

data/lib/chewy/strategy/active_job.rb

@@ -15,7 +15,7 @@ module Chewy
 
         def perform(type, ids, options = {})
           options[:refresh] = !Chewy.disable_refresh_async if Chewy.disable_refresh_async
-          type.constantize.import!(ids, options)
+          type.constantize.import!(ids, **options)
         end
       end
 

data/lib/chewy/strategy/sidekiq.rb

@@ -15,7 +15,7 @@ module Chewy
 
         def perform(type, ids, options = {})
           options[:refresh] = !Chewy.disable_refresh_async if Chewy.disable_refresh_async
-          type.constantize.import!(ids, options)
+          type.constantize.import!(ids, **options)
         end
       end
 

data/lib/chewy/version.rb

@@ -1,3 +1,3 @@
 module Chewy
-  VERSION = '7.2.3'.freeze
+  VERSION = '7.2.4'.freeze
 end

data/lib/chewy.rb

@@ -4,18 +4,18 @@ require 'active_support/deprecation'
 require 'active_support/json'
 require 'active_support/log_subscriber'
 
+require 'active_support/isolated_execution_state' if ActiveSupport::VERSION::MAJOR >= 7
 require 'active_support/core_ext/array/access'
 require 'active_support/core_ext/array/wrap'
 require 'active_support/core_ext/enumerable'
 require 'active_support/core_ext/hash/reverse_merge'
+require 'active_support/core_ext/hash/keys'
 require 'active_support/core_ext/numeric/time'
 require 'active_support/core_ext/numeric/bytes'
 require 'active_support/core_ext/object/blank'
 require 'active_support/core_ext/object/inclusion'
 require 'active_support/core_ext/string/inflections'
 
-require 'i18n/core_ext/hash'
-require 'chewy/backports/deep_dup' unless Object.respond_to?(:deep_dup)
 require 'singleton'
 require 'base64'
 

data/lib/tasks/chewy.rake

@@ -54,7 +54,7 @@ namespace :chewy do
 
   desc 'Update mapping of exising index with body hash'
   task :update_mapping, %i[index_name] => :environment do |_task, args|
-    Chewy::RakeHelper.update_mapping(name: args[:name])
+    Chewy::RakeHelper.update_mapping(name: args[:index_name])
   end
 
   namespace :parallel do

data/spec/chewy/fields/base_spec.rb

@@ -44,24 +44,9 @@ describe Chewy::Fields::Base do
     end
 
     context 'parent objects' do
-      let!(:country) do
-        described_class.new(:name, value: lambda { |country, crutches|
-                                            country.cities.map do |city|
-                                              double(districts: city.districts, name: crutches.city_name)
-                                            end
-                                          })
-      end
-      let!(:city) do
-        described_class.new(:name, value: lambda { |city, country, crutches|
-                                            city.districts.map do |district|
-                                              [district, country.name, crutches.suffix]
-                                            end
-                                          })
-      end
-      let(:district_value) { ->(district, city, country, crutches) { [district, city.name, country.name, crutches] } }
-      let!(:district) do
-        described_class.new(:name, value: district_value)
-      end
+      let!(:country) { described_class.new(:name, value: ->(country, crutches) { country.cities.map { |city| double(districts: city.districts, name: crutches.city_name) } }) }
+      let!(:city) { described_class.new(:name, value: ->(city, country, crutches) { city.districts.map { |district| [district, country.name, crutches.suffix] } }) }
+      let!(:district) { described_class.new(:name, value: ->(district, city, country, crutches) { [district, city.name, country.name, crutches] }) }
       let(:crutches) { double(suffix: 'suffix', city_name: 'Bangkok') }
 
       before do
@@ -556,6 +541,41 @@ describe Chewy::Fields::Base do
           end
         end
 
+        context 'join field type' do
+          before do
+            stub_model(:comment)
+            stub_index(:comments) do
+              index_scope Comment
+              field :id
+              field :hierarchy, type: :join, relations: {question: %i[answer comment], answer: :vote, vote: :subvote}, join: {type: :comment_type, id: :commented_id}
+            end
+          end
+
+          specify do
+            expect(
+              CommentsIndex.root.compose(
+                {'id' => 1, 'comment_type' => 'question'}
+              )
+            ).to eq(
+              {'id' => 1, 'hierarchy' => 'question'}
+            )
+
+            expect(
+              CommentsIndex.root.compose(
+                {'id' => 2, 'comment_type' => 'answer', 'commented_id' => 1}
+              )
+            ).to eq(
+              {'id' => 2, 'hierarchy' => {'name' => 'answer', 'parent' => 1}}
+            )
+
+            expect do
+              CommentsIndex.root.compose(
+                {'id' => 2, 'comment_type' => 'asd', 'commented_id' => 1}
+              )
+            end.to raise_error Chewy::InvalidJoinFieldType
+          end
+        end
+
         context 'without ignore_blank option' do
           before do
             stub_index(:countries) do

data/spec/chewy/index/adapter/active_record_spec.rb

@@ -1,5 +1,11 @@
 require 'spec_helper'
 
+RawCity = Struct.new(:id) do
+  def rating
+    id * 10
+  end
+end
+
 describe Chewy::Index::Adapter::ActiveRecord, :active_record do
   before do
     stub_model(:city)
@@ -571,5 +577,25 @@ describe Chewy::Index::Adapter::ActiveRecord, :active_record do
         ).to eq(cities.first(2) + [nil])
       end
     end
+
+    context 'with raw_import option' do
+      subject { described_class.new(City) }
+
+      let!(:cities) { Array.new(3) { |i| City.create!(rating: i / 2) } }
+      let(:city_ids) { cities.map(&:id) }
+
+      let(:raw_import) { ->(hash) { RawCity.new(hash['id']) } }
+      it 'uses the custom loader' do
+        raw_cities = subject.load(city_ids, _index: 'cities', raw_import: raw_import).map do |c|
+          {id: c.id, rating: c.rating}
+        end
+
+        expect(raw_cities).to eq([
+          {id: 1, rating: 10},
+          {id: 2, rating: 20},
+          {id: 3, rating: 30}
+        ])
+      end
+    end
   end
 end