chewy 7.2.1 → 7.6.0

data/lib/chewy/fields/base.rb

@@ -1,8 +1,8 @@
 module Chewy
   module Fields
     class Base
-      attr_reader :name, :options, :value, :children
-      attr_accessor :parent
+      attr_reader :name, :join_options, :options, :children
+      attr_accessor :parent # used by Chewy::Index::Mapping to expand nested fields
 
       def initialize(name, value: nil, **options)
         @name = name.to_sym
@@ -10,9 +10,11 @@ module Chewy
         update_options!(**options)
         @value = value
         @children = []
+        @allowed_relations = find_allowed_relations(options[:relations]) # for join fields
       end
 
       def update_options!(**options)
+        @join_options = options.delete(:join) || {}
         @options = options
       end
 
@@ -31,7 +33,7 @@ module Chewy
           else
             {}
           end
-        mapping.reverse_merge!(options)
+        mapping.reverse_merge!(options.except(:ignore_blank))
         mapping.reverse_merge!(type: (children.present? ? 'object' : Chewy.default_field_type))
 
         {name => mapping}
@@ -53,30 +55,70 @@ module Chewy
         {name => result}
       end
 
+      def value
+        if join_field?
+          join_type = join_options[:type]
+          join_id = join_options[:id]
+          # memoize
+          @value ||= proc do |object|
+            validate_join_type!(value_by_name_proc(join_type).call(object))
+            # If it's a join field and it has join_id, the value is compound and contains
+            # both name (type) and id of the parent object
+            if value_by_name_proc(join_id).call(object).present?
+              {
+                name: value_by_name_proc(join_type).call(object), # parent type
+                parent: value_by_name_proc(join_id).call(object)  # parent id
+              }
+            else
+              value_by_name_proc(join_type).call(object)
+            end
+          end
+        else
+          @value
+        end
+      end
+
     private
 
       def geo_point?
         @options[:type].to_s == 'geo_point'
       end
 
+      def join_field?
+        @options[:type].to_s == 'join'
+      end
+
       def ignore_blank?
         @options.fetch(:ignore_blank) { geo_point? }
       end
 
       def evaluate(objects)
-        object = objects.first
-
         if value.is_a?(Proc)
-          if value.arity.zero?
-            object.instance_exec(&value)
-          elsif value.arity.negative?
-            value.call(*object)
-          else
-            value.call(*objects.first(value.arity))
-          end
+          value_by_proc(objects, value)
         else
-          message = value.is_a?(Symbol) || value.is_a?(String) ? value.to_sym : name
+          value_by_name(objects, value)
+        end
+      end
 
+      def value_by_proc(objects, value)
+        object = objects.first
+        if value.arity.zero?
+          object.instance_exec(&value)
+        elsif value.arity.negative?
+          value.call(*object)
+        else
+          value.call(*objects.first(value.arity))
+        end
+      end
+
+      def value_by_name(objects, value)
+        object = objects.first
+        message = value.is_a?(Symbol) || value.is_a?(String) ? value.to_sym : name
+        value_by_name_proc(message).call(object)
+      end
+
+      def value_by_name_proc(message)
+        proc do |object|
           if object.is_a?(Hash)
             if object.key?(message)
               object[message]
@@ -89,6 +131,20 @@ module Chewy
         end
       end
 
+      def validate_join_type!(type)
+        return unless type
+        return if @allowed_relations.include?(type.to_sym)
+
+        raise Chewy::InvalidJoinFieldType.new(type, @name, options[:relations])
+      end
+
+      def find_allowed_relations(relations)
+        return [] unless relations
+        return relations unless relations.is_a?(Hash)
+
+        (relations.keys + relations.values).flatten.uniq
+      end
+
       def compose_children(value, *parent_objects)
         return unless value
 

data/lib/chewy/fields/root.rb

@@ -1,7 +1,7 @@
 module Chewy
   module Fields
     class Root < Chewy::Fields::Base
-      attr_reader :dynamic_templates, :id, :parent, :parent_id
+      attr_reader :dynamic_templates, :id
 
       def initialize(name, **options)
         super(name, **options)
@@ -12,9 +12,7 @@ module Chewy
 
       def update_options!(**options)
         @id = options.fetch(:id, options.fetch(:_id, @id))
-        @parent = options.fetch(:parent, options.fetch(:_parent, @parent))
-        @parent_id = options.fetch(:parent_id, @parent_id)
-        @options.merge!(options.except(:id, :_id, :parent, :_parent, :parent_id, :type))
+        @options.merge!(options.except(:id, :_id, :type))
       end
 
       def mappings_hash
@@ -50,12 +48,6 @@ module Chewy
         end
       end
 
-      def compose_parent(object)
-        return unless parent_id
-
-        parent_id.arity.zero? ? object.instance_exec(&parent_id) : parent_id.call(object)
-      end
-
       def compose_id(object)
         return unless id
 

data/lib/chewy/index/actions.rb

@@ -146,7 +146,7 @@ module Chewy
         # @param journal [true, false] journaling is switched off for import during reset by default
         # @param import_options [Hash] options, passed to the import call
         # @return [true, false] false in case of errors
-        def reset!(suffix = nil, apply_journal: true, journal: false, progressbar: false, **import_options)
+        def reset!(suffix = nil, apply_journal: true, journal: false, **import_options)
           result = if suffix.present?
             start_time = Time.now
             indexes = self.indexes - [index_name]
@@ -156,23 +156,27 @@ module Chewy
             suffixed_name = index_name(suffix: suffix)
 
             optimize_index_settings suffixed_name
-            result = import import_options.merge(
+            result = import(**import_options.merge(
               suffix: suffix,
               journal: journal,
-              refresh: !Chewy.reset_disable_refresh_interval,
-              progressbar: progressbar
-            )
+              refresh: !Chewy.reset_disable_refresh_interval
+            ))
             original_index_settings suffixed_name
 
             delete if indexes.blank?
-            update_aliases(indexes, general_name, suffixed_name)
+            client.indices.update_aliases body: {actions: [
+              *indexes.map do |index|
+                {remove: {index: index, alias: general_name}}
+              end,
+              {add: {index: suffixed_name, alias: general_name}}
+            ]}
             client.indices.delete index: indexes if indexes.present?
 
             self.journal.apply(start_time, **import_options) if apply_journal
             result
           else
             purge!
-            import import_options.merge(journal: journal)
+            import(**import_options.merge(journal: journal))
           end
 
           specification.lock!
@@ -233,15 +237,6 @@ module Chewy
 
       private
 
-        def update_aliases(indexes, general_name, suffixed_name)
-          client.indices.update_aliases body: {actions: [
-            *indexes.map do |index|
-              {remove: {index: index, alias: general_name}}
-            end,
-            {add: {index: suffixed_name, alias: general_name}}
-          ]}
-        end
-
         def optimize_index_settings(index_name)
           settings = {}
           settings[:refresh_interval] = -1 if Chewy.reset_disable_refresh_interval

data/lib/chewy/index/adapter/active_record.rb

@@ -6,16 +6,26 @@ module Chewy
       class ActiveRecord < Orm
         def self.accepts?(target)
           defined?(::ActiveRecord::Base) && (
-            target.is_a?(Class) && target < ::ActiveRecord::Base ||
+            (target.is_a?(Class) && target < ::ActiveRecord::Base) ||
             target.is_a?(::ActiveRecord::Relation))
         end
 
       private
 
         def cleanup_default_scope!
-          if Chewy.logger && (@default_scope.arel.orders.present? ||
+          behavior = Chewy.config.import_scope_cleanup_behavior
+
+          if behavior != :ignore && (@default_scope.arel.orders.present? ||
              @default_scope.arel.limit.present? || @default_scope.arel.offset.present?)
-            Chewy.logger.warn('Default type scope order, limit and offset are ignored and will be nullified')
+            if behavior == :warn && Chewy.logger
+              gem_dir = File.realpath('../..', __dir__)
+              source = caller.grep_v(Regexp.new(gem_dir)).first
+              Chewy.logger.warn(
+                "Default type scope order, limit and offset are ignored and will be nullified (called from: #{source})"
+              )
+            elsif behavior == :raise
+              raise ImportScopeCleanupError, 'Default type scope order, limit and offset are ignored and will be nullified'
+            end
           end
 
           @default_scope = @default_scope.reorder(nil).limit(nil).offset(nil)
@@ -94,6 +104,11 @@ module Chewy
           object_class.connection.execute(sql).map(&converter)
         end
 
+        def raw(scope, converter)
+          sql = scope.to_sql
+          object_class.connection.execute(sql).map(&converter)
+        end
+
         def relation_class
           ::ActiveRecord::Relation
         end

data/lib/chewy/index/adapter/object.rb

@@ -192,16 +192,6 @@ module Chewy
           end
         end
 
-        def import_count(*args)
-          collection = if args.first.empty? && @target.respond_to?(import_all_method)
-            @target.send(import_all_method)
-          else
-            args.flatten(1).compact
-          end
-
-          collection.count
-        end
-
       private
 
         def import_objects(objects, options)

data/lib/chewy/index/adapter/orm.rb

@@ -101,23 +101,13 @@ module Chewy
           additional_scope = options[options[:_index].to_sym].try(:[], :scope) || options[:scope]
 
           loaded_objects = load_scope_objects(scope, additional_scope)
-            .index_by do |object|
-              object.public_send(primary_key).to_s
-            end
-
-          ids.map { |id| loaded_objects[id.to_s] }
-        end
+          loaded_objects = raw(loaded_objects, options[:raw_import]) if options[:raw_import]
 
-        def import_count(*args)
-          collection = if args.first.empty?
-            default_scope
-          elsif args.first.is_a?(relation_class)
-            args.first
-          else
-            args.flatten.compact
+          indexed_objects = loaded_objects.index_by do |object|
+            object.public_send(primary_key).to_s
           end
 
-          collection.count
+          ids.map { |id| indexed_objects[id.to_s] }
         end
 
       private

data/lib/chewy/index/crutch.rb

@@ -12,13 +12,21 @@ module Chewy
         def initialize(index, collection)
           @index = index
           @collection = collection
-          @index._crutches.each_key do |name|
-            singleton_class.class_eval <<-METHOD, __FILE__, __LINE__ + 1
-              def #{name}
-                @#{name} ||= @index._crutches[:#{name}].call @collection
-              end
-            METHOD
-          end
+          @crutches_instances = {}
+        end
+
+        def method_missing(name, *, **)
+          return self[name] if @index._crutches.key?(name)
+
+          super
+        end
+
+        def respond_to_missing?(name, include_private = false)
+          @index._crutches.key?(name) || super
+        end
+
+        def [](name)
+          @crutches_instances[name] ||= @index._crutches[:"#{name}"].call(@collection)
         end
       end
 

data/lib/chewy/index/import/bulk_builder.rb

@@ -4,7 +4,7 @@ module Chewy
       # This class purpose is to build ES client-acceptable bulk
       # request body from the passed objects for index and deletion.
       # It handles parent-child relationships as well by fetching
-      # existing documents from ES, taking their `_parent` field and
+      # existing documents from ES and database, taking their join field values and
       # using it in the bulk body.
       # If fields are passed - it creates partial update entries except for
       # the cases when the type has parent and parent_id has been changed.
@@ -24,9 +24,11 @@ module Chewy
         # @see https://github.com/elastic/elasticsearch-ruby/blob/master/elasticsearch-api/lib/elasticsearch/api/actions/bulk.rb
         # @return [Array<Hash>] bulk body
         def bulk_body
+          populate_cache
+
           @bulk_body ||= @to_index.flat_map(&method(:index_entry)).concat(
             @delete.flat_map(&method(:delete_entry))
-          )
+          ).uniq
         end
 
         # The only purpose of this method is to cache document ids for
@@ -39,64 +41,249 @@ module Chewy
 
       private
 
-        def crutches
-          @crutches ||= Chewy::Index::Crutch::Crutches.new @index, @to_index
-        end
-
-        def parents
-          return unless type_root.parent_id
-
-          @parents ||= begin
-            ids = @index.map do |object|
-              object.respond_to?(:id) ? object.id : object
-            end
-            ids.concat(@delete.map do |object|
-              object.respond_to?(:id) ? object.id : object
-            end)
-            @index.filter(ids: {values: ids}).order('_doc').pluck(:_id, :_parent).to_h
-          end
+        def crutches_for_index
+          @crutches_for_index ||= Chewy::Index::Crutch::Crutches.new @index, @to_index
         end
 
         def index_entry(object)
           entry = {}
           entry[:_id] = index_object_ids[object] if index_object_ids[object]
+          entry[:routing] = routing(object) if join_field?
 
-          if parents
-            entry[:parent] = type_root.compose_parent(object)
-            parent = entry[:_id].present? && parents[entry[:_id].to_s]
-          end
-
-          if parent && entry[:parent].to_s != parent
-            entry[:data] = @index.compose(object, crutches)
-            [{delete: entry.except(:data).merge(parent: parent)}, {index: entry}]
+          parent = cache(entry[:_id])
+          data = data_for(object) if parent.present?
+          if parent.present? && parent_changed?(data, parent)
+            reindex_entries(object, data) + reindex_descendants(object)
           elsif @fields.present?
             return [] unless entry[:_id]
 
-            entry[:data] = {doc: @index.compose(object, crutches, fields: @fields)}
+            entry[:data] = {doc: data_for(object, fields: @fields)}
             [{update: entry}]
           else
-            entry[:data] = @index.compose(object, crutches)
+            entry[:data] = data || data_for(object)
             [{index: entry}]
           end
         end
 
+        def reindex_entries(object, data, root: object)
+          entry = {}
+          entry[:_id] = index_object_ids[object] || entry_id(object)
+          entry[:data] = data
+          entry[:routing] = routing(root) || routing(object) if join_field?
+          delete = delete_single_entry(object, root: root).first
+          index = {index: entry}
+          [delete, index]
+        end
+
+        def reindex_descendants(root)
+          descendants = load_descendants(root)
+          crutches = Chewy::Index::Crutch::Crutches.new @index, [root, *descendants]
+          descendants.flat_map do |object|
+            reindex_entries(
+              object,
+              data_for(object, crutches: crutches),
+              root: root
+            )
+          end
+        end
+
         def delete_entry(object)
+          delete_single_entry(object)
+        end
+
+        def delete_single_entry(object, root: object)
           entry = {}
           entry[:_id] = entry_id(object)
           entry[:_id] ||= object.as_json
 
           return [] if entry[:_id].blank?
 
-          if parents
-            parent = entry[:_id].present? && parents[entry[:_id].to_s]
-            return [] unless parent
+          if join_field?
+            cached_parent = cache(entry[:_id])
+            entry_parent_id =
+              if cached_parent
+                cached_parent[:parent_id]
+              else
+                find_parent_id(object)
+              end
 
-            entry[:parent] = parent
+            entry[:routing] = existing_routing(root.try(:id)) || existing_routing(object.id)
+            entry[:parent] = entry_parent_id if entry_parent_id
           end
 
           [{delete: entry}]
         end
 
+        def load_descendants(root)
+          root_type = join_field_type(root)
+          return [] unless root_type
+
+          descendant_ids = []
+          grouped_parents = {root_type => [root.id]}
+          # iteratively fetch all the descendants (with grouped_parents as a queue for next iteration)
+          until grouped_parents.empty?
+            children_data = grouped_parents.flat_map do |parent_type, parent_ids|
+              @index.query(
+                has_parent: {
+                  parent_type: parent_type,
+                  # ignore_unmapped to avoid error for the leaves of the tree
+                  # (types without children)
+                  ignore_unmapped: true,
+                  query: {ids: {values: parent_ids}}
+                }
+              ).pluck(:_id, join_field).map { |id, join| [join['name'], id] }
+            end
+            descendant_ids |= children_data.map(&:last)
+
+            grouped_parents = {}
+            children_data.each do |name, id|
+              next unless name
+
+              grouped_parents[name] ||= []
+              grouped_parents[name] << id
+            end
+          end
+          # query the primary database to load the descentants' records
+          @index.adapter.load(descendant_ids, _index: @index.base_name, raw_import: @index._default_import_options[:raw_import])
+        end
+
+        def populate_cache
+          @cache = load_cache
+        end
+
+        def cache(id)
+          @cache[id.to_s]
+        end
+
+        def load_cache
+          return {} unless join_field?
+
+          @index
+            .filter(ids: {values: ids_for_cache})
+            .order('_doc')
+            .pluck(:_id, :_routing, join_field)
+            .to_h do |id, routing, join|
+              [
+                id,
+                {routing: routing, parent_id: join['parent']}
+              ]
+            end
+        end
+
+        def existing_routing(id)
+          # All objects needed here should be cached in #load_cache,
+          # if not, we return nil. In some cases we don't have existing routing cached,
+          # e.g. for loaded descendants
+          return unless cache(id)
+
+          cache(id)[:routing]
+        end
+
+        # Two types of ids:
+        # * of parents of the objects to be indexed
+        # * of objects to be deleted
+        def ids_for_cache
+          ids = @to_index.flat_map do |object|
+            [find_parent_id(object), object.id] if object.respond_to?(:id)
+          end
+          ids.concat(@delete.map do |object|
+            object.id if object.respond_to?(:id)
+          end)
+          ids.uniq.compact
+        end
+
+        def routing(object)
+          # filter out non-model objects, early return on object==nil
+          return unless object.respond_to?(:id)
+
+          parent_id = find_parent_id(object)
+          if parent_id
+            routing(index_objects_by_id[parent_id.to_s]) || existing_routing(parent_id)
+          else
+            object.id.to_s
+          end
+        end
+
+        def find_parent_id(object)
+          return unless object.respond_to?(:id)
+
+          join = data_for(object, fields: [join_field.to_sym])[join_field]
+          join['parent'] if join
+        end
+
+        def join_field
+          return @join_field if defined?(@join_field)
+
+          @join_field = find_join_field
+        end
+
+        def find_join_field
+          type_settings = @index.mappings_hash[:mappings]
+          return unless type_settings
+
+          properties = type_settings[:properties]
+          join_fields = properties.find { |_, options| options[:type] == :join }
+          return unless join_fields
+
+          join_fields.first.to_s
+        end
+
+        def join_field_type(object)
+          return unless join_field?
+
+          raw_object =
+            if @index._default_import_options[:raw_import]
+              @index._default_import_options[:raw_import].call(object.attributes)
+            else
+              object
+            end
+
+          join_field_value = data_for(
+            raw_object,
+            fields: [join_field.to_sym], # build only the field that is needed
+            crutches: Chewy::Index::Crutch::Crutches.new(@index, [raw_object])
+          )[join_field]
+
+          case join_field_value
+          when String
+            join_field_value
+          when Hash
+            join_field_value['name']
+          end
+        end
+
+        def join_field?
+          join_field && !join_field.empty?
+        end
+
+        def data_for(object, fields: [], crutches: crutches_for_index)
+          @index.compose(object, crutches, fields: fields)
+        end
+
+        def parent_changed?(data, old_parent)
+          return false unless old_parent
+          return false unless join_field?
+          return false unless @fields.include?(join_field.to_sym)
+          return false unless data.key?(join_field)
+
+          # The join field value can be a hash, e.g.:
+          # {"name": "child", "parent": "123"} for a child
+          # {"name": "parent"} for a parent
+          # but it can also be a string: (e.g. "parent") for a parent:
+          # https://www.elastic.co/guide/en/elasticsearch/reference/current/parent-join.html#parent-join
+          new_join_field_value = data[join_field]
+          if new_join_field_value.is_a? Hash
+            # If we have a hash in the join field,
+            # we're taking the `parent` field that holds the parent id.
+            new_parent_id = new_join_field_value['parent']
+            new_parent_id != old_parent[:parent_id]
+          else
+            # If there is a non-hash value (String or nil), it means that the join field is changed
+            # and the current object is no longer a child.
+            true
+          end
+        end
+
         def entry_id(object)
           if type_root.id
             type_root.compose_id(object)

data/lib/chewy/index/import/bulk_request.rb

@@ -33,7 +33,7 @@ module Chewy
           return [] if body.blank?
 
           request_bodies(body).each_with_object([]) do |request_body, results|
-            response = @index.client.bulk request_base.merge(body: request_body) if request_body.present?
+            response = @index.client.bulk(**request_base.merge(body: request_body)) if request_body.present?
 
             next unless response.try(:[], 'errors')
 

data/lib/chewy/index/import/routine.rb

@@ -51,9 +51,9 @@ module Chewy
           @parallel_options = @options.delete(:parallel)
           if @parallel_options && !@parallel_options.is_a?(Hash)
             @parallel_options = if @parallel_options.is_a?(Integer)
-              {in_threads: @parallel_options}
+              {in_processes: @parallel_options}
             else
-              {in_threads: [::Parallel.processor_count, ActiveRecord::Base.connection_pool.size].min}
+              {}
             end
           end
           @errors = []
@@ -64,7 +64,7 @@ module Chewy
         # Creates the journal index and the corresponding index if necessary.
         # @return [Object] whatever
         def create_indexes!
-          Chewy::Stash::Journal.create if @options[:journal]
+          Chewy::Stash::Journal.create if @options[:journal] && !Chewy.configuration[:skip_journal_creation_on_import]
           return if Chewy.configuration[:skip_index_creation_on_import]
 
           @index.create!(**@bulk_options.slice(:suffix)) unless @index.exists?