chewy 6.0.0 → 7.6.0

data/lib/chewy/{type → index}/adapter/active_record.rb

@@ -1,21 +1,31 @@
-require 'chewy/type/adapter/orm'
+require 'chewy/index/adapter/orm'
 
 module Chewy
-  class Type
+  class Index
     module Adapter
       class ActiveRecord < Orm
         def self.accepts?(target)
           defined?(::ActiveRecord::Base) && (
-            target.is_a?(Class) && target < ::ActiveRecord::Base ||
+            (target.is_a?(Class) && target < ::ActiveRecord::Base) ||
             target.is_a?(::ActiveRecord::Relation))
         end
 
       private
 
         def cleanup_default_scope!
-          if Chewy.logger && (@default_scope.arel.orders.present? ||
+          behavior = Chewy.config.import_scope_cleanup_behavior
+
+          if behavior != :ignore && (@default_scope.arel.orders.present? ||
              @default_scope.arel.limit.present? || @default_scope.arel.offset.present?)
-            Chewy.logger.warn('Default type scope order, limit and offset are ignored and will be nullified')
+            if behavior == :warn && Chewy.logger
+              gem_dir = File.realpath('../..', __dir__)
+              source = caller.grep_v(Regexp.new(gem_dir)).first
+              Chewy.logger.warn(
+                "Default type scope order, limit and offset are ignored and will be nullified (called from: #{source})"
+              )
+            elsif behavior == :raise
+              raise ImportScopeCleanupError, 'Default type scope order, limit and offset are ignored and will be nullified'
+            end
           end
 
           @default_scope = @default_scope.reorder(nil).limit(nil).offset(nil)
@@ -60,7 +70,15 @@ module Chewy
         end
 
         def pluck_in_batches(scope, fields: [], batch_size: nil, typecast: true)
-          return enum_for(:pluck_in_batches, scope, fields: fields, batch_size: batch_size, typecast: typecast) unless block_given?
+          unless block_given?
+            return enum_for(
+              :pluck_in_batches,
+              scope,
+              fields: fields,
+              batch_size: batch_size,
+              typecast: typecast
+            )
+          end
 
           scope = scope.reorder(target_id.asc).limit(batch_size)
           ids = pluck(scope, fields: fields, typecast: typecast)
@@ -69,6 +87,7 @@ module Chewy
           while ids.present?
             yield ids
             break if ids.size < batch_size
+
             last_id = ids.last.is_a?(Array) ? ids.last.first : ids.last
             ids = pluck(scope.where(target_id.gt(last_id)), fields: fields, typecast: typecast)
           end
@@ -85,6 +104,11 @@ module Chewy
           object_class.connection.execute(sql).map(&converter)
         end
 
+        def raw(scope, converter)
+          sql = scope.to_sql
+          object_class.connection.execute(sql).map(&converter)
+        end
+
         def relation_class
           ::ActiveRecord::Relation
         end

data/lib/chewy/{type → index}/adapter/base.rb

@@ -1,5 +1,5 @@
 module Chewy
-  class Type
+  class Index
     module Adapter
       # Basic adapter class. Contains interface, need to implement to add any classes support
       class Base
@@ -13,8 +13,7 @@ module Chewy
           true
         end
 
-        # Camelcased name, used as type class constant name.
-        # For returned value 'Product' will be generated class name `ProductsIndex::Product`
+        # Camelcased name.
         #
         def name
           raise NotImplementedError

data/lib/chewy/{type → index}/adapter/object.rb

@@ -1,7 +1,7 @@
-require 'chewy/type/adapter/base'
+require 'chewy/index/adapter/base'
 
 module Chewy
-  class Type
+  class Index
     module Adapter
       # This adapter provides an ability to import documents from any
       # source. You can actually use any class or even a symbol as
@@ -14,15 +14,15 @@ module Chewy
       # @see #import
       # @see #load
       class Object < Base
-        # The signature of the type definition.
+        # The signature of the index scope definition.
         #
         # @example
-        #   define_type :geoname
-        #   define_type Geoname
-        #   define_type -> { Geoname.all_the_places }, name: 'geoname'
+        #   index_scope :geoname
+        #   index_scope Geoname
+        #   index_scope -> { Geoname.all_the_places }, name: 'geoname'
         #
         # @param target [Class, Symbol, String, Proc] a source of data and everything
-        # @option options [String, Symbol] :name redefines the inferred type name if necessary
+        # @option options [String, Symbol] :name redefines the inferred name if necessary
         # @option options [String, Symbol] :import_all_method redefines import method name
         # @option options [String, Symbol] :load_all_method redefines batch load method name
         # @option options [String, Symbol] :load_one_method redefines per-object load method name
@@ -31,14 +31,13 @@ module Chewy
           @options = options
         end
 
-        # Name is used for the type class creation. Inferred from the target
-        # by default if possible.
+        # Inferred from the target by default if possible.
         #
         # @example
-        #   # defines MyIndex::Geoname
-        #   define_type :geoname
-        #   # still defines MyIndex::Geoname
-        #   define_type -> { Geoname.all_the_places }, name: 'geoname'
+        #   # defines name = Geoname
+        #   index_scope :geoname
+        #   # still defines name = Geoname
+        #   index_scope -> { Geoname.all_the_places }, name: 'geoname'
         #
         # @return [String]
         def name
@@ -54,14 +53,14 @@ module Chewy
           Array.wrap(collection)
         end
 
-        # This method is used internally by `Chewy::Type.import`.
+        # This method is used internally by `Chewy::Index.import`.
         #
         # The idea is that any object can be imported to ES if
         # it responds to `#to_json` method.
         #
         # If method `destroyed?` is defined for object (or, in case of hash object,
         # it has `:_destroyed` or `'_destroyed'` key) and returns `true` or object
-        # satisfy `delete_if` type option then object will be deleted from index.
+        # satisfy `delete_if` option then object will be deleted from index.
         # But in order to be destroyable, objects need to respond to `id` method
         # or have an `id` key so ElasticSearch could know which one to delete.
         #
@@ -78,10 +77,10 @@ module Chewy
         #     end
         #   end
         #
-        #   # All the folloving variants will work:
-        #   define_type Geoname
-        #   define_type Geoname, import_all_method: 'import_all'
-        #   define_type -> { FancyGeoAPI.all_points_collection }, name: 'geoname'
+        #   # All the following variants will work:
+        #   index_scope Geoname
+        #   index_scope Geoname, import_all_method: 'import_all'
+        #   index_scope -> { FancyGeoAPI.all_points_collection }, name: 'geoname'
         #
         # @param args [Array<#to_json>]
         # @option options [Integer] :batch_size import processing batch size
@@ -113,16 +112,15 @@ module Chewy
         #     end
         #   end
         #
-        # @see Chewy::Type::Adapter::Base#import_fields
-        def import_fields(*args)
+        # @see Chewy::Index::Adapter::Base#import_fields
+        def import_fields(*args, &block)
           return enum_for(:import_fields, *args) unless block_given?
+
           options = args.extract_options!
           options[:batch_size] ||= BATCH_SIZE
 
           if args.empty? && @target.respond_to?(pluck_method)
-            @target.send(pluck_method, :id, *options[:fields]).each_slice(options[:batch_size]) do |batch|
-              yield batch
-            end
+            @target.send(pluck_method, :id, *options[:fields]).each_slice(options[:batch_size], &block)
           elsif options[:fields].blank?
             import_references(*args, options) do |batch|
               yield batch.map { |object| object_field(object, :id) || object }
@@ -140,14 +138,12 @@ module Chewy
 
         # For the Object adapter returns the objects themselves in batches.
         #
-        # @see Chewy::Type::Adapter::Base#import_references
-        def import_references(*args)
+        # @see Chewy::Index::Adapter::Base#import_references
+        def import_references(*args, &block)
           return enum_for(:import_references, *args) unless block_given?
 
           collection, options = import_args(*args)
-          collection.each_slice(options[:batch_size]) do |batch|
-            yield batch
-          end
+          collection.each_slice(options[:batch_size], &block)
         end
 
         # This method is used internally by the request DSL when the
@@ -157,7 +153,7 @@ module Chewy
         #
         # If none of the `load_all_method` or `load_one_method` is implemented
         # for the target - the method will return nil. This means that the
-        # loader will return an array `Chewy::Type` objects that actually was passed.
+        # loader will return an array `Chewy::Index` objects that actually was passed.
         #
         # To use loading for objects it is obviously required to provide
         # some meaningful ids for ES documents.
@@ -175,7 +171,7 @@ module Chewy
         #     end
         #   end
         #
-        #   MyIndex::Geoname.load(additional_data: true).objects
+        #   MyIndex.load(additional_data: true).objects
         #
         # @param ids [Array<Hash>] an array of ids from ES hits
         # @param options [Hash] any options passed here with the request DSL `load` method.

data/lib/chewy/{type → index}/adapter/orm.rb

@@ -1,7 +1,7 @@
-require 'chewy/type/adapter/base'
+require 'chewy/index/adapter/base'
 
 module Chewy
-  class Type
+  class Index
     module Adapter
       class Orm < Base
         attr_reader :default_scope
@@ -54,24 +54,23 @@ module Chewy
         #
         #   users = User.all
         #   users.each { |user| user.destroy if user.inactive? }
-        #   UsersIndex::User.import users # inactive users will be deleted from index
+        #   UsersIndex.import users # inactive users will be deleted from index
         #   # or
-        #   UsersIndex::User.import users.map(&:id) # deleted user ids will be deleted from index
+        #   UsersIndex.import users.map(&:id) # deleted user ids will be deleted from index
         #
         # Also there is custom type option `delete_if`. It it returns `true`
         # object will be deleted from index. Note that if this option is defined and
         # return `false` Chewy will still check `destroyed?` method. This is useful
         # for paranoid objects deleting implementation.
         #
-        #   define_type User, delete_if: ->{ deleted_at } do
-        #     ...
-        #   end
+        #   index_scope User, delete_if: ->{ deleted_at }
+        #   ...
         #
         #   users = User.all
         #   users.each { |user| user.deleted_at = Time.now }
-        #   UsersIndex::User.import users # paranoid deleted users will be deleted from index
+        #   UsersIndex.import users # paranoid deleted users will be deleted from index
         #   # or
-        #   UsersIndex::User.import users.map(&:id) # user ids will be deleted from index
+        #   UsersIndex.import users.map(&:id) # user ids will be deleted from index
         #
         def import(*args, &block)
           collection, options = import_args(*args)
@@ -92,30 +91,30 @@ module Chewy
             collection = all_scope_where_ids_in(identify(collection)) unless collection.is_a?(relation_class)
             pluck_in_batches(collection, **options.slice(:fields, :batch_size, :typecast), &block)
           else
-            identify(collection).each_slice(options[:batch_size]) do |batch|
-              yield batch
-            end
+            identify(collection).each_slice(options[:batch_size], &block)
           end
         end
         alias_method :import_references, :import_fields
 
         def load(ids, **options)
           scope = all_scope_where_ids_in(ids)
-          additional_scope = options[options[:_type].type_name.to_sym].try(:[], :scope) || options[:scope]
+          additional_scope = options[options[:_index].to_sym].try(:[], :scope) || options[:scope]
 
           loaded_objects = load_scope_objects(scope, additional_scope)
-            .index_by do |object|
-              object.public_send(primary_key).to_s
-            end
+          loaded_objects = raw(loaded_objects, options[:raw_import]) if options[:raw_import]
+
+          indexed_objects = loaded_objects.index_by do |object|
+            object.public_send(primary_key).to_s
+          end
 
-          ids.map { |id| loaded_objects[id.to_s] }
+          ids.map { |id| indexed_objects[id.to_s] }
         end
 
       private
 
         def import_objects(collection, options)
           collection_ids = identify(collection)
-          hash = Hash[collection_ids.map(&:to_s).zip(collection)]
+          hash = collection_ids.map(&:to_s).zip(collection).to_h
 
           indexed = collection_ids.each_slice(options[:batch_size]).map do |ids|
             batch = if options[:raw_import]

data/lib/chewy/index/aliases.rb

@@ -5,14 +5,23 @@ module Chewy
 
       module ClassMethods
         def indexes
-          client.indices.get_alias(name: index_name).keys
-        rescue Elasticsearch::Transport::Transport::Errors::NotFound
-          []
+          indexes = empty_if_not_found { client.indices.get(index: index_name).keys }
+          indexes += empty_if_not_found { client.indices.get_alias(name: index_name).keys }
+          indexes.compact.uniq
         end
 
         def aliases
-          name = index_name
-          client.indices.get_alias(index: name, name: '*')[name].try(:[], 'aliases').try(:keys) || []
+          empty_if_not_found do
+            client.indices.get_alias(index: index_name, name: '*').values.flat_map do |aliases|
+              aliases['aliases'].keys
+            end
+          end.compact.uniq
+        end
+
+      private
+
+        def empty_if_not_found
+          yield
         rescue Elasticsearch::Transport::Transport::Errors::NotFound
           []
         end

data/lib/chewy/index/crutch.rb

@@ -0,0 +1,40 @@
+module Chewy
+  class Index
+    module Crutch
+      extend ActiveSupport::Concern
+
+      included do
+        class_attribute :_crutches
+        self._crutches = {}
+      end
+
+      class Crutches
+        def initialize(index, collection)
+          @index = index
+          @collection = collection
+          @crutches_instances = {}
+        end
+
+        def method_missing(name, *, **)
+          return self[name] if @index._crutches.key?(name)
+
+          super
+        end
+
+        def respond_to_missing?(name, include_private = false)
+          @index._crutches.key?(name) || super
+        end
+
+        def [](name)
+          @crutches_instances[name] ||= @index._crutches[:"#{name}"].call(@collection)
+        end
+      end
+
+      module ClassMethods
+        def crutch(name, &block)
+          self._crutches = _crutches.merge(name.to_sym => block)
+        end
+      end
+    end
+  end
+end

data/lib/chewy/index/import/bulk_builder.rb

@@ -0,0 +1,311 @@
+module Chewy
+  class Index
+    module Import
+      # This class purpose is to build ES client-acceptable bulk
+      # request body from the passed objects for index and deletion.
+      # It handles parent-child relationships as well by fetching
+      # existing documents from ES and database, taking their join field values and
+      # using it in the bulk body.
+      # If fields are passed - it creates partial update entries except for
+      # the cases when the type has parent and parent_id has been changed.
+      class BulkBuilder
+        # @param index [Chewy::Index] desired index
+        # @param to_index [Array<Object>] objects to index
+        # @param delete [Array<Object>] objects or ids to delete
+        # @param fields [Array<Symbol, String>] and array of fields for documents update
+        def initialize(index, to_index: [], delete: [], fields: [])
+          @index = index
+          @to_index = to_index
+          @delete = delete
+          @fields = fields.map!(&:to_sym)
+        end
+
+        # Returns ES API-ready bulk requiest body.
+        # @see https://github.com/elastic/elasticsearch-ruby/blob/master/elasticsearch-api/lib/elasticsearch/api/actions/bulk.rb
+        # @return [Array<Hash>] bulk body
+        def bulk_body
+          populate_cache
+
+          @bulk_body ||= @to_index.flat_map(&method(:index_entry)).concat(
+            @delete.flat_map(&method(:delete_entry))
+          ).uniq
+        end
+
+        # The only purpose of this method is to cache document ids for
+        # all the passed object for index to avoid ids recalculation.
+        #
+        # @return [Hash[String => Object]] an ids-objects index hash
+        def index_objects_by_id
+          @index_objects_by_id ||= index_object_ids.invert.stringify_keys!
+        end
+
+      private
+
+        def crutches_for_index
+          @crutches_for_index ||= Chewy::Index::Crutch::Crutches.new @index, @to_index
+        end
+
+        def index_entry(object)
+          entry = {}
+          entry[:_id] = index_object_ids[object] if index_object_ids[object]
+          entry[:routing] = routing(object) if join_field?
+
+          parent = cache(entry[:_id])
+          data = data_for(object) if parent.present?
+          if parent.present? && parent_changed?(data, parent)
+            reindex_entries(object, data) + reindex_descendants(object)
+          elsif @fields.present?
+            return [] unless entry[:_id]
+
+            entry[:data] = {doc: data_for(object, fields: @fields)}
+            [{update: entry}]
+          else
+            entry[:data] = data || data_for(object)
+            [{index: entry}]
+          end
+        end
+
+        def reindex_entries(object, data, root: object)
+          entry = {}
+          entry[:_id] = index_object_ids[object] || entry_id(object)
+          entry[:data] = data
+          entry[:routing] = routing(root) || routing(object) if join_field?
+          delete = delete_single_entry(object, root: root).first
+          index = {index: entry}
+          [delete, index]
+        end
+
+        def reindex_descendants(root)
+          descendants = load_descendants(root)
+          crutches = Chewy::Index::Crutch::Crutches.new @index, [root, *descendants]
+          descendants.flat_map do |object|
+            reindex_entries(
+              object,
+              data_for(object, crutches: crutches),
+              root: root
+            )
+          end
+        end
+
+        def delete_entry(object)
+          delete_single_entry(object)
+        end
+
+        def delete_single_entry(object, root: object)
+          entry = {}
+          entry[:_id] = entry_id(object)
+          entry[:_id] ||= object.as_json
+
+          return [] if entry[:_id].blank?
+
+          if join_field?
+            cached_parent = cache(entry[:_id])
+            entry_parent_id =
+              if cached_parent
+                cached_parent[:parent_id]
+              else
+                find_parent_id(object)
+              end
+
+            entry[:routing] = existing_routing(root.try(:id)) || existing_routing(object.id)
+            entry[:parent] = entry_parent_id if entry_parent_id
+          end
+
+          [{delete: entry}]
+        end
+
+        def load_descendants(root)
+          root_type = join_field_type(root)
+          return [] unless root_type
+
+          descendant_ids = []
+          grouped_parents = {root_type => [root.id]}
+          # iteratively fetch all the descendants (with grouped_parents as a queue for next iteration)
+          until grouped_parents.empty?
+            children_data = grouped_parents.flat_map do |parent_type, parent_ids|
+              @index.query(
+                has_parent: {
+                  parent_type: parent_type,
+                  # ignore_unmapped to avoid error for the leaves of the tree
+                  # (types without children)
+                  ignore_unmapped: true,
+                  query: {ids: {values: parent_ids}}
+                }
+              ).pluck(:_id, join_field).map { |id, join| [join['name'], id] }
+            end
+            descendant_ids |= children_data.map(&:last)
+
+            grouped_parents = {}
+            children_data.each do |name, id|
+              next unless name
+
+              grouped_parents[name] ||= []
+              grouped_parents[name] << id
+            end
+          end
+          # query the primary database to load the descentants' records
+          @index.adapter.load(descendant_ids, _index: @index.base_name, raw_import: @index._default_import_options[:raw_import])
+        end
+
+        def populate_cache
+          @cache = load_cache
+        end
+
+        def cache(id)
+          @cache[id.to_s]
+        end
+
+        def load_cache
+          return {} unless join_field?
+
+          @index
+            .filter(ids: {values: ids_for_cache})
+            .order('_doc')
+            .pluck(:_id, :_routing, join_field)
+            .to_h do |id, routing, join|
+              [
+                id,
+                {routing: routing, parent_id: join['parent']}
+              ]
+            end
+        end
+
+        def existing_routing(id)
+          # All objects needed here should be cached in #load_cache,
+          # if not, we return nil. In some cases we don't have existing routing cached,
+          # e.g. for loaded descendants
+          return unless cache(id)
+
+          cache(id)[:routing]
+        end
+
+        # Two types of ids:
+        # * of parents of the objects to be indexed
+        # * of objects to be deleted
+        def ids_for_cache
+          ids = @to_index.flat_map do |object|
+            [find_parent_id(object), object.id] if object.respond_to?(:id)
+          end
+          ids.concat(@delete.map do |object|
+            object.id if object.respond_to?(:id)
+          end)
+          ids.uniq.compact
+        end
+
+        def routing(object)
+          # filter out non-model objects, early return on object==nil
+          return unless object.respond_to?(:id)
+
+          parent_id = find_parent_id(object)
+          if parent_id
+            routing(index_objects_by_id[parent_id.to_s]) || existing_routing(parent_id)
+          else
+            object.id.to_s
+          end
+        end
+
+        def find_parent_id(object)
+          return unless object.respond_to?(:id)
+
+          join = data_for(object, fields: [join_field.to_sym])[join_field]
+          join['parent'] if join
+        end
+
+        def join_field
+          return @join_field if defined?(@join_field)
+
+          @join_field = find_join_field
+        end
+
+        def find_join_field
+          type_settings = @index.mappings_hash[:mappings]
+          return unless type_settings
+
+          properties = type_settings[:properties]
+          join_fields = properties.find { |_, options| options[:type] == :join }
+          return unless join_fields
+
+          join_fields.first.to_s
+        end
+
+        def join_field_type(object)
+          return unless join_field?
+
+          raw_object =
+            if @index._default_import_options[:raw_import]
+              @index._default_import_options[:raw_import].call(object.attributes)
+            else
+              object
+            end
+
+          join_field_value = data_for(
+            raw_object,
+            fields: [join_field.to_sym], # build only the field that is needed
+            crutches: Chewy::Index::Crutch::Crutches.new(@index, [raw_object])
+          )[join_field]
+
+          case join_field_value
+          when String
+            join_field_value
+          when Hash
+            join_field_value['name']
+          end
+        end
+
+        def join_field?
+          join_field && !join_field.empty?
+        end
+
+        def data_for(object, fields: [], crutches: crutches_for_index)
+          @index.compose(object, crutches, fields: fields)
+        end
+
+        def parent_changed?(data, old_parent)
+          return false unless old_parent
+          return false unless join_field?
+          return false unless @fields.include?(join_field.to_sym)
+          return false unless data.key?(join_field)
+
+          # The join field value can be a hash, e.g.:
+          # {"name": "child", "parent": "123"} for a child
+          # {"name": "parent"} for a parent
+          # but it can also be a string: (e.g. "parent") for a parent:
+          # https://www.elastic.co/guide/en/elasticsearch/reference/current/parent-join.html#parent-join
+          new_join_field_value = data[join_field]
+          if new_join_field_value.is_a? Hash
+            # If we have a hash in the join field,
+            # we're taking the `parent` field that holds the parent id.
+            new_parent_id = new_join_field_value['parent']
+            new_parent_id != old_parent[:parent_id]
+          else
+            # If there is a non-hash value (String or nil), it means that the join field is changed
+            # and the current object is no longer a child.
+            true
+          end
+        end
+
+        def entry_id(object)
+          if type_root.id
+            type_root.compose_id(object)
+          else
+            id = object.id if object.respond_to?(:id)
+            id ||= object[:id] || object['id'] if object.is_a?(Hash)
+            id = id.to_s if defined?(BSON) && id.is_a?(BSON::ObjectId)
+            id
+          end
+        end
+
+        def index_object_ids
+          @index_object_ids ||= @to_index.each_with_object({}) do |object, result|
+            id = entry_id(object)
+            result[object] = id if id.present?
+          end
+        end
+
+        def type_root
+          @type_root ||= @index.root
+        end
+      end
+    end
+  end
+end