chewy 0.8.4 → 7.3.4

data/lib/chewy/index/adapter/object.rb

@@ -0,0 +1,249 @@
+require 'chewy/index/adapter/base'
+
+module Chewy
+  class Index
+    module Adapter
+      # This adapter provides an ability to import documents from any
+      # source. You can actually use any class or even a symbol as
+      # a target.
+      #
+      # In case if a class is used - some of the additional features
+      # are available: it is possible to provide the default import
+      # data (used on reset) and source objects loading logic.
+      #
+      # @see #import
+      # @see #load
+      class Object < Base
+        # The signature of the index scope definition.
+        #
+        # @example
+        #   index_scope :geoname
+        #   index_scope Geoname
+        #   index_scope -> { Geoname.all_the_places }, name: 'geoname'
+        #
+        # @param target [Class, Symbol, String, Proc] a source of data and everything
+        # @option options [String, Symbol] :name redefines the inferred name if necessary
+        # @option options [String, Symbol] :import_all_method redefines import method name
+        # @option options [String, Symbol] :load_all_method redefines batch load method name
+        # @option options [String, Symbol] :load_one_method redefines per-object load method name
+        def initialize(target, **options)
+          @target = target
+          @options = options
+        end
+
+        # Inferred from the target by default if possible.
+        #
+        # @example
+        #   # defines name = Geoname
+        #   index_scope :geoname
+        #   # still defines name = Geoname
+        #   index_scope -> { Geoname.all_the_places }, name: 'geoname'
+        #
+        # @return [String]
+        def name
+          @name ||= (options[:name] || @target).to_s.camelize.demodulize
+        end
+
+        # While for ORM adapters it returns an array of ids for the passed
+        # collection, for the object adapter it returns the collection itself.
+        #
+        # @param collection [Array<Object>, Object] a collection or an object
+        # @return [Array<Object>]
+        def identify(collection)
+          Array.wrap(collection)
+        end
+
+        # This method is used internally by `Chewy::Index.import`.
+        #
+        # The idea is that any object can be imported to ES if
+        # it responds to `#to_json` method.
+        #
+        # If method `destroyed?` is defined for object (or, in case of hash object,
+        # it has `:_destroyed` or `'_destroyed'` key) and returns `true` or object
+        # satisfy `delete_if` option then object will be deleted from index.
+        # But in order to be destroyable, objects need to respond to `id` method
+        # or have an `id` key so ElasticSearch could know which one to delete.
+        #
+        # If nothing is passed the method tries to call `import_all_method`,
+        # which is `call` by default, on target to get the default objects batch.
+        #
+        # @example
+        #   class Geoname
+        #     self < class
+        #       def self.call
+        #         FancyGeoAPI.all_points_collection
+        #       end
+        #       alias_method :import_all, :call
+        #     end
+        #   end
+        #
+        #   # All the following variants will work:
+        #   index_scope Geoname
+        #   index_scope Geoname, import_all_method: 'import_all'
+        #   index_scope -> { FancyGeoAPI.all_points_collection }, name: 'geoname'
+        #
+        # @param args [Array<#to_json>]
+        # @option options [Integer] :batch_size import processing batch size
+        # @return [true, false]
+        ruby2_keywords def import(*args, &block)
+          collection, options = import_args(*args)
+          import_objects(collection, options, &block)
+        end
+
+        # For the object adapter this method tries to fetch :id and requested
+        # fields from the passed collection or the target's `import_all_method`
+        # when defined. Otherwise it tries to call the target `pluck_method`,
+        # which is configurable and `pluck` by default. The `pluck_method` have
+        # to act exactly the same way as the AR one. It returns an empty array
+        # when none of the methods are found.
+        #
+        # @example
+        #   class Geoname
+        #     self < class
+        #       def self.pluck(*fields)
+        #         if fields.one?
+        #           whatever_source.map { |object| object.send(fields.first) }
+        #         else
+        #           whatever_source.map do |object|
+        #             fields.map { |field| object.send(field) }
+        #           end
+        #         end
+        #       end
+        #     end
+        #   end
+        #
+        # @see Chewy::Index::Adapter::Base#import_fields
+        ruby2_keywords def import_fields(*args, &block)
+          return enum_for(:import_fields, *args) unless block_given?
+
+          options = args.extract_options!
+          options[:batch_size] ||= BATCH_SIZE
+
+          if args.empty? && @target.respond_to?(pluck_method)
+            @target.send(pluck_method, :id, *options[:fields]).each_slice(options[:batch_size], &block)
+          elsif options[:fields].blank?
+            import_references(*args, options) do |batch|
+              yield batch.map { |object| object_field(object, :id) || object }
+            end
+          else
+            import_references(*args, options) do |batch|
+              batch = batch.map do |object|
+                options[:fields].map { |field| object_field(object, field) }
+                  .unshift(object_field(object, :id) || object)
+              end
+              yield batch
+            end
+          end
+        end
+
+        # For the Object adapter returns the objects themselves in batches.
+        #
+        # @see Chewy::Index::Adapter::Base#import_references
+        ruby2_keywords def import_references(*args, &block)
+          return enum_for(:import_references, *args) unless block_given?
+
+          collection, options = import_args(*args)
+          collection.each_slice(options[:batch_size], &block)
+        end
+
+        # This method is used internally by the request DSL when the
+        # collection of ORM/ODM objects is requested.
+        #
+        # Options usage is implemented by `load_all_method` and `load_one_method`.
+        #
+        # If none of the `load_all_method` or `load_one_method` is implemented
+        # for the target - the method will return nil. This means that the
+        # loader will return an array `Chewy::Index` objects that actually was passed.
+        #
+        # To use loading for objects it is obviously required to provide
+        # some meaningful ids for ES documents.
+        #
+        # @example
+        #   class Geoname
+        #     def self.load_all(wrappers, options)
+        #       if options[:additional_data]
+        #         wrappers.map do |wrapper|
+        #           FancyGeoAPI.point_by_name(wrapper.name)
+        #         end
+        #       else
+        #         wrappers
+        #       end
+        #     end
+        #   end
+        #
+        #   MyIndex.load(additional_data: true).objects
+        #
+        # @param ids [Array<Hash>] an array of ids from ES hits
+        # @param options [Hash] any options passed here with the request DSL `load` method.
+        # @return [Array<Object>, nil]
+        def load(ids, **options)
+          if target.respond_to?(load_all_method)
+            if target.method(load_all_method).arity == 1
+              target.send(load_all_method, ids)
+            else
+              target.send(load_all_method, ids, options)
+            end
+          elsif target.respond_to?(load_one_method)
+            if target.method(load_one_method).arity == 1
+              ids.map { |hit| target.send(load_one_method, hit) }
+            else
+              ids.map { |hit| target.send(load_one_method, hit, options) }
+            end
+          end
+        end
+
+      private
+
+        def import_objects(objects, options)
+          objects.each_slice(options[:batch_size]).map do |group|
+            yield grouped_objects(group)
+          end.all?
+        end
+
+        def delete_from_index?(object)
+          delete = super
+          delete ||= object.destroyed? if object.respond_to?(:destroyed?)
+          delete ||= object[:_destroyed] || object['_destroyed'] if object.is_a?(Hash)
+          !!delete
+        end
+
+        def object_field(object, name)
+          if object.respond_to?(name)
+            object.send(name)
+          elsif object.is_a?(Hash)
+            object[name.to_sym] || object[name.to_s]
+          end
+        end
+
+        def import_all_method
+          @import_all_method ||= options[:import_all_method] || :call
+        end
+
+        def pluck_method
+          @pluck_method ||= options[:pluck_method] || :pluck
+        end
+
+        def load_all_method
+          @load_all_method ||= options[:load_all_method] || :load_all
+        end
+
+        def load_one_method
+          @load_one_method ||= options[:load_one_method] || :load_one
+        end
+
+        def import_args(*args)
+          options = args.extract_options!
+          options[:batch_size] ||= BATCH_SIZE
+
+          collection = if args.empty? && @target.respond_to?(import_all_method)
+            @target.send(import_all_method)
+          else
+            args.flatten(1).compact
+          end
+
+          [collection, options]
+        end
+      end
+    end
+  end
+end

data/lib/chewy/index/adapter/orm.rb

@@ -0,0 +1,194 @@
+require 'chewy/index/adapter/base'
+
+module Chewy
+  class Index
+    module Adapter
+      class Orm < Base
+        attr_reader :default_scope
+
+        def initialize(target, **options)
+          if target.is_a?(relation_class)
+            @target = model_of_relation(target)
+            @default_scope = target
+          else
+            @target = target
+            @default_scope = all_scope
+          end
+          @options = options
+          cleanup_default_scope!
+        end
+
+        def name
+          @name ||= (options[:name].presence || target.name).to_s.camelize.demodulize
+        end
+
+        def identify(collection)
+          if collection.is_a?(relation_class)
+            pluck(collection)
+          else
+            Array.wrap(collection).map do |entity|
+              entity.respond_to?(primary_key) ? entity.public_send(primary_key) : entity
+            end
+          end
+        end
+
+        # Import method for ORM takes import data and import options
+        #
+        # Import data types:
+        #
+        #   * Nothing passed - imports all the model data according to type
+        #     default scope
+        #   * ORM scope
+        #   * Objects collection
+        #   * Ids collection
+        #
+        # Import options:
+        #
+        #   <tt>:batch_size</tt> - import batch size, 1000 objects by default
+        #   <tt>:direct_import</tt> - import objects without reloading
+        #
+        # Method handles destroyed objects as well. In case of objects ORM scope
+        # or array passed, objects, responding with true to `destroyed?` method will be deleted
+        # from index. In case of ids array passed - documents with missing source object ids will be
+        # deleted from index:
+        #
+        #   users = User.all
+        #   users.each { |user| user.destroy if user.inactive? }
+        #   UsersIndex.import users # inactive users will be deleted from index
+        #   # or
+        #   UsersIndex.import users.map(&:id) # deleted user ids will be deleted from index
+        #
+        # Also there is custom type option `delete_if`. It it returns `true`
+        # object will be deleted from index. Note that if this option is defined and
+        # return `false` Chewy will still check `destroyed?` method. This is useful
+        # for paranoid objects deleting implementation.
+        #
+        #   index_scope User, delete_if: ->{ deleted_at }
+        #   ...
+        #
+        #   users = User.all
+        #   users.each { |user| user.deleted_at = Time.now }
+        #   UsersIndex.import users # paranoid deleted users will be deleted from index
+        #   # or
+        #   UsersIndex.import users.map(&:id) # user ids will be deleted from index
+        #
+        ruby2_keywords def import(*args, &block)
+          collection, options = import_args(*args)
+
+          if !collection.is_a?(relation_class) || options[:direct_import]
+            import_objects(collection, options, &block)
+          else
+            import_scope(collection, options, &block)
+          end
+        end
+
+        ruby2_keywords def import_fields(*args, &block)
+          return enum_for(:import_fields, *args) unless block_given?
+
+          collection, options = import_args(*args)
+
+          if options[:fields].present? || collection.is_a?(relation_class)
+            collection = all_scope_where_ids_in(identify(collection)) unless collection.is_a?(relation_class)
+            pluck_in_batches(collection, **options.slice(:fields, :batch_size, :typecast), &block)
+          else
+            identify(collection).each_slice(options[:batch_size], &block)
+          end
+        end
+        alias_method :import_references, :import_fields
+
+        def load(ids, **options)
+          scope = all_scope_where_ids_in(ids)
+          additional_scope = options[options[:_index].to_sym].try(:[], :scope) || options[:scope]
+
+          loaded_objects = load_scope_objects(scope, additional_scope)
+          loaded_objects = raw(loaded_objects, options[:raw_import]) if options[:raw_import]
+
+          indexed_objects = loaded_objects.index_by do |object|
+            object.public_send(primary_key).to_s
+          end
+
+          ids.map { |id| indexed_objects[id.to_s] }
+        end
+
+      private
+
+        def import_objects(collection, options)
+          collection_ids = identify(collection)
+          hash = collection_ids.map(&:to_s).zip(collection).to_h
+
+          indexed = collection_ids.each_slice(options[:batch_size]).map do |ids|
+            batch = if options[:raw_import]
+              raw_default_scope_where_ids_in(ids, options[:raw_import])
+            elsif options[:direct_import]
+              hash.values_at(*ids.map(&:to_s))
+            else
+              default_scope_where_ids_in(ids)
+            end
+
+            if batch.empty?
+              true
+            else
+              batch.each { |object| hash.delete(object.send(primary_key).to_s) }
+              yield grouped_objects(batch)
+            end
+          end.all?
+
+          deleted = hash.keys.each_slice(options[:batch_size]).map do |group|
+            yield delete: hash.values_at(*group)
+          end.all?
+
+          indexed && deleted
+        end
+
+        def primary_key
+          :id
+        end
+
+        def default_scope_where_ids_in(ids)
+          scope_where_ids_in(default_scope, ids)
+        end
+
+        def all_scope_where_ids_in(ids)
+          scope_where_ids_in(all_scope, ids)
+        end
+
+        def all_scope
+          target.where(nil)
+        end
+
+        def model_of_relation(relation)
+          relation.klass
+        end
+
+        def load_scope_objects(scope, additional_scope = nil)
+          if additional_scope.is_a?(Proc)
+            scope.instance_exec(&additional_scope)
+          elsif additional_scope.is_a?(relation_class) && scope.respond_to?(:merge)
+            scope.merge(additional_scope)
+          else
+            scope
+          end
+        end
+
+        def grouped_objects(objects)
+          options[:delete_if] ? super : {index: objects.to_a}
+        end
+
+        def import_args(*args)
+          options = args.extract_options!
+          options[:batch_size] ||= BATCH_SIZE
+
+          collection = if args.empty?
+            default_scope
+          elsif args.one? && args.first.is_a?(relation_class)
+            args.first
+          else
+            args.flatten.compact
+          end
+
+          [collection, options]
+        end
+      end
+    end
+  end
+end

data/lib/chewy/index/aliases.rb

@@ -5,13 +5,23 @@ module Chewy
 
       module ClassMethods
         def indexes
-          client.indices.get_alias(name: index_name).keys
-        rescue Elasticsearch::Transport::Transport::Errors::NotFound
-          []
+          indexes = empty_if_not_found { client.indices.get(index: index_name).keys }
+          indexes += empty_if_not_found { client.indices.get_alias(name: index_name).keys }
+          indexes.compact.uniq
         end
 
         def aliases
-          client.indices.get_alias(index: index_name, name: '*')[index_name].try(:[], 'aliases').try(:keys) || []
+          empty_if_not_found do
+            client.indices.get_alias(index: index_name, name: '*').values.flat_map do |aliases|
+              aliases['aliases'].keys
+            end
+          end.compact.uniq
+        end
+
+      private
+
+        def empty_if_not_found
+          yield
         rescue Elasticsearch::Transport::Transport::Errors::NotFound
           []
         end

data/lib/chewy/index/crutch.rb

@@ -0,0 +1,40 @@
+module Chewy
+  class Index
+    module Crutch
+      extend ActiveSupport::Concern
+
+      included do
+        class_attribute :_crutches
+        self._crutches = {}
+      end
+
+      class Crutches
+        def initialize(index, collection)
+          @index = index
+          @collection = collection
+          @crutches_instances = {}
+        end
+
+        def method_missing(name, *, **)
+          return self[name] if @index._crutches.key?(name)
+
+          super
+        end
+
+        def respond_to_missing?(name, include_private = false)
+          @index._crutches.key?(name) || super
+        end
+
+        def [](name)
+          @crutches_instances[name] ||= @index._crutches[:"#{name}"].call(@collection)
+        end
+      end
+
+      module ClassMethods
+        def crutch(name, &block)
+          self._crutches = _crutches.merge(name.to_sym => block)
+        end
+      end
+    end
+  end
+end